1 //
    2 // Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // architecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182 
  183   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  184   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  185   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  186   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  187 
  188   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  189   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  190   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  191   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  192 
  193   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  194   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  195   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  196   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  197 
  198   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  199   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  200   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  201   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  202 
  203   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  204   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  205   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  206   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  207 
  208   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  209   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  210   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  211   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  212 
  213   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  214   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  215   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  216   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  217 
  218   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  219   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  220   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  221   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  222 
  223   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  224   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  225   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  226   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  227 
  228   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  229   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  230   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  231   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  232 
  233   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  234   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  235   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  236   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  237 
  238   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  239   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  240   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  241   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  242 
  243   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  244   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  245   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  246   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  247 
  248   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  249   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  250   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  251   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  252 
  253   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  254   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  255   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  256   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  257 
  258   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  259   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  260   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  261   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  262 
  263   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  264   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  265   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  266   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  267 
  268   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  269   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  270   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  271   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  272 
  273   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  274   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  275   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  276   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  277 
  278   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  279   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  280   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  281   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  282 
  283   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  284   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  285   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  286   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  287 
  288   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  289   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  290   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  291   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  292 
  293   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  294   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  295   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  296   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  297 
  298   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  299   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  300   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  301   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  302 
  303   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  304   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  305   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  306   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  307 
  308   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  309   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  310   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  311   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  312 
  313   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  314   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  315   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  316   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  317 
  318   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  319   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  320   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  321   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  322 
  323   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  324   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  325   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  326   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  327 
  328   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  329   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  330   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  331   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  332 
  333   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  334   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  335   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  336   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  337 
  338 // ----------------------------
  339 // SVE Predicate Registers
  340 // ----------------------------
  341   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  342   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  343   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  344   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  345   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  346   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  347   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  348   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  349   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  350   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  351   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  352   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  353   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  354   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  355   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  356   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  357 
  358 // ----------------------------
  359 // Special Registers
  360 // ----------------------------
  361 
  362 // the AArch64 CSPR status flag register is not directly accessible as
  363 // instruction operand. the FPSR status flag register is a system
  364 // register which can be written/read using MSR/MRS but again does not
  365 // appear as an operand (a code identifying the FSPR occurs as an
  366 // immediate value in the instruction).
  367 
  368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  369 
  370 // Specify priority of register selection within phases of register
  371 // allocation.  Highest priority is first.  A useful heuristic is to
  372 // give registers a low priority when they are required by machine
  373 // instructions, like EAX and EDX on I486, and choose no-save registers
  374 // before save-on-call, & save-on-call before save-on-entry.  Registers
  375 // which participate in fixed calling sequences should come last.
  376 // Registers which are used as pairs must fall on an even boundary.
  377 
  378 alloc_class chunk0(
  379     // volatiles
  380     R10, R10_H,
  381     R11, R11_H,
  382     R12, R12_H,
  383     R13, R13_H,
  384     R14, R14_H,
  385     R15, R15_H,
  386     R16, R16_H,
  387     R17, R17_H,
  388     R18, R18_H,
  389 
  390     // arg registers
  391     R0, R0_H,
  392     R1, R1_H,
  393     R2, R2_H,
  394     R3, R3_H,
  395     R4, R4_H,
  396     R5, R5_H,
  397     R6, R6_H,
  398     R7, R7_H,
  399 
  400     // non-volatiles
  401     R19, R19_H,
  402     R20, R20_H,
  403     R21, R21_H,
  404     R22, R22_H,
  405     R23, R23_H,
  406     R24, R24_H,
  407     R25, R25_H,
  408     R26, R26_H,
  409 
  410     // non-allocatable registers
  411 
  412     R27, R27_H, // heapbase
  413     R28, R28_H, // thread
  414     R29, R29_H, // fp
  415     R30, R30_H, // lr
  416     R31, R31_H, // sp
  417     R8, R8_H,   // rscratch1
  418     R9, R9_H,   // rscratch2
  419 );
  420 
  421 alloc_class chunk1(
  422 
  423     // no save
  424     V16, V16_H, V16_J, V16_K,
  425     V17, V17_H, V17_J, V17_K,
  426     V18, V18_H, V18_J, V18_K,
  427     V19, V19_H, V19_J, V19_K,
  428     V20, V20_H, V20_J, V20_K,
  429     V21, V21_H, V21_J, V21_K,
  430     V22, V22_H, V22_J, V22_K,
  431     V23, V23_H, V23_J, V23_K,
  432     V24, V24_H, V24_J, V24_K,
  433     V25, V25_H, V25_J, V25_K,
  434     V26, V26_H, V26_J, V26_K,
  435     V27, V27_H, V27_J, V27_K,
  436     V28, V28_H, V28_J, V28_K,
  437     V29, V29_H, V29_J, V29_K,
  438     V30, V30_H, V30_J, V30_K,
  439     V31, V31_H, V31_J, V31_K,
  440 
  441     // arg registers
  442     V0, V0_H, V0_J, V0_K,
  443     V1, V1_H, V1_J, V1_K,
  444     V2, V2_H, V2_J, V2_K,
  445     V3, V3_H, V3_J, V3_K,
  446     V4, V4_H, V4_J, V4_K,
  447     V5, V5_H, V5_J, V5_K,
  448     V6, V6_H, V6_J, V6_K,
  449     V7, V7_H, V7_J, V7_K,
  450 
  451     // non-volatiles
  452     V8, V8_H, V8_J, V8_K,
  453     V9, V9_H, V9_J, V9_K,
  454     V10, V10_H, V10_J, V10_K,
  455     V11, V11_H, V11_J, V11_K,
  456     V12, V12_H, V12_J, V12_K,
  457     V13, V13_H, V13_J, V13_K,
  458     V14, V14_H, V14_J, V14_K,
  459     V15, V15_H, V15_J, V15_K,
  460 );
  461 
  462 alloc_class chunk2 (
  463     // Governing predicates for load/store and arithmetic
  464     P0,
  465     P1,
  466     P2,
  467     P3,
  468     P4,
  469     P5,
  470     P6,
  471 
  472     // Extra predicates
  473     P8,
  474     P9,
  475     P10,
  476     P11,
  477     P12,
  478     P13,
  479     P14,
  480     P15,
  481 
  482     // Preserved for all-true predicate
  483     P7,
  484 );
  485 
  486 alloc_class chunk3(RFLAGS);
  487 
  488 //----------Architecture Description Register Classes--------------------------
  489 // Several register classes are automatically defined based upon information in
  490 // this architecture description.
  491 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  493 //
  494 
  495 // Class for all 32 bit general purpose registers
  496 reg_class all_reg32(
  497     R0,
  498     R1,
  499     R2,
  500     R3,
  501     R4,
  502     R5,
  503     R6,
  504     R7,
  505     R10,
  506     R11,
  507     R12,
  508     R13,
  509     R14,
  510     R15,
  511     R16,
  512     R17,
  513     R18,
  514     R19,
  515     R20,
  516     R21,
  517     R22,
  518     R23,
  519     R24,
  520     R25,
  521     R26,
  522     R27,
  523     R28,
  524     R29,
  525     R30,
  526     R31
  527 );
  528 
  529 
  530 // Class for all 32 bit integer registers (excluding SP which
  531 // will never be used as an integer register)
  532 reg_class any_reg32 %{
  533   return _ANY_REG32_mask;
  534 %}
  535 
  536 // Singleton class for R0 int register
  537 reg_class int_r0_reg(R0);
  538 
  539 // Singleton class for R2 int register
  540 reg_class int_r2_reg(R2);
  541 
  542 // Singleton class for R3 int register
  543 reg_class int_r3_reg(R3);
  544 
  545 // Singleton class for R4 int register
  546 reg_class int_r4_reg(R4);
  547 
  548 // Singleton class for R31 int register
  549 reg_class int_r31_reg(R31);
  550 
  551 // Class for all 64 bit general purpose registers
  552 reg_class all_reg(
  553     R0, R0_H,
  554     R1, R1_H,
  555     R2, R2_H,
  556     R3, R3_H,
  557     R4, R4_H,
  558     R5, R5_H,
  559     R6, R6_H,
  560     R7, R7_H,
  561     R10, R10_H,
  562     R11, R11_H,
  563     R12, R12_H,
  564     R13, R13_H,
  565     R14, R14_H,
  566     R15, R15_H,
  567     R16, R16_H,
  568     R17, R17_H,
  569     R18, R18_H,
  570     R19, R19_H,
  571     R20, R20_H,
  572     R21, R21_H,
  573     R22, R22_H,
  574     R23, R23_H,
  575     R24, R24_H,
  576     R25, R25_H,
  577     R26, R26_H,
  578     R27, R27_H,
  579     R28, R28_H,
  580     R29, R29_H,
  581     R30, R30_H,
  582     R31, R31_H
  583 );
  584 
  585 // Class for all long integer registers (including SP)
  586 reg_class any_reg %{
  587   return _ANY_REG_mask;
  588 %}
  589 
  590 // Class for non-allocatable 32 bit registers
  591 reg_class non_allocatable_reg32(
  592 #ifdef R18_RESERVED
  593     // See comment in register_aarch64.hpp
  594     R18,                        // tls on Windows
  595 #endif
  596     R28,                        // thread
  597     R30,                        // lr
  598     R31                         // sp
  599 );
  600 
  601 // Class for non-allocatable 64 bit registers
  602 reg_class non_allocatable_reg(
  603 #ifdef R18_RESERVED
  604     // See comment in register_aarch64.hpp
  605     R18, R18_H,                 // tls on Windows, platform register on macOS
  606 #endif
  607     R28, R28_H,                 // thread
  608     R30, R30_H,                 // lr
  609     R31, R31_H                  // sp
  610 );
  611 
  612 // Class for all non-special integer registers
  613 reg_class no_special_reg32 %{
  614   return _NO_SPECIAL_REG32_mask;
  615 %}
  616 
  617 // Class for all non-special long integer registers
  618 reg_class no_special_reg %{
  619   return _NO_SPECIAL_REG_mask;
  620 %}
  621 
  622 // Class for 64 bit register r0
  623 reg_class r0_reg(
  624     R0, R0_H
  625 );
  626 
  627 // Class for 64 bit register r1
  628 reg_class r1_reg(
  629     R1, R1_H
  630 );
  631 
  632 // Class for 64 bit register r2
  633 reg_class r2_reg(
  634     R2, R2_H
  635 );
  636 
  637 // Class for 64 bit register r3
  638 reg_class r3_reg(
  639     R3, R3_H
  640 );
  641 
  642 // Class for 64 bit register r4
  643 reg_class r4_reg(
  644     R4, R4_H
  645 );
  646 
  647 // Class for 64 bit register r5
  648 reg_class r5_reg(
  649     R5, R5_H
  650 );
  651 
  652 // Class for 64 bit register r10
  653 reg_class r10_reg(
  654     R10, R10_H
  655 );
  656 
  657 // Class for 64 bit register r11
  658 reg_class r11_reg(
  659     R11, R11_H
  660 );
  661 
  662 // Class for method register
  663 reg_class method_reg(
  664     R12, R12_H
  665 );
  666 
  667 // Class for thread register
  668 reg_class thread_reg(
  669     R28, R28_H
  670 );
  671 
  672 // Class for frame pointer register
  673 reg_class fp_reg(
  674     R29, R29_H
  675 );
  676 
  677 // Class for link register
  678 reg_class lr_reg(
  679     R30, R30_H
  680 );
  681 
  682 // Class for long sp register
  683 reg_class sp_reg(
  684   R31, R31_H
  685 );
  686 
  687 // Class for all pointer registers
  688 reg_class ptr_reg %{
  689   return _PTR_REG_mask;
  690 %}
  691 
  692 // Class for all non_special pointer registers
  693 reg_class no_special_ptr_reg %{
  694   return _NO_SPECIAL_PTR_REG_mask;
  695 %}
  696 
  697 // Class for all float registers
  698 reg_class float_reg(
  699     V0,
  700     V1,
  701     V2,
  702     V3,
  703     V4,
  704     V5,
  705     V6,
  706     V7,
  707     V8,
  708     V9,
  709     V10,
  710     V11,
  711     V12,
  712     V13,
  713     V14,
  714     V15,
  715     V16,
  716     V17,
  717     V18,
  718     V19,
  719     V20,
  720     V21,
  721     V22,
  722     V23,
  723     V24,
  724     V25,
  725     V26,
  726     V27,
  727     V28,
  728     V29,
  729     V30,
  730     V31
  731 );
  732 
  733 // Double precision float registers have virtual `high halves' that
  734 // are needed by the allocator.
  735 // Class for all double registers
  736 reg_class double_reg(
  737     V0, V0_H,
  738     V1, V1_H,
  739     V2, V2_H,
  740     V3, V3_H,
  741     V4, V4_H,
  742     V5, V5_H,
  743     V6, V6_H,
  744     V7, V7_H,
  745     V8, V8_H,
  746     V9, V9_H,
  747     V10, V10_H,
  748     V11, V11_H,
  749     V12, V12_H,
  750     V13, V13_H,
  751     V14, V14_H,
  752     V15, V15_H,
  753     V16, V16_H,
  754     V17, V17_H,
  755     V18, V18_H,
  756     V19, V19_H,
  757     V20, V20_H,
  758     V21, V21_H,
  759     V22, V22_H,
  760     V23, V23_H,
  761     V24, V24_H,
  762     V25, V25_H,
  763     V26, V26_H,
  764     V27, V27_H,
  765     V28, V28_H,
  766     V29, V29_H,
  767     V30, V30_H,
  768     V31, V31_H
  769 );
  770 
  771 // Class for all SVE vector registers.
  772 reg_class vectora_reg (
  773     V0, V0_H, V0_J, V0_K,
  774     V1, V1_H, V1_J, V1_K,
  775     V2, V2_H, V2_J, V2_K,
  776     V3, V3_H, V3_J, V3_K,
  777     V4, V4_H, V4_J, V4_K,
  778     V5, V5_H, V5_J, V5_K,
  779     V6, V6_H, V6_J, V6_K,
  780     V7, V7_H, V7_J, V7_K,
  781     V8, V8_H, V8_J, V8_K,
  782     V9, V9_H, V9_J, V9_K,
  783     V10, V10_H, V10_J, V10_K,
  784     V11, V11_H, V11_J, V11_K,
  785     V12, V12_H, V12_J, V12_K,
  786     V13, V13_H, V13_J, V13_K,
  787     V14, V14_H, V14_J, V14_K,
  788     V15, V15_H, V15_J, V15_K,
  789     V16, V16_H, V16_J, V16_K,
  790     V17, V17_H, V17_J, V17_K,
  791     V18, V18_H, V18_J, V18_K,
  792     V19, V19_H, V19_J, V19_K,
  793     V20, V20_H, V20_J, V20_K,
  794     V21, V21_H, V21_J, V21_K,
  795     V22, V22_H, V22_J, V22_K,
  796     V23, V23_H, V23_J, V23_K,
  797     V24, V24_H, V24_J, V24_K,
  798     V25, V25_H, V25_J, V25_K,
  799     V26, V26_H, V26_J, V26_K,
  800     V27, V27_H, V27_J, V27_K,
  801     V28, V28_H, V28_J, V28_K,
  802     V29, V29_H, V29_J, V29_K,
  803     V30, V30_H, V30_J, V30_K,
  804     V31, V31_H, V31_J, V31_K,
  805 );
  806 
  807 // Class for all 64bit vector registers
  808 reg_class vectord_reg(
  809     V0, V0_H,
  810     V1, V1_H,
  811     V2, V2_H,
  812     V3, V3_H,
  813     V4, V4_H,
  814     V5, V5_H,
  815     V6, V6_H,
  816     V7, V7_H,
  817     V8, V8_H,
  818     V9, V9_H,
  819     V10, V10_H,
  820     V11, V11_H,
  821     V12, V12_H,
  822     V13, V13_H,
  823     V14, V14_H,
  824     V15, V15_H,
  825     V16, V16_H,
  826     V17, V17_H,
  827     V18, V18_H,
  828     V19, V19_H,
  829     V20, V20_H,
  830     V21, V21_H,
  831     V22, V22_H,
  832     V23, V23_H,
  833     V24, V24_H,
  834     V25, V25_H,
  835     V26, V26_H,
  836     V27, V27_H,
  837     V28, V28_H,
  838     V29, V29_H,
  839     V30, V30_H,
  840     V31, V31_H
  841 );
  842 
  843 // Class for all 128bit vector registers
  844 reg_class vectorx_reg(
  845     V0, V0_H, V0_J, V0_K,
  846     V1, V1_H, V1_J, V1_K,
  847     V2, V2_H, V2_J, V2_K,
  848     V3, V3_H, V3_J, V3_K,
  849     V4, V4_H, V4_J, V4_K,
  850     V5, V5_H, V5_J, V5_K,
  851     V6, V6_H, V6_J, V6_K,
  852     V7, V7_H, V7_J, V7_K,
  853     V8, V8_H, V8_J, V8_K,
  854     V9, V9_H, V9_J, V9_K,
  855     V10, V10_H, V10_J, V10_K,
  856     V11, V11_H, V11_J, V11_K,
  857     V12, V12_H, V12_J, V12_K,
  858     V13, V13_H, V13_J, V13_K,
  859     V14, V14_H, V14_J, V14_K,
  860     V15, V15_H, V15_J, V15_K,
  861     V16, V16_H, V16_J, V16_K,
  862     V17, V17_H, V17_J, V17_K,
  863     V18, V18_H, V18_J, V18_K,
  864     V19, V19_H, V19_J, V19_K,
  865     V20, V20_H, V20_J, V20_K,
  866     V21, V21_H, V21_J, V21_K,
  867     V22, V22_H, V22_J, V22_K,
  868     V23, V23_H, V23_J, V23_K,
  869     V24, V24_H, V24_J, V24_K,
  870     V25, V25_H, V25_J, V25_K,
  871     V26, V26_H, V26_J, V26_K,
  872     V27, V27_H, V27_J, V27_K,
  873     V28, V28_H, V28_J, V28_K,
  874     V29, V29_H, V29_J, V29_K,
  875     V30, V30_H, V30_J, V30_K,
  876     V31, V31_H, V31_J, V31_K
  877 );
  878 
  879 // Class for 128 bit register v0
  880 reg_class v0_reg(
  881     V0, V0_H
  882 );
  883 
  884 // Class for 128 bit register v1
  885 reg_class v1_reg(
  886     V1, V1_H
  887 );
  888 
  889 // Class for 128 bit register v2
  890 reg_class v2_reg(
  891     V2, V2_H
  892 );
  893 
  894 // Class for 128 bit register v3
  895 reg_class v3_reg(
  896     V3, V3_H
  897 );
  898 
  899 // Class for 128 bit register v4
  900 reg_class v4_reg(
  901     V4, V4_H
  902 );
  903 
  904 // Class for 128 bit register v5
  905 reg_class v5_reg(
  906     V5, V5_H
  907 );
  908 
  909 // Class for 128 bit register v6
  910 reg_class v6_reg(
  911     V6, V6_H
  912 );
  913 
  914 // Class for 128 bit register v7
  915 reg_class v7_reg(
  916     V7, V7_H
  917 );
  918 
  919 // Class for 128 bit register v8
  920 reg_class v8_reg(
  921     V8, V8_H
  922 );
  923 
  924 // Class for 128 bit register v9
  925 reg_class v9_reg(
  926     V9, V9_H
  927 );
  928 
  929 // Class for 128 bit register v10
  930 reg_class v10_reg(
  931     V10, V10_H
  932 );
  933 
  934 // Class for 128 bit register v11
  935 reg_class v11_reg(
  936     V11, V11_H
  937 );
  938 
  939 // Class for 128 bit register v12
  940 reg_class v12_reg(
  941     V12, V12_H
  942 );
  943 
  944 // Class for 128 bit register v13
  945 reg_class v13_reg(
  946     V13, V13_H
  947 );
  948 
  949 // Class for 128 bit register v14
  950 reg_class v14_reg(
  951     V14, V14_H
  952 );
  953 
  954 // Class for 128 bit register v15
  955 reg_class v15_reg(
  956     V15, V15_H
  957 );
  958 
  959 // Class for 128 bit register v16
  960 reg_class v16_reg(
  961     V16, V16_H
  962 );
  963 
  964 // Class for 128 bit register v17
  965 reg_class v17_reg(
  966     V17, V17_H
  967 );
  968 
  969 // Class for 128 bit register v18
  970 reg_class v18_reg(
  971     V18, V18_H
  972 );
  973 
  974 // Class for 128 bit register v19
  975 reg_class v19_reg(
  976     V19, V19_H
  977 );
  978 
  979 // Class for 128 bit register v20
  980 reg_class v20_reg(
  981     V20, V20_H
  982 );
  983 
  984 // Class for 128 bit register v21
  985 reg_class v21_reg(
  986     V21, V21_H
  987 );
  988 
  989 // Class for 128 bit register v22
  990 reg_class v22_reg(
  991     V22, V22_H
  992 );
  993 
  994 // Class for 128 bit register v23
  995 reg_class v23_reg(
  996     V23, V23_H
  997 );
  998 
  999 // Class for 128 bit register v24
 1000 reg_class v24_reg(
 1001     V24, V24_H
 1002 );
 1003 
 1004 // Class for 128 bit register v25
 1005 reg_class v25_reg(
 1006     V25, V25_H
 1007 );
 1008 
 1009 // Class for 128 bit register v26
 1010 reg_class v26_reg(
 1011     V26, V26_H
 1012 );
 1013 
 1014 // Class for 128 bit register v27
 1015 reg_class v27_reg(
 1016     V27, V27_H
 1017 );
 1018 
 1019 // Class for 128 bit register v28
 1020 reg_class v28_reg(
 1021     V28, V28_H
 1022 );
 1023 
 1024 // Class for 128 bit register v29
 1025 reg_class v29_reg(
 1026     V29, V29_H
 1027 );
 1028 
 1029 // Class for 128 bit register v30
 1030 reg_class v30_reg(
 1031     V30, V30_H
 1032 );
 1033 
 1034 // Class for 128 bit register v31
 1035 reg_class v31_reg(
 1036     V31, V31_H
 1037 );
 1038 
 1039 // Class for all SVE predicate registers.
 1040 reg_class pr_reg (
 1041     P0,
 1042     P1,
 1043     P2,
 1044     P3,
 1045     P4,
 1046     P5,
 1047     P6,
 1048     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1049     P8,
 1050     P9,
 1051     P10,
 1052     P11,
 1053     P12,
 1054     P13,
 1055     P14,
 1056     P15
 1057 );
 1058 
 1059 // Class for SVE governing predicate registers, which are used
 1060 // to determine the active elements of a predicated instruction.
 1061 reg_class gov_pr (
 1062     P0,
 1063     P1,
 1064     P2,
 1065     P3,
 1066     P4,
 1067     P5,
 1068     P6,
 1069     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1070 );
 1071 
 1072 reg_class p0_reg(P0);
 1073 reg_class p1_reg(P1);
 1074 
 1075 // Singleton class for condition codes
 1076 reg_class int_flags(RFLAGS);
 1077 
 1078 %}
 1079 
 1080 //----------DEFINITION BLOCK---------------------------------------------------
 1081 // Define name --> value mappings to inform the ADLC of an integer valued name
 1082 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1083 // Format:
 1084 //        int_def  <name>         ( <int_value>, <expression>);
 1085 // Generated Code in ad_<arch>.hpp
 1086 //        #define  <name>   (<expression>)
 1087 //        // value == <int_value>
 1088 // Generated code in ad_<arch>.cpp adlc_verification()
 1089 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1090 //
 1091 
 1092 // we follow the ppc-aix port in using a simple cost model which ranks
 1093 // register operations as cheap, memory ops as more expensive and
 1094 // branches as most expensive. the first two have a low as well as a
 1095 // normal cost. huge cost appears to be a way of saying don't do
 1096 // something
 1097 
 1098 definitions %{
 1099   // The default cost (of a register move instruction).
 1100   int_def INSN_COST            (    100,     100);
 1101   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1102   int_def CALL_COST            (    200,     2 * INSN_COST);
 1103   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1104 %}
 1105 
 1106 
 1107 //----------SOURCE BLOCK-------------------------------------------------------
 1108 // This is a block of C++ code which provides values, functions, and
 1109 // definitions necessary in the rest of the architecture description
 1110 
 1111 source_hpp %{
 1112 
 1113 #include "asm/macroAssembler.hpp"
 1114 #include "gc/shared/barrierSetAssembler.hpp"
 1115 #include "gc/shared/cardTable.hpp"
 1116 #include "gc/shared/cardTableBarrierSet.hpp"
 1117 #include "gc/shared/collectedHeap.hpp"
 1118 #include "opto/addnode.hpp"
 1119 #include "opto/convertnode.hpp"
 1120 #include "runtime/objectMonitor.hpp"
 1121 
 1122 extern RegMask _ANY_REG32_mask;
 1123 extern RegMask _ANY_REG_mask;
 1124 extern RegMask _PTR_REG_mask;
 1125 extern RegMask _NO_SPECIAL_REG32_mask;
 1126 extern RegMask _NO_SPECIAL_REG_mask;
 1127 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1128 
 1129 class CallStubImpl {
 1130 
 1131   //--------------------------------------------------------------
 1132   //---<  Used for optimization in Compile::shorten_branches  >---
 1133   //--------------------------------------------------------------
 1134 
 1135  public:
 1136   // Size of call trampoline stub.
 1137   static uint size_call_trampoline() {
 1138     return 0; // no call trampolines on this platform
 1139   }
 1140 
 1141   // number of relocations needed by a call trampoline stub
 1142   static uint reloc_call_trampoline() {
 1143     return 0; // no call trampolines on this platform
 1144   }
 1145 };
 1146 
 1147 class HandlerImpl {
 1148 
 1149  public:
 1150 
 1151   static int emit_exception_handler(CodeBuffer &cbuf);
 1152   static int emit_deopt_handler(CodeBuffer& cbuf);
 1153 
 1154   static uint size_exception_handler() {
 1155     return MacroAssembler::far_codestub_branch_size();
 1156   }
 1157 
 1158   static uint size_deopt_handler() {
 1159     // count one adr and one far branch instruction
 1160     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1161   }
 1162 };
 1163 
 1164 class Node::PD {
 1165 public:
 1166   enum NodeFlags {
 1167     _last_flag = Node::_last_flag
 1168   };
 1169 };
 1170 
 1171   bool is_CAS(int opcode, bool maybe_volatile);
 1172 
 1173   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1174 
 1175   bool unnecessary_acquire(const Node *barrier);
 1176   bool needs_acquiring_load(const Node *load);
 1177 
 1178   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1179 
 1180   bool unnecessary_release(const Node *barrier);
 1181   bool unnecessary_volatile(const Node *barrier);
 1182   bool needs_releasing_store(const Node *store);
 1183 
 1184   // predicate controlling translation of CompareAndSwapX
 1185   bool needs_acquiring_load_exclusive(const Node *load);
 1186 
 1187   // predicate controlling addressing modes
 1188   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1189 
 1190   // Convert BootTest condition to Assembler condition.
 1191   // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 1192   Assembler::Condition to_assembler_cond(BoolTest::mask cond);
 1193 %}
 1194 
 1195 source %{
 1196 
 1197   // Derived RegMask with conditionally allocatable registers
 1198 
 1199   void PhaseOutput::pd_perform_mach_node_analysis() {
 1200   }
 1201 
 1202   int MachNode::pd_alignment_required() const {
 1203     return 1;
 1204   }
 1205 
 1206   int MachNode::compute_padding(int current_offset) const {
 1207     return 0;
 1208   }
 1209 
 1210   RegMask _ANY_REG32_mask;
 1211   RegMask _ANY_REG_mask;
 1212   RegMask _PTR_REG_mask;
 1213   RegMask _NO_SPECIAL_REG32_mask;
 1214   RegMask _NO_SPECIAL_REG_mask;
 1215   RegMask _NO_SPECIAL_PTR_REG_mask;
 1216 
 1217   void reg_mask_init() {
 1218     // We derive below RegMask(s) from the ones which are auto-generated from
 1219     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1220     // registers conditionally reserved.
 1221 
 1222     _ANY_REG32_mask = _ALL_REG32_mask;
 1223     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1224 
 1225     _ANY_REG_mask = _ALL_REG_mask;
 1226 
 1227     _PTR_REG_mask = _ALL_REG_mask;
 1228 
 1229     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1230     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1231 
 1232     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1233     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1234 
 1235     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1236     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1237 
 1238     // r27 is not allocatable when compressed oops is on and heapbase is not
 1239     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1240     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1241       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1242       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1243       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1244     }
 1245 
 1246     // r29 is not allocatable when PreserveFramePointer is on
 1247     if (PreserveFramePointer) {
 1248       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1249       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1250       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1251     }
 1252   }
 1253 
 1254   // Optimizaton of volatile gets and puts
 1255   // -------------------------------------
 1256   //
 1257   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1258   // use to implement volatile reads and writes. For a volatile read
 1259   // we simply need
 1260   //
 1261   //   ldar<x>
 1262   //
 1263   // and for a volatile write we need
 1264   //
 1265   //   stlr<x>
 1266   //
 1267   // Alternatively, we can implement them by pairing a normal
 1268   // load/store with a memory barrier. For a volatile read we need
 1269   //
 1270   //   ldr<x>
 1271   //   dmb ishld
 1272   //
 1273   // for a volatile write
 1274   //
 1275   //   dmb ish
 1276   //   str<x>
 1277   //   dmb ish
 1278   //
 1279   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1280   // sequences. These are normally translated to an instruction
 1281   // sequence like the following
 1282   //
 1283   //   dmb      ish
 1284   // retry:
 1285   //   ldxr<x>   rval raddr
 1286   //   cmp       rval rold
 1287   //   b.ne done
 1288   //   stlxr<x>  rval, rnew, rold
 1289   //   cbnz      rval retry
 1290   // done:
 1291   //   cset      r0, eq
 1292   //   dmb ishld
 1293   //
 1294   // Note that the exclusive store is already using an stlxr
 1295   // instruction. That is required to ensure visibility to other
 1296   // threads of the exclusive write (assuming it succeeds) before that
 1297   // of any subsequent writes.
 1298   //
 1299   // The following instruction sequence is an improvement on the above
 1300   //
 1301   // retry:
 1302   //   ldaxr<x>  rval raddr
 1303   //   cmp       rval rold
 1304   //   b.ne done
 1305   //   stlxr<x>  rval, rnew, rold
 1306   //   cbnz      rval retry
 1307   // done:
 1308   //   cset      r0, eq
 1309   //
 1310   // We don't need the leading dmb ish since the stlxr guarantees
 1311   // visibility of prior writes in the case that the swap is
 1312   // successful. Crucially we don't have to worry about the case where
 1313   // the swap is not successful since no valid program should be
 1314   // relying on visibility of prior changes by the attempting thread
 1315   // in the case where the CAS fails.
 1316   //
 1317   // Similarly, we don't need the trailing dmb ishld if we substitute
 1318   // an ldaxr instruction since that will provide all the guarantees we
 1319   // require regarding observation of changes made by other threads
 1320   // before any change to the CAS address observed by the load.
 1321   //
 1322   // In order to generate the desired instruction sequence we need to
 1323   // be able to identify specific 'signature' ideal graph node
 1324   // sequences which i) occur as a translation of a volatile reads or
 1325   // writes or CAS operations and ii) do not occur through any other
 1326   // translation or graph transformation. We can then provide
 1327   // alternative aldc matching rules which translate these node
 1328   // sequences to the desired machine code sequences. Selection of the
 1329   // alternative rules can be implemented by predicates which identify
 1330   // the relevant node sequences.
 1331   //
 1332   // The ideal graph generator translates a volatile read to the node
 1333   // sequence
 1334   //
 1335   //   LoadX[mo_acquire]
 1336   //   MemBarAcquire
 1337   //
 1338   // As a special case when using the compressed oops optimization we
 1339   // may also see this variant
 1340   //
 1341   //   LoadN[mo_acquire]
 1342   //   DecodeN
 1343   //   MemBarAcquire
 1344   //
 1345   // A volatile write is translated to the node sequence
 1346   //
 1347   //   MemBarRelease
 1348   //   StoreX[mo_release] {CardMark}-optional
 1349   //   MemBarVolatile
 1350   //
 1351   // n.b. the above node patterns are generated with a strict
 1352   // 'signature' configuration of input and output dependencies (see
 1353   // the predicates below for exact details). The card mark may be as
 1354   // simple as a few extra nodes or, in a few GC configurations, may
 1355   // include more complex control flow between the leading and
 1356   // trailing memory barriers. However, whatever the card mark
 1357   // configuration these signatures are unique to translated volatile
 1358   // reads/stores -- they will not appear as a result of any other
 1359   // bytecode translation or inlining nor as a consequence of
 1360   // optimizing transforms.
 1361   //
 1362   // We also want to catch inlined unsafe volatile gets and puts and
 1363   // be able to implement them using either ldar<x>/stlr<x> or some
 1364   // combination of ldr<x>/stlr<x> and dmb instructions.
 1365   //
 1366   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1367   // normal volatile put node sequence containing an extra cpuorder
 1368   // membar
 1369   //
 1370   //   MemBarRelease
 1371   //   MemBarCPUOrder
 1372   //   StoreX[mo_release] {CardMark}-optional
 1373   //   MemBarCPUOrder
 1374   //   MemBarVolatile
 1375   //
 1376   // n.b. as an aside, a cpuorder membar is not itself subject to
 1377   // matching and translation by adlc rules.  However, the rule
 1378   // predicates need to detect its presence in order to correctly
 1379   // select the desired adlc rules.
 1380   //
 1381   // Inlined unsafe volatile gets manifest as a slightly different
 1382   // node sequence to a normal volatile get because of the
 1383   // introduction of some CPUOrder memory barriers to bracket the
 1384   // Load. However, but the same basic skeleton of a LoadX feeding a
 1385   // MemBarAcquire, possibly through an optional DecodeN, is still
 1386   // present
 1387   //
 1388   //   MemBarCPUOrder
 1389   //        ||       \\
 1390   //   MemBarCPUOrder LoadX[mo_acquire]
 1391   //        ||            |
 1392   //        ||       {DecodeN} optional
 1393   //        ||       /
 1394   //     MemBarAcquire
 1395   //
 1396   // In this case the acquire membar does not directly depend on the
 1397   // load. However, we can be sure that the load is generated from an
 1398   // inlined unsafe volatile get if we see it dependent on this unique
 1399   // sequence of membar nodes. Similarly, given an acquire membar we
 1400   // can know that it was added because of an inlined unsafe volatile
 1401   // get if it is fed and feeds a cpuorder membar and if its feed
 1402   // membar also feeds an acquiring load.
 1403   //
 1404   // Finally an inlined (Unsafe) CAS operation is translated to the
 1405   // following ideal graph
 1406   //
 1407   //   MemBarRelease
 1408   //   MemBarCPUOrder
 1409   //   CompareAndSwapX {CardMark}-optional
 1410   //   MemBarCPUOrder
 1411   //   MemBarAcquire
 1412   //
 1413   // So, where we can identify these volatile read and write
 1414   // signatures we can choose to plant either of the above two code
 1415   // sequences. For a volatile read we can simply plant a normal
 1416   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1417   // also choose to inhibit translation of the MemBarAcquire and
 1418   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1419   //
 1420   // When we recognise a volatile store signature we can choose to
 1421   // plant at a dmb ish as a translation for the MemBarRelease, a
 1422   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1423   // Alternatively, we can inhibit translation of the MemBarRelease
 1424   // and MemBarVolatile and instead plant a simple stlr<x>
 1425   // instruction.
 1426   //
 1427   // when we recognise a CAS signature we can choose to plant a dmb
 1428   // ish as a translation for the MemBarRelease, the conventional
 1429   // macro-instruction sequence for the CompareAndSwap node (which
 1430   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1431   // Alternatively, we can elide generation of the dmb instructions
 1432   // and plant the alternative CompareAndSwap macro-instruction
 1433   // sequence (which uses ldaxr<x>).
 1434   //
 1435   // Of course, the above only applies when we see these signature
 1436   // configurations. We still want to plant dmb instructions in any
 1437   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1438   // MemBarVolatile. For example, at the end of a constructor which
 1439   // writes final/volatile fields we will see a MemBarRelease
 1440   // instruction and this needs a 'dmb ish' lest we risk the
 1441   // constructed object being visible without making the
 1442   // final/volatile field writes visible.
 1443   //
 1444   // n.b. the translation rules below which rely on detection of the
 1445   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1446   // If we see anything other than the signature configurations we
 1447   // always just translate the loads and stores to ldr<x> and str<x>
 1448   // and translate acquire, release and volatile membars to the
 1449   // relevant dmb instructions.
 1450   //
 1451 
 1452   // is_CAS(int opcode, bool maybe_volatile)
 1453   //
 1454   // return true if opcode is one of the possible CompareAndSwapX
 1455   // values otherwise false.
 1456 
 1457   bool is_CAS(int opcode, bool maybe_volatile)
 1458   {
 1459     switch(opcode) {
 1460       // We handle these
 1461     case Op_CompareAndSwapI:
 1462     case Op_CompareAndSwapL:
 1463     case Op_CompareAndSwapP:
 1464     case Op_CompareAndSwapN:
 1465     case Op_ShenandoahCompareAndSwapP:
 1466     case Op_ShenandoahCompareAndSwapN:
 1467     case Op_CompareAndSwapB:
 1468     case Op_CompareAndSwapS:
 1469     case Op_GetAndSetI:
 1470     case Op_GetAndSetL:
 1471     case Op_GetAndSetP:
 1472     case Op_GetAndSetN:
 1473     case Op_GetAndAddI:
 1474     case Op_GetAndAddL:
 1475       return true;
 1476     case Op_CompareAndExchangeI:
 1477     case Op_CompareAndExchangeN:
 1478     case Op_CompareAndExchangeB:
 1479     case Op_CompareAndExchangeS:
 1480     case Op_CompareAndExchangeL:
 1481     case Op_CompareAndExchangeP:
 1482     case Op_WeakCompareAndSwapB:
 1483     case Op_WeakCompareAndSwapS:
 1484     case Op_WeakCompareAndSwapI:
 1485     case Op_WeakCompareAndSwapL:
 1486     case Op_WeakCompareAndSwapP:
 1487     case Op_WeakCompareAndSwapN:
 1488     case Op_ShenandoahWeakCompareAndSwapP:
 1489     case Op_ShenandoahWeakCompareAndSwapN:
 1490     case Op_ShenandoahCompareAndExchangeP:
 1491     case Op_ShenandoahCompareAndExchangeN:
 1492       return maybe_volatile;
 1493     default:
 1494       return false;
 1495     }
 1496   }
 1497 
 1498   // helper to determine the maximum number of Phi nodes we may need to
 1499   // traverse when searching from a card mark membar for the merge mem
 1500   // feeding a trailing membar or vice versa
 1501 
 1502 // predicates controlling emit of ldr<x>/ldar<x>
 1503 
 1504 bool unnecessary_acquire(const Node *barrier)
 1505 {
 1506   assert(barrier->is_MemBar(), "expecting a membar");
 1507 
 1508   MemBarNode* mb = barrier->as_MemBar();
 1509 
 1510   if (mb->trailing_load()) {
 1511     return true;
 1512   }
 1513 
 1514   if (mb->trailing_load_store()) {
 1515     Node* load_store = mb->in(MemBarNode::Precedent);
 1516     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1517     return is_CAS(load_store->Opcode(), true);
 1518   }
 1519 
 1520   return false;
 1521 }
 1522 
 1523 bool needs_acquiring_load(const Node *n)
 1524 {
 1525   assert(n->is_Load(), "expecting a load");
 1526   LoadNode *ld = n->as_Load();
 1527   return ld->is_acquire();
 1528 }
 1529 
 1530 bool unnecessary_release(const Node *n)
 1531 {
 1532   assert((n->is_MemBar() &&
 1533           n->Opcode() == Op_MemBarRelease),
 1534          "expecting a release membar");
 1535 
 1536   MemBarNode *barrier = n->as_MemBar();
 1537   if (!barrier->leading()) {
 1538     return false;
 1539   } else {
 1540     Node* trailing = barrier->trailing_membar();
 1541     MemBarNode* trailing_mb = trailing->as_MemBar();
 1542     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1543     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1544 
 1545     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1546     if (mem->is_Store()) {
 1547       assert(mem->as_Store()->is_release(), "");
 1548       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1549       return true;
 1550     } else {
 1551       assert(mem->is_LoadStore(), "");
 1552       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1553       return is_CAS(mem->Opcode(), true);
 1554     }
 1555   }
 1556   return false;
 1557 }
 1558 
 1559 bool unnecessary_volatile(const Node *n)
 1560 {
 1561   // assert n->is_MemBar();
 1562   MemBarNode *mbvol = n->as_MemBar();
 1563 
 1564   bool release = mbvol->trailing_store();
 1565   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1566 #ifdef ASSERT
 1567   if (release) {
 1568     Node* leading = mbvol->leading_membar();
 1569     assert(leading->Opcode() == Op_MemBarRelease, "");
 1570     assert(leading->as_MemBar()->leading_store(), "");
 1571     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1572   }
 1573 #endif
 1574 
 1575   return release;
 1576 }
 1577 
 1578 // predicates controlling emit of str<x>/stlr<x>
 1579 
 1580 bool needs_releasing_store(const Node *n)
 1581 {
 1582   // assert n->is_Store();
 1583   StoreNode *st = n->as_Store();
 1584   return st->trailing_membar() != NULL;
 1585 }
 1586 
 1587 // predicate controlling translation of CAS
 1588 //
 1589 // returns true if CAS needs to use an acquiring load otherwise false
 1590 
 1591 bool needs_acquiring_load_exclusive(const Node *n)
 1592 {
 1593   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1594   LoadStoreNode* ldst = n->as_LoadStore();
 1595   if (is_CAS(n->Opcode(), false)) {
 1596     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1597   } else {
 1598     return ldst->trailing_membar() != NULL;
 1599   }
 1600 
 1601   // so we can just return true here
 1602   return true;
 1603 }
 1604 
 1605 #define __ _masm.
 1606 
 1607 // advance declarations for helper functions to convert register
 1608 // indices to register objects
 1609 
 1610 // the ad file has to provide implementations of certain methods
 1611 // expected by the generic code
 1612 //
 1613 // REQUIRED FUNCTIONALITY
 1614 
 1615 //=============================================================================
 1616 
 1617 // !!!!! Special hack to get all types of calls to specify the byte offset
 1618 //       from the start of the call to the point where the return address
 1619 //       will point.
 1620 
 1621 int MachCallStaticJavaNode::ret_addr_offset()
 1622 {
 1623   // call should be a simple bl
 1624   int off = 4;
 1625   return off;
 1626 }
 1627 
 1628 int MachCallDynamicJavaNode::ret_addr_offset()
 1629 {
 1630   return 16; // movz, movk, movk, bl
 1631 }
 1632 
 1633 int MachCallRuntimeNode::ret_addr_offset() {
 1634   // for generated stubs the call will be
 1635   //   bl(addr)
 1636   // or with far branches
 1637   //   bl(trampoline_stub)
 1638   // for real runtime callouts it will be six instructions
 1639   // see aarch64_enc_java_to_runtime
 1640   //   adr(rscratch2, retaddr)
 1641   //   lea(rscratch1, RuntimeAddress(addr)
 1642   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1643   //   blr(rscratch1)
 1644   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1645   if (cb) {
 1646     return 1 * NativeInstruction::instruction_size;
 1647   } else {
 1648     return 6 * NativeInstruction::instruction_size;
 1649   }
 1650 }
 1651 
 1652 //=============================================================================
 1653 
 1654 #ifndef PRODUCT
 1655 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1656   st->print("BREAKPOINT");
 1657 }
 1658 #endif
 1659 
 1660 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1661   C2_MacroAssembler _masm(&cbuf);
 1662   __ brk(0);
 1663 }
 1664 
 1665 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1666   return MachNode::size(ra_);
 1667 }
 1668 
 1669 //=============================================================================
 1670 
 1671 #ifndef PRODUCT
 1672   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1673     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1674   }
 1675 #endif
 1676 
 1677   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1678     C2_MacroAssembler _masm(&cbuf);
 1679     for (int i = 0; i < _count; i++) {
 1680       __ nop();
 1681     }
 1682   }
 1683 
 1684   uint MachNopNode::size(PhaseRegAlloc*) const {
 1685     return _count * NativeInstruction::instruction_size;
 1686   }
 1687 
 1688 //=============================================================================
 1689 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1690 
 1691 int ConstantTable::calculate_table_base_offset() const {
 1692   return 0;  // absolute addressing, no offset
 1693 }
 1694 
 1695 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1696 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1697   ShouldNotReachHere();
 1698 }
 1699 
 1700 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1701   // Empty encoding
 1702 }
 1703 
 1704 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1705   return 0;
 1706 }
 1707 
 1708 #ifndef PRODUCT
 1709 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1710   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1711 }
 1712 #endif
 1713 
 1714 #ifndef PRODUCT
 1715 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1716   Compile* C = ra_->C;
 1717 
 1718   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1719 
 1720   if (C->output()->need_stack_bang(framesize))
 1721     st->print("# stack bang size=%d\n\t", framesize);
 1722 
 1723   if (VM_Version::use_rop_protection()) {
 1724     st->print("ldr zr, [lr]\n\t");
 1725     st->print("pacia  lr, rfp\n\t");
 1726   }
 1727   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1728     st->print("sub  sp, sp, #%d\n\t", framesize);
 1729     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1730     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1731   } else {
 1732     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1733     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1734     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1735     st->print("sub  sp, sp, rscratch1");
 1736   }
 1737   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1738     st->print("\n\t");
 1739     st->print("ldr  rscratch1, [guard]\n\t");
 1740     st->print("dmb ishld\n\t");
 1741     st->print("ldr  rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
 1742     st->print("cmp  rscratch1, rscratch2\n\t");
 1743     st->print("b.eq skip");
 1744     st->print("\n\t");
 1745     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1746     st->print("b skip\n\t");
 1747     st->print("guard: int\n\t");
 1748     st->print("\n\t");
 1749     st->print("skip:\n\t");
 1750   }
 1751 }
 1752 #endif
 1753 
 1754 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1755   Compile* C = ra_->C;
 1756   C2_MacroAssembler _masm(&cbuf);
 1757 
 1758   // n.b. frame size includes space for return pc and rfp
 1759   const int framesize = C->output()->frame_size_in_bytes();
 1760 
 1761   // insert a nop at the start of the prolog so we can patch in a
 1762   // branch if we need to invalidate the method later
 1763   __ nop();
 1764 
 1765   if (C->clinit_barrier_on_entry()) {
 1766     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1767 
 1768     Label L_skip_barrier;
 1769 
 1770     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1771     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1772     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1773     __ bind(L_skip_barrier);
 1774   }
 1775 
 1776   if (C->max_vector_size() > 0) {
 1777     __ reinitialize_ptrue();
 1778   }
 1779 
 1780   int bangsize = C->output()->bang_size_in_bytes();
 1781   if (C->output()->need_stack_bang(bangsize))
 1782     __ generate_stack_overflow_check(bangsize);
 1783 
 1784   __ build_frame(framesize);
 1785 
 1786   if (C->stub_function() == NULL) {
 1787     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1788     if (BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1789       // Dummy labels for just measuring the code size
 1790       Label dummy_slow_path;
 1791       Label dummy_continuation;
 1792       Label dummy_guard;
 1793       Label* slow_path = &dummy_slow_path;
 1794       Label* continuation = &dummy_continuation;
 1795       Label* guard = &dummy_guard;
 1796       if (!Compile::current()->output()->in_scratch_emit_size()) {
 1797         // Use real labels from actual stub when not emitting code for the purpose of measuring its size
 1798         C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
 1799         Compile::current()->output()->add_stub(stub);
 1800         slow_path = &stub->entry();
 1801         continuation = &stub->continuation();
 1802         guard = &stub->guard();
 1803       }
 1804       // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
 1805       bs->nmethod_entry_barrier(&_masm, slow_path, continuation, guard);
 1806     }
 1807   }
 1808 
 1809   if (VerifyStackAtCalls) {
 1810     Unimplemented();
 1811   }
 1812 
 1813   C->output()->set_frame_complete(cbuf.insts_size());
 1814 
 1815   if (C->has_mach_constant_base_node()) {
 1816     // NOTE: We set the table base offset here because users might be
 1817     // emitted before MachConstantBaseNode.
 1818     ConstantTable& constant_table = C->output()->constant_table();
 1819     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1820   }
 1821 }
 1822 
 1823 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1824 {
 1825   return MachNode::size(ra_); // too many variables; just compute it
 1826                               // the hard way
 1827 }
 1828 
 1829 int MachPrologNode::reloc() const
 1830 {
 1831   return 0;
 1832 }
 1833 
 1834 //=============================================================================
 1835 
 1836 #ifndef PRODUCT
 1837 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1838   Compile* C = ra_->C;
 1839   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1840 
 1841   st->print("# pop frame %d\n\t",framesize);
 1842 
 1843   if (framesize == 0) {
 1844     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1845   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1846     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1847     st->print("add  sp, sp, #%d\n\t", framesize);
 1848   } else {
 1849     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1850     st->print("add  sp, sp, rscratch1\n\t");
 1851     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1852   }
 1853   if (VM_Version::use_rop_protection()) {
 1854     st->print("autia lr, rfp\n\t");
 1855     st->print("ldr zr, [lr]\n\t");
 1856   }
 1857 
 1858   if (do_polling() && C->is_method_compilation()) {
 1859     st->print("# test polling word\n\t");
 1860     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1861     st->print("cmp  sp, rscratch1\n\t");
 1862     st->print("bhi #slow_path");
 1863   }
 1864 }
 1865 #endif
 1866 
 1867 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1868   Compile* C = ra_->C;
 1869   C2_MacroAssembler _masm(&cbuf);
 1870   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1871 
 1872   __ remove_frame(framesize);
 1873 
 1874   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1875     __ reserved_stack_check();
 1876   }
 1877 
 1878   if (do_polling() && C->is_method_compilation()) {
 1879     Label dummy_label;
 1880     Label* code_stub = &dummy_label;
 1881     if (!C->output()->in_scratch_emit_size()) {
 1882       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
 1883       C->output()->add_stub(stub);
 1884       code_stub = &stub->entry();
 1885     }
 1886     __ relocate(relocInfo::poll_return_type);
 1887     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1888   }
 1889 }
 1890 
 1891 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 1892   // Variable size. Determine dynamically.
 1893   return MachNode::size(ra_);
 1894 }
 1895 
 1896 int MachEpilogNode::reloc() const {
 1897   // Return number of relocatable values contained in this instruction.
 1898   return 1; // 1 for polling page.
 1899 }
 1900 
 1901 const Pipeline * MachEpilogNode::pipeline() const {
 1902   return MachNode::pipeline_class();
 1903 }
 1904 
 1905 //=============================================================================
 1906 
 1907 // Figure out which register class each belongs in: rc_int, rc_float or
 1908 // rc_stack.
 1909 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 1910 
 1911 static enum RC rc_class(OptoReg::Name reg) {
 1912 
 1913   if (reg == OptoReg::Bad) {
 1914     return rc_bad;
 1915   }
 1916 
 1917   // we have 32 int registers * 2 halves
 1918   int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
 1919 
 1920   if (reg < slots_of_int_registers) {
 1921     return rc_int;
 1922   }
 1923 
 1924   // we have 32 float register * 8 halves
 1925   int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
 1926   if (reg < slots_of_int_registers + slots_of_float_registers) {
 1927     return rc_float;
 1928   }
 1929 
 1930   int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
 1931   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 1932     return rc_predicate;
 1933   }
 1934 
 1935   // Between predicate regs & stack is the flags.
 1936   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 1937 
 1938   return rc_stack;
 1939 }
 1940 
 1941 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 1942   Compile* C = ra_->C;
 1943 
 1944   // Get registers to move.
 1945   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 1946   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 1947   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 1948   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 1949 
 1950   enum RC src_hi_rc = rc_class(src_hi);
 1951   enum RC src_lo_rc = rc_class(src_lo);
 1952   enum RC dst_hi_rc = rc_class(dst_hi);
 1953   enum RC dst_lo_rc = rc_class(dst_lo);
 1954 
 1955   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 1956 
 1957   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 1958     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 1959            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 1960            "expected aligned-adjacent pairs");
 1961   }
 1962 
 1963   if (src_lo == dst_lo && src_hi == dst_hi) {
 1964     return 0;            // Self copy, no move.
 1965   }
 1966 
 1967   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 1968               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 1969   int src_offset = ra_->reg2offset(src_lo);
 1970   int dst_offset = ra_->reg2offset(dst_lo);
 1971 
 1972   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 1973     uint ireg = ideal_reg();
 1974     if (ireg == Op_VecA && cbuf) {
 1975       C2_MacroAssembler _masm(cbuf);
 1976       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 1977       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1978         // stack->stack
 1979         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 1980                                                 sve_vector_reg_size_in_bytes);
 1981       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 1982         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 1983                             sve_vector_reg_size_in_bytes);
 1984       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 1985         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 1986                               sve_vector_reg_size_in_bytes);
 1987       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 1988         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1989                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 1990                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 1991       } else {
 1992         ShouldNotReachHere();
 1993       }
 1994     } else if (cbuf) {
 1995       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 1996       C2_MacroAssembler _masm(cbuf);
 1997       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 1998       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1999         // stack->stack
 2000         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2001         if (ireg == Op_VecD) {
 2002           __ unspill(rscratch1, true, src_offset);
 2003           __ spill(rscratch1, true, dst_offset);
 2004         } else {
 2005           __ spill_copy128(src_offset, dst_offset);
 2006         }
 2007       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2008         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2009                ireg == Op_VecD ? __ T8B : __ T16B,
 2010                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2011       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2012         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2013                  ireg == Op_VecD ? __ D : __ Q,
 2014                  ra_->reg2offset(dst_lo));
 2015       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2016         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2017                    ireg == Op_VecD ? __ D : __ Q,
 2018                    ra_->reg2offset(src_lo));
 2019       } else {
 2020         ShouldNotReachHere();
 2021       }
 2022     }
 2023   } else if (cbuf) {
 2024     C2_MacroAssembler _masm(cbuf);
 2025     switch (src_lo_rc) {
 2026     case rc_int:
 2027       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2028         if (is64) {
 2029             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2030                    as_Register(Matcher::_regEncode[src_lo]));
 2031         } else {
 2032             C2_MacroAssembler _masm(cbuf);
 2033             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2034                     as_Register(Matcher::_regEncode[src_lo]));
 2035         }
 2036       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2037         if (is64) {
 2038             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2039                      as_Register(Matcher::_regEncode[src_lo]));
 2040         } else {
 2041             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2042                      as_Register(Matcher::_regEncode[src_lo]));
 2043         }
 2044       } else {                    // gpr --> stack spill
 2045         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2046         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2047       }
 2048       break;
 2049     case rc_float:
 2050       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2051         if (is64) {
 2052             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2053                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2054         } else {
 2055             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2056                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2057         }
 2058       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2059         if (is64) {
 2060             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2061                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2062         } else {
 2063             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2064                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2065         }
 2066       } else {                    // fpr --> stack spill
 2067         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2068         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2069                  is64 ? __ D : __ S, dst_offset);
 2070       }
 2071       break;
 2072     case rc_stack:
 2073       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2074         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2075       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2076         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2077                    is64 ? __ D : __ S, src_offset);
 2078       } else if (dst_lo_rc == rc_predicate) {
 2079         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2080                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2081       } else {                    // stack --> stack copy
 2082         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2083         if (ideal_reg() == Op_RegVectMask) {
 2084           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2085                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2086         } else {
 2087           __ unspill(rscratch1, is64, src_offset);
 2088           __ spill(rscratch1, is64, dst_offset);
 2089         }
 2090       }
 2091       break;
 2092     case rc_predicate:
 2093       if (dst_lo_rc == rc_predicate) {
 2094         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2095       } else if (dst_lo_rc == rc_stack) {
 2096         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2097                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2098       } else {
 2099         assert(false, "bad src and dst rc_class combination.");
 2100         ShouldNotReachHere();
 2101       }
 2102       break;
 2103     default:
 2104       assert(false, "bad rc_class for spill");
 2105       ShouldNotReachHere();
 2106     }
 2107   }
 2108 
 2109   if (st) {
 2110     st->print("spill ");
 2111     if (src_lo_rc == rc_stack) {
 2112       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2113     } else {
 2114       st->print("%s -> ", Matcher::regName[src_lo]);
 2115     }
 2116     if (dst_lo_rc == rc_stack) {
 2117       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2118     } else {
 2119       st->print("%s", Matcher::regName[dst_lo]);
 2120     }
 2121     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2122       int vsize = 0;
 2123       switch (ideal_reg()) {
 2124       case Op_VecD:
 2125         vsize = 64;
 2126         break;
 2127       case Op_VecX:
 2128         vsize = 128;
 2129         break;
 2130       case Op_VecA:
 2131         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2132         break;
 2133       default:
 2134         assert(false, "bad register type for spill");
 2135         ShouldNotReachHere();
 2136       }
 2137       st->print("\t# vector spill size = %d", vsize);
 2138     } else if (ideal_reg() == Op_RegVectMask) {
 2139       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2140       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2141       st->print("\t# predicate spill size = %d", vsize);
 2142     } else {
 2143       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2144     }
 2145   }
 2146 
 2147   return 0;
 2148 
 2149 }
 2150 
 2151 #ifndef PRODUCT
 2152 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2153   if (!ra_)
 2154     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2155   else
 2156     implementation(NULL, ra_, false, st);
 2157 }
 2158 #endif
 2159 
 2160 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2161   implementation(&cbuf, ra_, false, NULL);
 2162 }
 2163 
 2164 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2165   return MachNode::size(ra_);
 2166 }
 2167 
 2168 //=============================================================================
 2169 
 2170 #ifndef PRODUCT
 2171 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2172   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2173   int reg = ra_->get_reg_first(this);
 2174   st->print("add %s, rsp, #%d]\t# box lock",
 2175             Matcher::regName[reg], offset);
 2176 }
 2177 #endif
 2178 
 2179 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2180   C2_MacroAssembler _masm(&cbuf);
 2181 
 2182   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2183   int reg    = ra_->get_encode(this);
 2184 
 2185   // This add will handle any 24-bit signed offset. 24 bits allows an
 2186   // 8 megabyte stack frame.
 2187   __ add(as_Register(reg), sp, offset);
 2188 }
 2189 
 2190 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2191   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2192   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2193 
 2194   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2195     return NativeInstruction::instruction_size;
 2196   } else {
 2197     return 2 * NativeInstruction::instruction_size;
 2198   }
 2199 }
 2200 
 2201 //=============================================================================
 2202 
 2203 #ifndef PRODUCT
 2204 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2205 {
 2206   st->print_cr("# MachUEPNode");
 2207   if (UseCompressedClassPointers) {
 2208     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2209     if (CompressedKlassPointers::shift() != 0) {
 2210       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2211     }
 2212   } else {
 2213    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2214   }
 2215   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2216   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2217 }
 2218 #endif
 2219 
 2220 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2221 {
 2222   // This is the unverified entry point.
 2223   C2_MacroAssembler _masm(&cbuf);
 2224 
 2225   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2226   Label skip;
 2227   // TODO
 2228   // can we avoid this skip and still use a reloc?
 2229   __ br(Assembler::EQ, skip);
 2230   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2231   __ bind(skip);
 2232 }
 2233 
 2234 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2235 {
 2236   return MachNode::size(ra_);
 2237 }
 2238 
 2239 // REQUIRED EMIT CODE
 2240 
 2241 //=============================================================================
 2242 
 2243 // Emit exception handler code.
 2244 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2245 {
 2246   // mov rscratch1 #exception_blob_entry_point
 2247   // br rscratch1
 2248   // Note that the code buffer's insts_mark is always relative to insts.
 2249   // That's why we must use the macroassembler to generate a handler.
 2250   C2_MacroAssembler _masm(&cbuf);
 2251   address base = __ start_a_stub(size_exception_handler());
 2252   if (base == NULL) {
 2253     ciEnv::current()->record_failure("CodeCache is full");
 2254     return 0;  // CodeBuffer::expand failed
 2255   }
 2256   int offset = __ offset();
 2257   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2258   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2259   __ end_a_stub();
 2260   return offset;
 2261 }
 2262 
 2263 // Emit deopt handler code.
 2264 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2265 {
 2266   // Note that the code buffer's insts_mark is always relative to insts.
 2267   // That's why we must use the macroassembler to generate a handler.
 2268   C2_MacroAssembler _masm(&cbuf);
 2269   address base = __ start_a_stub(size_deopt_handler());
 2270   if (base == NULL) {
 2271     ciEnv::current()->record_failure("CodeCache is full");
 2272     return 0;  // CodeBuffer::expand failed
 2273   }
 2274   int offset = __ offset();
 2275 
 2276   __ adr(lr, __ pc());
 2277   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2278 
 2279   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2280   __ end_a_stub();
 2281   return offset;
 2282 }
 2283 
 2284 // REQUIRED MATCHER CODE
 2285 
 2286 //=============================================================================
 2287 
 2288 const bool Matcher::match_rule_supported(int opcode) {
 2289   if (!has_match_rule(opcode))
 2290     return false;
 2291 
 2292   bool ret_value = true;
 2293   switch (opcode) {
 2294     case Op_OnSpinWait:
 2295       return VM_Version::supports_on_spin_wait();
 2296     case Op_CacheWB:
 2297     case Op_CacheWBPreSync:
 2298     case Op_CacheWBPostSync:
 2299       if (!VM_Version::supports_data_cache_line_flush()) {
 2300         ret_value = false;
 2301       }
 2302       break;
 2303     case Op_ExpandBits:
 2304     case Op_CompressBits:
 2305       if (!(UseSVE > 1 && VM_Version::supports_svebitperm())) {
 2306         ret_value = false;
 2307       }
 2308       break;
 2309   }
 2310 
 2311   return ret_value; // Per default match rules are supported.
 2312 }
 2313 
 2314 const RegMask* Matcher::predicate_reg_mask(void) {
 2315   return &_PR_REG_mask;
 2316 }
 2317 
 2318 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2319   return new TypeVectMask(elemTy, length);
 2320 }
 2321 
 2322 // Vector calling convention not yet implemented.
 2323 const bool Matcher::supports_vector_calling_convention(void) {
 2324   return false;
 2325 }
 2326 
 2327 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2328   Unimplemented();
 2329   return OptoRegPair(0, 0);
 2330 }
 2331 
 2332 // Is this branch offset short enough that a short branch can be used?
 2333 //
 2334 // NOTE: If the platform does not provide any short branch variants, then
 2335 //       this method should return false for offset 0.
 2336 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2337   // The passed offset is relative to address of the branch.
 2338 
 2339   return (-32768 <= offset && offset < 32768);
 2340 }
 2341 
 2342 // Vector width in bytes.
 2343 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2344   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2345   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2346   // Minimum 2 values in vector
 2347   if (size < 2*type2aelembytes(bt)) size = 0;
 2348   // But never < 4
 2349   if (size < 4) size = 0;
 2350   return size;
 2351 }
 2352 
 2353 // Limits on vector size (number of elements) loaded into vector.
 2354 const int Matcher::max_vector_size(const BasicType bt) {
 2355   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2356 }
 2357 
 2358 const int Matcher::min_vector_size(const BasicType bt) {
 2359   int max_size = max_vector_size(bt);
 2360   // Limit the min vector size to 8 bytes.
 2361   int size = 8 / type2aelembytes(bt);
 2362   if (bt == T_BYTE) {
 2363     // To support vector api shuffle/rearrange.
 2364     size = 4;
 2365   } else if (bt == T_BOOLEAN) {
 2366     // To support vector api load/store mask.
 2367     size = 2;
 2368   }
 2369   if (size < 2) size = 2;
 2370   return MIN2(size, max_size);
 2371 }
 2372 
 2373 const int Matcher::superword_max_vector_size(const BasicType bt) {
 2374   return Matcher::max_vector_size(bt);
 2375 }
 2376 
 2377 // Actual max scalable vector register length.
 2378 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2379   return Matcher::max_vector_size(bt);
 2380 }
 2381 
 2382 // Vector ideal reg.
 2383 const uint Matcher::vector_ideal_reg(int len) {
 2384   if (UseSVE > 0 && 16 < len && len <= 256) {
 2385     return Op_VecA;
 2386   }
 2387   switch(len) {
 2388     // For 16-bit/32-bit mask vector, reuse VecD.
 2389     case  2:
 2390     case  4:
 2391     case  8: return Op_VecD;
 2392     case 16: return Op_VecX;
 2393   }
 2394   ShouldNotReachHere();
 2395   return 0;
 2396 }
 2397 
 2398 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
 2399   assert(Matcher::is_generic_vector(generic_opnd), "not generic");
 2400   switch (ideal_reg) {
 2401     case Op_VecA: return new vecAOper();
 2402     case Op_VecD: return new vecDOper();
 2403     case Op_VecX: return new vecXOper();
 2404   }
 2405   ShouldNotReachHere();
 2406   return NULL;
 2407 }
 2408 
 2409 bool Matcher::is_reg2reg_move(MachNode* m) {
 2410   return false;
 2411 }
 2412 
 2413 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2414   return opnd->opcode() == VREG;
 2415 }
 2416 
 2417 // Return whether or not this register is ever used as an argument.
 2418 // This function is used on startup to build the trampoline stubs in
 2419 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2420 // call in the trampoline, and arguments in those registers not be
 2421 // available to the callee.
 2422 bool Matcher::can_be_java_arg(int reg)
 2423 {
 2424   return
 2425     reg ==  R0_num || reg == R0_H_num ||
 2426     reg ==  R1_num || reg == R1_H_num ||
 2427     reg ==  R2_num || reg == R2_H_num ||
 2428     reg ==  R3_num || reg == R3_H_num ||
 2429     reg ==  R4_num || reg == R4_H_num ||
 2430     reg ==  R5_num || reg == R5_H_num ||
 2431     reg ==  R6_num || reg == R6_H_num ||
 2432     reg ==  R7_num || reg == R7_H_num ||
 2433     reg ==  V0_num || reg == V0_H_num ||
 2434     reg ==  V1_num || reg == V1_H_num ||
 2435     reg ==  V2_num || reg == V2_H_num ||
 2436     reg ==  V3_num || reg == V3_H_num ||
 2437     reg ==  V4_num || reg == V4_H_num ||
 2438     reg ==  V5_num || reg == V5_H_num ||
 2439     reg ==  V6_num || reg == V6_H_num ||
 2440     reg ==  V7_num || reg == V7_H_num;
 2441 }
 2442 
 2443 bool Matcher::is_spillable_arg(int reg)
 2444 {
 2445   return can_be_java_arg(reg);
 2446 }
 2447 
 2448 uint Matcher::int_pressure_limit()
 2449 {
 2450   // JDK-8183543: When taking the number of available registers as int
 2451   // register pressure threshold, the jtreg test:
 2452   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2453   // failed due to C2 compilation failure with
 2454   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2455   //
 2456   // A derived pointer is live at CallNode and then is flagged by RA
 2457   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2458   // derived pointers and lastly fail to spill after reaching maximum
 2459   // number of iterations. Lowering the default pressure threshold to
 2460   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2461   // a high register pressure area of the code so that split_DEF can
 2462   // generate DefinitionSpillCopy for the derived pointer.
 2463   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2464   if (!PreserveFramePointer) {
 2465     // When PreserveFramePointer is off, frame pointer is allocatable,
 2466     // but different from other SOC registers, it is excluded from
 2467     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2468     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2469     // See check_pressure_at_fatproj().
 2470     default_int_pressure_threshold--;
 2471   }
 2472   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2473 }
 2474 
 2475 uint Matcher::float_pressure_limit()
 2476 {
 2477   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2478   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2479 }
 2480 
 2481 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2482   return false;
 2483 }
 2484 
 2485 RegMask Matcher::divI_proj_mask() {
 2486   ShouldNotReachHere();
 2487   return RegMask();
 2488 }
 2489 
 2490 // Register for MODI projection of divmodI.
 2491 RegMask Matcher::modI_proj_mask() {
 2492   ShouldNotReachHere();
 2493   return RegMask();
 2494 }
 2495 
 2496 // Register for DIVL projection of divmodL.
 2497 RegMask Matcher::divL_proj_mask() {
 2498   ShouldNotReachHere();
 2499   return RegMask();
 2500 }
 2501 
 2502 // Register for MODL projection of divmodL.
 2503 RegMask Matcher::modL_proj_mask() {
 2504   ShouldNotReachHere();
 2505   return RegMask();
 2506 }
 2507 
 2508 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2509   return FP_REG_mask();
 2510 }
 2511 
 2512 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2513   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2514     Node* u = addp->fast_out(i);
 2515     if (u->is_LoadStore()) {
 2516       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2517       // instructions) only take register indirect as an operand, so
 2518       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2519       // must fail.
 2520       return false;
 2521     }
 2522     if (u->is_Mem()) {
 2523       int opsize = u->as_Mem()->memory_size();
 2524       assert(opsize > 0, "unexpected memory operand size");
 2525       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2526         return false;
 2527       }
 2528     }
 2529   }
 2530   return true;
 2531 }
 2532 
 2533 // Convert BootTest condition to Assembler condition.
 2534 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 2535 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
 2536   Assembler::Condition result;
 2537   switch(cond) {
 2538     case BoolTest::eq:
 2539       result = Assembler::EQ; break;
 2540     case BoolTest::ne:
 2541       result = Assembler::NE; break;
 2542     case BoolTest::le:
 2543       result = Assembler::LE; break;
 2544     case BoolTest::ge:
 2545       result = Assembler::GE; break;
 2546     case BoolTest::lt:
 2547       result = Assembler::LT; break;
 2548     case BoolTest::gt:
 2549       result = Assembler::GT; break;
 2550     case BoolTest::ule:
 2551       result = Assembler::LS; break;
 2552     case BoolTest::uge:
 2553       result = Assembler::HS; break;
 2554     case BoolTest::ult:
 2555       result = Assembler::LO; break;
 2556     case BoolTest::ugt:
 2557       result = Assembler::HI; break;
 2558     case BoolTest::overflow:
 2559       result = Assembler::VS; break;
 2560     case BoolTest::no_overflow:
 2561       result = Assembler::VC; break;
 2562     default:
 2563       ShouldNotReachHere();
 2564       return Assembler::Condition(-1);
 2565   }
 2566 
 2567   // Check conversion
 2568   if (cond & BoolTest::unsigned_compare) {
 2569     assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
 2570   } else {
 2571     assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
 2572   }
 2573 
 2574   return result;
 2575 }
 2576 
 2577 // Binary src (Replicate con)
 2578 bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
 2579   if (n == NULL || m == NULL) {
 2580     return false;
 2581   }
 2582 
 2583   if (UseSVE == 0 || !VectorNode::is_invariant_vector(m)) {
 2584     return false;
 2585   }
 2586 
 2587   Node* imm_node = m->in(1);
 2588   if (!imm_node->is_Con()) {
 2589     return false;
 2590   }
 2591 
 2592   const Type* t = imm_node->bottom_type();
 2593   if (!(t->isa_int() || t->isa_long())) {
 2594     return false;
 2595   }
 2596 
 2597   switch (n->Opcode()) {
 2598   case Op_AndV:
 2599   case Op_OrV:
 2600   case Op_XorV: {
 2601     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
 2602     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2603     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2604   }
 2605   case Op_AddVB:
 2606     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2607   case Op_AddVS:
 2608   case Op_AddVI:
 2609     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2610   case Op_AddVL:
 2611     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2612   default:
 2613     return false;
 2614   }
 2615 }
 2616 
 2617 // (XorV src (Replicate m1))
 2618 // (XorVMask src (MaskAll m1))
 2619 bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
 2620   if (n != NULL && m != NULL) {
 2621     return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
 2622            VectorNode::is_all_ones_vector(m);
 2623   }
 2624   return false;
 2625 }
 2626 
 2627 // Should the matcher clone input 'm' of node 'n'?
 2628 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2629   if (is_vshift_con_pattern(n, m) ||
 2630       is_vector_bitwise_not_pattern(n, m) ||
 2631       is_valid_sve_arith_imm_pattern(n, m)) {
 2632     mstack.push(m, Visit);
 2633     return true;
 2634   }
 2635   return false;
 2636 }
 2637 
 2638 // Should the Matcher clone shifts on addressing modes, expecting them
 2639 // to be subsumed into complex addressing expressions or compute them
 2640 // into registers?
 2641 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2642   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2643     return true;
 2644   }
 2645 
 2646   Node *off = m->in(AddPNode::Offset);
 2647   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2648       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2649       // Are there other uses besides address expressions?
 2650       !is_visited(off)) {
 2651     address_visited.set(off->_idx); // Flag as address_visited
 2652     mstack.push(off->in(2), Visit);
 2653     Node *conv = off->in(1);
 2654     if (conv->Opcode() == Op_ConvI2L &&
 2655         // Are there other uses besides address expressions?
 2656         !is_visited(conv)) {
 2657       address_visited.set(conv->_idx); // Flag as address_visited
 2658       mstack.push(conv->in(1), Pre_Visit);
 2659     } else {
 2660       mstack.push(conv, Pre_Visit);
 2661     }
 2662     address_visited.test_set(m->_idx); // Flag as address_visited
 2663     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2664     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2665     return true;
 2666   } else if (off->Opcode() == Op_ConvI2L &&
 2667              // Are there other uses besides address expressions?
 2668              !is_visited(off)) {
 2669     address_visited.test_set(m->_idx); // Flag as address_visited
 2670     address_visited.set(off->_idx); // Flag as address_visited
 2671     mstack.push(off->in(1), Pre_Visit);
 2672     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2673     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2674     return true;
 2675   }
 2676   return false;
 2677 }
 2678 
 2679 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2680   C2_MacroAssembler _masm(&cbuf);                                       \
 2681   {                                                                     \
 2682     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2683     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2684     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2685     __ INSN(REG, as_Register(BASE));                                    \
 2686   }
 2687 
 2688 
 2689 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2690   {
 2691     Address::extend scale;
 2692 
 2693     // Hooboy, this is fugly.  We need a way to communicate to the
 2694     // encoder that the index needs to be sign extended, so we have to
 2695     // enumerate all the cases.
 2696     switch (opcode) {
 2697     case INDINDEXSCALEDI2L:
 2698     case INDINDEXSCALEDI2LN:
 2699     case INDINDEXI2L:
 2700     case INDINDEXI2LN:
 2701       scale = Address::sxtw(size);
 2702       break;
 2703     default:
 2704       scale = Address::lsl(size);
 2705     }
 2706 
 2707     if (index == -1) {
 2708       return Address(base, disp);
 2709     } else {
 2710       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2711       return Address(base, as_Register(index), scale);
 2712     }
 2713   }
 2714 
 2715 
 2716 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2717 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2718 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2719 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2720                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2721 
 2722   // Used for all non-volatile memory accesses.  The use of
 2723   // $mem->opcode() to discover whether this pattern uses sign-extended
 2724   // offsets is something of a kludge.
 2725   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2726                         Register reg, int opcode,
 2727                         Register base, int index, int scale, int disp,
 2728                         int size_in_memory)
 2729   {
 2730     Address addr = mem2address(opcode, base, index, scale, disp);
 2731     if (addr.getMode() == Address::base_plus_offset) {
 2732       /* If we get an out-of-range offset it is a bug in the compiler,
 2733          so we assert here. */
 2734       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2735              "c2 compiler bug");
 2736       /* Fix up any out-of-range offsets. */
 2737       assert_different_registers(rscratch1, base);
 2738       assert_different_registers(rscratch1, reg);
 2739       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2740     }
 2741     (masm.*insn)(reg, addr);
 2742   }
 2743 
 2744   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2745                         FloatRegister reg, int opcode,
 2746                         Register base, int index, int size, int disp,
 2747                         int size_in_memory)
 2748   {
 2749     Address::extend scale;
 2750 
 2751     switch (opcode) {
 2752     case INDINDEXSCALEDI2L:
 2753     case INDINDEXSCALEDI2LN:
 2754       scale = Address::sxtw(size);
 2755       break;
 2756     default:
 2757       scale = Address::lsl(size);
 2758     }
 2759 
 2760     if (index == -1) {
 2761       /* If we get an out-of-range offset it is a bug in the compiler,
 2762          so we assert here. */
 2763       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2764       /* Fix up any out-of-range offsets. */
 2765       assert_different_registers(rscratch1, base);
 2766       Address addr = Address(base, disp);
 2767       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2768       (masm.*insn)(reg, addr);
 2769     } else {
 2770       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2771       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2772     }
 2773   }
 2774 
 2775   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2776                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2777                         int opcode, Register base, int index, int size, int disp)
 2778   {
 2779     if (index == -1) {
 2780       (masm.*insn)(reg, T, Address(base, disp));
 2781     } else {
 2782       assert(disp == 0, "unsupported address mode");
 2783       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2784     }
 2785   }
 2786 
 2787 %}
 2788 
 2789 
 2790 
 2791 //----------ENCODING BLOCK-----------------------------------------------------
 2792 // This block specifies the encoding classes used by the compiler to
 2793 // output byte streams.  Encoding classes are parameterized macros
 2794 // used by Machine Instruction Nodes in order to generate the bit
 2795 // encoding of the instruction.  Operands specify their base encoding
 2796 // interface with the interface keyword.  There are currently
 2797 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2798 // COND_INTER.  REG_INTER causes an operand to generate a function
 2799 // which returns its register number when queried.  CONST_INTER causes
 2800 // an operand to generate a function which returns the value of the
 2801 // constant when queried.  MEMORY_INTER causes an operand to generate
 2802 // four functions which return the Base Register, the Index Register,
 2803 // the Scale Value, and the Offset Value of the operand when queried.
 2804 // COND_INTER causes an operand to generate six functions which return
 2805 // the encoding code (ie - encoding bits for the instruction)
 2806 // associated with each basic boolean condition for a conditional
 2807 // instruction.
 2808 //
 2809 // Instructions specify two basic values for encoding.  Again, a
 2810 // function is available to check if the constant displacement is an
 2811 // oop. They use the ins_encode keyword to specify their encoding
 2812 // classes (which must be a sequence of enc_class names, and their
 2813 // parameters, specified in the encoding block), and they use the
 2814 // opcode keyword to specify, in order, their primary, secondary, and
 2815 // tertiary opcode.  Only the opcode sections which a particular
 2816 // instruction needs for encoding need to be specified.
 2817 encode %{
 2818   // Build emit functions for each basic byte or larger field in the
 2819   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2820   // from C++ code in the enc_class source block.  Emit functions will
 2821   // live in the main source block for now.  In future, we can
 2822   // generalize this by adding a syntax that specifies the sizes of
 2823   // fields in an order, so that the adlc can build the emit functions
 2824   // automagically
 2825 
 2826   // catch all for unimplemented encodings
 2827   enc_class enc_unimplemented %{
 2828     C2_MacroAssembler _masm(&cbuf);
 2829     __ unimplemented("C2 catch all");
 2830   %}
 2831 
 2832   // BEGIN Non-volatile memory access
 2833 
 2834   // This encoding class is generated automatically from ad_encode.m4.
 2835   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2836   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2837     Register dst_reg = as_Register($dst$$reg);
 2838     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2839                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2840   %}
 2841 
 2842   // This encoding class is generated automatically from ad_encode.m4.
 2843   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2844   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2845     Register dst_reg = as_Register($dst$$reg);
 2846     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2847                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2848   %}
 2849 
 2850   // This encoding class is generated automatically from ad_encode.m4.
 2851   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2852   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2853     Register dst_reg = as_Register($dst$$reg);
 2854     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2855                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2856   %}
 2857 
 2858   // This encoding class is generated automatically from ad_encode.m4.
 2859   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2860   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2861     Register dst_reg = as_Register($dst$$reg);
 2862     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2863                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2864   %}
 2865 
 2866   // This encoding class is generated automatically from ad_encode.m4.
 2867   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2868   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2869     Register dst_reg = as_Register($dst$$reg);
 2870     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2871                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2872   %}
 2873 
 2874   // This encoding class is generated automatically from ad_encode.m4.
 2875   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2876   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2877     Register dst_reg = as_Register($dst$$reg);
 2878     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2879                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2880   %}
 2881 
 2882   // This encoding class is generated automatically from ad_encode.m4.
 2883   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2884   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2885     Register dst_reg = as_Register($dst$$reg);
 2886     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2887                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2888   %}
 2889 
 2890   // This encoding class is generated automatically from ad_encode.m4.
 2891   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2892   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2893     Register dst_reg = as_Register($dst$$reg);
 2894     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2895                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2896   %}
 2897 
 2898   // This encoding class is generated automatically from ad_encode.m4.
 2899   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2900   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2901     Register dst_reg = as_Register($dst$$reg);
 2902     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2903                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2904   %}
 2905 
 2906   // This encoding class is generated automatically from ad_encode.m4.
 2907   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2908   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2909     Register dst_reg = as_Register($dst$$reg);
 2910     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2911                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2912   %}
 2913 
 2914   // This encoding class is generated automatically from ad_encode.m4.
 2915   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2916   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2917     Register dst_reg = as_Register($dst$$reg);
 2918     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2919                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2920   %}
 2921 
 2922   // This encoding class is generated automatically from ad_encode.m4.
 2923   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2924   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2925     Register dst_reg = as_Register($dst$$reg);
 2926     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2927                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2928   %}
 2929 
 2930   // This encoding class is generated automatically from ad_encode.m4.
 2931   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2932   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2933     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2934     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2935                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2936   %}
 2937 
 2938   // This encoding class is generated automatically from ad_encode.m4.
 2939   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2940   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 2941     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2942     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2943                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2944   %}
 2945 
 2946   // This encoding class is generated automatically from ad_encode.m4.
 2947   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2948   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 2949     Register src_reg = as_Register($src$$reg);
 2950     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 2951                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2952   %}
 2953 
 2954   // This encoding class is generated automatically from ad_encode.m4.
 2955   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2956   enc_class aarch64_enc_strb0(memory1 mem) %{
 2957     C2_MacroAssembler _masm(&cbuf);
 2958     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2959                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2960   %}
 2961 
 2962   // This encoding class is generated automatically from ad_encode.m4.
 2963   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2964   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 2965     Register src_reg = as_Register($src$$reg);
 2966     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 2967                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2968   %}
 2969 
 2970   // This encoding class is generated automatically from ad_encode.m4.
 2971   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2972   enc_class aarch64_enc_strh0(memory2 mem) %{
 2973     C2_MacroAssembler _masm(&cbuf);
 2974     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 2975                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2976   %}
 2977 
 2978   // This encoding class is generated automatically from ad_encode.m4.
 2979   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2980   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 2981     Register src_reg = as_Register($src$$reg);
 2982     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 2983                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2984   %}
 2985 
 2986   // This encoding class is generated automatically from ad_encode.m4.
 2987   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2988   enc_class aarch64_enc_strw0(memory4 mem) %{
 2989     C2_MacroAssembler _masm(&cbuf);
 2990     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 2991                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2992   %}
 2993 
 2994   // This encoding class is generated automatically from ad_encode.m4.
 2995   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2996   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 2997     Register src_reg = as_Register($src$$reg);
 2998     // we sometimes get asked to store the stack pointer into the
 2999     // current thread -- we cannot do that directly on AArch64
 3000     if (src_reg == r31_sp) {
 3001       C2_MacroAssembler _masm(&cbuf);
 3002       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3003       __ mov(rscratch2, sp);
 3004       src_reg = rscratch2;
 3005     }
 3006     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 3007                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3008   %}
 3009 
 3010   // This encoding class is generated automatically from ad_encode.m4.
 3011   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3012   enc_class aarch64_enc_str0(memory8 mem) %{
 3013     C2_MacroAssembler _masm(&cbuf);
 3014     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 3015                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3016   %}
 3017 
 3018   // This encoding class is generated automatically from ad_encode.m4.
 3019   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3020   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3021     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3022     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 3023                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3024   %}
 3025 
 3026   // This encoding class is generated automatically from ad_encode.m4.
 3027   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3028   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3029     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3030     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 3031                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3032   %}
 3033 
 3034   // This encoding class is generated automatically from ad_encode.m4.
 3035   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3036   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3037       C2_MacroAssembler _masm(&cbuf);
 3038       __ membar(Assembler::StoreStore);
 3039       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3040                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3041   %}
 3042 
 3043   // END Non-volatile memory access
 3044 
 3045   // Vector loads and stores
 3046   enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
 3047     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3048     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3049        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3050   %}
 3051 
 3052   enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
 3053     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3054     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3055        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3056   %}
 3057 
 3058   enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
 3059     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3060     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3061        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3062   %}
 3063 
 3064   enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
 3065     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3066     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3067        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3068   %}
 3069 
 3070   enc_class aarch64_enc_strvH(vReg src, memory mem) %{
 3071     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3072     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3073        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3074   %}
 3075 
 3076   enc_class aarch64_enc_strvS(vReg src, memory mem) %{
 3077     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3078     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3079        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3080   %}
 3081 
 3082   enc_class aarch64_enc_strvD(vReg src, memory mem) %{
 3083     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3084     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3085        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3086   %}
 3087 
 3088   enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
 3089     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3090     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3091        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3092   %}
 3093 
 3094   // volatile loads and stores
 3095 
 3096   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3097     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3098                  rscratch1, stlrb);
 3099   %}
 3100 
 3101   enc_class aarch64_enc_stlrb0(memory mem) %{
 3102     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3103                  rscratch1, stlrb);
 3104   %}
 3105 
 3106   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3107     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3108                  rscratch1, stlrh);
 3109   %}
 3110 
 3111   enc_class aarch64_enc_stlrh0(memory mem) %{
 3112     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3113                  rscratch1, stlrh);
 3114   %}
 3115 
 3116   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3117     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3118                  rscratch1, stlrw);
 3119   %}
 3120 
 3121   enc_class aarch64_enc_stlrw0(memory mem) %{
 3122     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3123                  rscratch1, stlrw);
 3124   %}
 3125 
 3126   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3127     Register dst_reg = as_Register($dst$$reg);
 3128     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3129              rscratch1, ldarb);
 3130     __ sxtbw(dst_reg, dst_reg);
 3131   %}
 3132 
 3133   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3134     Register dst_reg = as_Register($dst$$reg);
 3135     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3136              rscratch1, ldarb);
 3137     __ sxtb(dst_reg, dst_reg);
 3138   %}
 3139 
 3140   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3141     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3142              rscratch1, ldarb);
 3143   %}
 3144 
 3145   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3146     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3147              rscratch1, ldarb);
 3148   %}
 3149 
 3150   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3151     Register dst_reg = as_Register($dst$$reg);
 3152     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3153              rscratch1, ldarh);
 3154     __ sxthw(dst_reg, dst_reg);
 3155   %}
 3156 
 3157   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3158     Register dst_reg = as_Register($dst$$reg);
 3159     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3160              rscratch1, ldarh);
 3161     __ sxth(dst_reg, dst_reg);
 3162   %}
 3163 
 3164   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3165     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3166              rscratch1, ldarh);
 3167   %}
 3168 
 3169   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3170     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3171              rscratch1, ldarh);
 3172   %}
 3173 
 3174   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3175     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3176              rscratch1, ldarw);
 3177   %}
 3178 
 3179   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3180     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3181              rscratch1, ldarw);
 3182   %}
 3183 
 3184   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3185     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3186              rscratch1, ldar);
 3187   %}
 3188 
 3189   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3190     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3191              rscratch1, ldarw);
 3192     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3193   %}
 3194 
 3195   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3196     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3197              rscratch1, ldar);
 3198     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3199   %}
 3200 
 3201   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3202     Register src_reg = as_Register($src$$reg);
 3203     // we sometimes get asked to store the stack pointer into the
 3204     // current thread -- we cannot do that directly on AArch64
 3205     if (src_reg == r31_sp) {
 3206       C2_MacroAssembler _masm(&cbuf);
 3207       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3208       __ mov(rscratch2, sp);
 3209       src_reg = rscratch2;
 3210     }
 3211     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3212                  rscratch1, stlr);
 3213   %}
 3214 
 3215   enc_class aarch64_enc_stlr0(memory mem) %{
 3216     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3217                  rscratch1, stlr);
 3218   %}
 3219 
 3220   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3221     {
 3222       C2_MacroAssembler _masm(&cbuf);
 3223       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3224       __ fmovs(rscratch2, src_reg);
 3225     }
 3226     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3227                  rscratch1, stlrw);
 3228   %}
 3229 
 3230   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3231     {
 3232       C2_MacroAssembler _masm(&cbuf);
 3233       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3234       __ fmovd(rscratch2, src_reg);
 3235     }
 3236     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3237                  rscratch1, stlr);
 3238   %}
 3239 
 3240   // synchronized read/update encodings
 3241 
 3242   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3243     C2_MacroAssembler _masm(&cbuf);
 3244     Register dst_reg = as_Register($dst$$reg);
 3245     Register base = as_Register($mem$$base);
 3246     int index = $mem$$index;
 3247     int scale = $mem$$scale;
 3248     int disp = $mem$$disp;
 3249     if (index == -1) {
 3250        if (disp != 0) {
 3251         __ lea(rscratch1, Address(base, disp));
 3252         __ ldaxr(dst_reg, rscratch1);
 3253       } else {
 3254         // TODO
 3255         // should we ever get anything other than this case?
 3256         __ ldaxr(dst_reg, base);
 3257       }
 3258     } else {
 3259       Register index_reg = as_Register(index);
 3260       if (disp == 0) {
 3261         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3262         __ ldaxr(dst_reg, rscratch1);
 3263       } else {
 3264         __ lea(rscratch1, Address(base, disp));
 3265         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3266         __ ldaxr(dst_reg, rscratch1);
 3267       }
 3268     }
 3269   %}
 3270 
 3271   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3272     C2_MacroAssembler _masm(&cbuf);
 3273     Register src_reg = as_Register($src$$reg);
 3274     Register base = as_Register($mem$$base);
 3275     int index = $mem$$index;
 3276     int scale = $mem$$scale;
 3277     int disp = $mem$$disp;
 3278     if (index == -1) {
 3279        if (disp != 0) {
 3280         __ lea(rscratch2, Address(base, disp));
 3281         __ stlxr(rscratch1, src_reg, rscratch2);
 3282       } else {
 3283         // TODO
 3284         // should we ever get anything other than this case?
 3285         __ stlxr(rscratch1, src_reg, base);
 3286       }
 3287     } else {
 3288       Register index_reg = as_Register(index);
 3289       if (disp == 0) {
 3290         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3291         __ stlxr(rscratch1, src_reg, rscratch2);
 3292       } else {
 3293         __ lea(rscratch2, Address(base, disp));
 3294         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3295         __ stlxr(rscratch1, src_reg, rscratch2);
 3296       }
 3297     }
 3298     __ cmpw(rscratch1, zr);
 3299   %}
 3300 
 3301   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3302     C2_MacroAssembler _masm(&cbuf);
 3303     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3304     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3305                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3306                /*weak*/ false, noreg);
 3307   %}
 3308 
 3309   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3310     C2_MacroAssembler _masm(&cbuf);
 3311     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3312     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3313                Assembler::word, /*acquire*/ false, /*release*/ true,
 3314                /*weak*/ false, noreg);
 3315   %}
 3316 
 3317   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3318     C2_MacroAssembler _masm(&cbuf);
 3319     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3320     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3321                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3322                /*weak*/ false, noreg);
 3323   %}
 3324 
 3325   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3326     C2_MacroAssembler _masm(&cbuf);
 3327     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3328     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3329                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3330                /*weak*/ false, noreg);
 3331   %}
 3332 
 3333 
 3334   // The only difference between aarch64_enc_cmpxchg and
 3335   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3336   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3337   // lock.
 3338   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3339     C2_MacroAssembler _masm(&cbuf);
 3340     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3341     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3342                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3343                /*weak*/ false, noreg);
 3344   %}
 3345 
 3346   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3347     C2_MacroAssembler _masm(&cbuf);
 3348     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3349     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3350                Assembler::word, /*acquire*/ true, /*release*/ true,
 3351                /*weak*/ false, noreg);
 3352   %}
 3353 
 3354   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3355     C2_MacroAssembler _masm(&cbuf);
 3356     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3357     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3358                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3359                /*weak*/ false, noreg);
 3360   %}
 3361 
 3362   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3363     C2_MacroAssembler _masm(&cbuf);
 3364     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3365     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3366                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3367                /*weak*/ false, noreg);
 3368   %}
 3369 
 3370   // auxiliary used for CompareAndSwapX to set result register
 3371   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3372     C2_MacroAssembler _masm(&cbuf);
 3373     Register res_reg = as_Register($res$$reg);
 3374     __ cset(res_reg, Assembler::EQ);
 3375   %}
 3376 
 3377   // prefetch encodings
 3378 
 3379   enc_class aarch64_enc_prefetchw(memory mem) %{
 3380     C2_MacroAssembler _masm(&cbuf);
 3381     Register base = as_Register($mem$$base);
 3382     int index = $mem$$index;
 3383     int scale = $mem$$scale;
 3384     int disp = $mem$$disp;
 3385     if (index == -1) {
 3386       __ prfm(Address(base, disp), PSTL1KEEP);
 3387     } else {
 3388       Register index_reg = as_Register(index);
 3389       if (disp == 0) {
 3390         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3391       } else {
 3392         __ lea(rscratch1, Address(base, disp));
 3393 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3394       }
 3395     }
 3396   %}
 3397 
 3398   /// mov envcodings
 3399 
 3400   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3401     C2_MacroAssembler _masm(&cbuf);
 3402     uint32_t con = (uint32_t)$src$$constant;
 3403     Register dst_reg = as_Register($dst$$reg);
 3404     if (con == 0) {
 3405       __ movw(dst_reg, zr);
 3406     } else {
 3407       __ movw(dst_reg, con);
 3408     }
 3409   %}
 3410 
 3411   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3412     C2_MacroAssembler _masm(&cbuf);
 3413     Register dst_reg = as_Register($dst$$reg);
 3414     uint64_t con = (uint64_t)$src$$constant;
 3415     if (con == 0) {
 3416       __ mov(dst_reg, zr);
 3417     } else {
 3418       __ mov(dst_reg, con);
 3419     }
 3420   %}
 3421 
 3422   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3423     C2_MacroAssembler _masm(&cbuf);
 3424     Register dst_reg = as_Register($dst$$reg);
 3425     address con = (address)$src$$constant;
 3426     if (con == NULL || con == (address)1) {
 3427       ShouldNotReachHere();
 3428     } else {
 3429       relocInfo::relocType rtype = $src->constant_reloc();
 3430       if (rtype == relocInfo::oop_type) {
 3431         __ movoop(dst_reg, (jobject)con);
 3432       } else if (rtype == relocInfo::metadata_type) {
 3433         __ mov_metadata(dst_reg, (Metadata*)con);
 3434       } else {
 3435         assert(rtype == relocInfo::none, "unexpected reloc type");
 3436         if (! __ is_valid_AArch64_address(con) ||
 3437             con < (address)(uintptr_t)os::vm_page_size()) {
 3438           __ mov(dst_reg, con);
 3439         } else {
 3440           uint64_t offset;
 3441           __ adrp(dst_reg, con, offset);
 3442           __ add(dst_reg, dst_reg, offset);
 3443         }
 3444       }
 3445     }
 3446   %}
 3447 
 3448   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3449     C2_MacroAssembler _masm(&cbuf);
 3450     Register dst_reg = as_Register($dst$$reg);
 3451     __ mov(dst_reg, zr);
 3452   %}
 3453 
 3454   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3455     C2_MacroAssembler _masm(&cbuf);
 3456     Register dst_reg = as_Register($dst$$reg);
 3457     __ mov(dst_reg, (uint64_t)1);
 3458   %}
 3459 
 3460   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3461     C2_MacroAssembler _masm(&cbuf);
 3462     __ load_byte_map_base($dst$$Register);
 3463   %}
 3464 
 3465   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3466     C2_MacroAssembler _masm(&cbuf);
 3467     Register dst_reg = as_Register($dst$$reg);
 3468     address con = (address)$src$$constant;
 3469     if (con == NULL) {
 3470       ShouldNotReachHere();
 3471     } else {
 3472       relocInfo::relocType rtype = $src->constant_reloc();
 3473       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3474       __ set_narrow_oop(dst_reg, (jobject)con);
 3475     }
 3476   %}
 3477 
 3478   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3479     C2_MacroAssembler _masm(&cbuf);
 3480     Register dst_reg = as_Register($dst$$reg);
 3481     __ mov(dst_reg, zr);
 3482   %}
 3483 
 3484   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3485     C2_MacroAssembler _masm(&cbuf);
 3486     Register dst_reg = as_Register($dst$$reg);
 3487     address con = (address)$src$$constant;
 3488     if (con == NULL) {
 3489       ShouldNotReachHere();
 3490     } else {
 3491       relocInfo::relocType rtype = $src->constant_reloc();
 3492       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3493       __ set_narrow_klass(dst_reg, (Klass *)con);
 3494     }
 3495   %}
 3496 
 3497   // arithmetic encodings
 3498 
 3499   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3500     C2_MacroAssembler _masm(&cbuf);
 3501     Register dst_reg = as_Register($dst$$reg);
 3502     Register src_reg = as_Register($src1$$reg);
 3503     int32_t con = (int32_t)$src2$$constant;
 3504     // add has primary == 0, subtract has primary == 1
 3505     if ($primary) { con = -con; }
 3506     if (con < 0) {
 3507       __ subw(dst_reg, src_reg, -con);
 3508     } else {
 3509       __ addw(dst_reg, src_reg, con);
 3510     }
 3511   %}
 3512 
 3513   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3514     C2_MacroAssembler _masm(&cbuf);
 3515     Register dst_reg = as_Register($dst$$reg);
 3516     Register src_reg = as_Register($src1$$reg);
 3517     int32_t con = (int32_t)$src2$$constant;
 3518     // add has primary == 0, subtract has primary == 1
 3519     if ($primary) { con = -con; }
 3520     if (con < 0) {
 3521       __ sub(dst_reg, src_reg, -con);
 3522     } else {
 3523       __ add(dst_reg, src_reg, con);
 3524     }
 3525   %}
 3526 
 3527   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3528     C2_MacroAssembler _masm(&cbuf);
 3529    Register dst_reg = as_Register($dst$$reg);
 3530    Register src1_reg = as_Register($src1$$reg);
 3531    Register src2_reg = as_Register($src2$$reg);
 3532     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3533   %}
 3534 
 3535   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3536     C2_MacroAssembler _masm(&cbuf);
 3537    Register dst_reg = as_Register($dst$$reg);
 3538    Register src1_reg = as_Register($src1$$reg);
 3539    Register src2_reg = as_Register($src2$$reg);
 3540     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3541   %}
 3542 
 3543   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3544     C2_MacroAssembler _masm(&cbuf);
 3545    Register dst_reg = as_Register($dst$$reg);
 3546    Register src1_reg = as_Register($src1$$reg);
 3547    Register src2_reg = as_Register($src2$$reg);
 3548     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3549   %}
 3550 
 3551   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3552     C2_MacroAssembler _masm(&cbuf);
 3553    Register dst_reg = as_Register($dst$$reg);
 3554    Register src1_reg = as_Register($src1$$reg);
 3555    Register src2_reg = as_Register($src2$$reg);
 3556     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3557   %}
 3558 
 3559   // compare instruction encodings
 3560 
 3561   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3562     C2_MacroAssembler _masm(&cbuf);
 3563     Register reg1 = as_Register($src1$$reg);
 3564     Register reg2 = as_Register($src2$$reg);
 3565     __ cmpw(reg1, reg2);
 3566   %}
 3567 
 3568   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3569     C2_MacroAssembler _masm(&cbuf);
 3570     Register reg = as_Register($src1$$reg);
 3571     int32_t val = $src2$$constant;
 3572     if (val >= 0) {
 3573       __ subsw(zr, reg, val);
 3574     } else {
 3575       __ addsw(zr, reg, -val);
 3576     }
 3577   %}
 3578 
 3579   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3580     C2_MacroAssembler _masm(&cbuf);
 3581     Register reg1 = as_Register($src1$$reg);
 3582     uint32_t val = (uint32_t)$src2$$constant;
 3583     __ movw(rscratch1, val);
 3584     __ cmpw(reg1, rscratch1);
 3585   %}
 3586 
 3587   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3588     C2_MacroAssembler _masm(&cbuf);
 3589     Register reg1 = as_Register($src1$$reg);
 3590     Register reg2 = as_Register($src2$$reg);
 3591     __ cmp(reg1, reg2);
 3592   %}
 3593 
 3594   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3595     C2_MacroAssembler _masm(&cbuf);
 3596     Register reg = as_Register($src1$$reg);
 3597     int64_t val = $src2$$constant;
 3598     if (val >= 0) {
 3599       __ subs(zr, reg, val);
 3600     } else if (val != -val) {
 3601       __ adds(zr, reg, -val);
 3602     } else {
 3603     // aargh, Long.MIN_VALUE is a special case
 3604       __ orr(rscratch1, zr, (uint64_t)val);
 3605       __ subs(zr, reg, rscratch1);
 3606     }
 3607   %}
 3608 
 3609   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3610     C2_MacroAssembler _masm(&cbuf);
 3611     Register reg1 = as_Register($src1$$reg);
 3612     uint64_t val = (uint64_t)$src2$$constant;
 3613     __ mov(rscratch1, val);
 3614     __ cmp(reg1, rscratch1);
 3615   %}
 3616 
 3617   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3618     C2_MacroAssembler _masm(&cbuf);
 3619     Register reg1 = as_Register($src1$$reg);
 3620     Register reg2 = as_Register($src2$$reg);
 3621     __ cmp(reg1, reg2);
 3622   %}
 3623 
 3624   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3625     C2_MacroAssembler _masm(&cbuf);
 3626     Register reg1 = as_Register($src1$$reg);
 3627     Register reg2 = as_Register($src2$$reg);
 3628     __ cmpw(reg1, reg2);
 3629   %}
 3630 
 3631   enc_class aarch64_enc_testp(iRegP src) %{
 3632     C2_MacroAssembler _masm(&cbuf);
 3633     Register reg = as_Register($src$$reg);
 3634     __ cmp(reg, zr);
 3635   %}
 3636 
 3637   enc_class aarch64_enc_testn(iRegN src) %{
 3638     C2_MacroAssembler _masm(&cbuf);
 3639     Register reg = as_Register($src$$reg);
 3640     __ cmpw(reg, zr);
 3641   %}
 3642 
 3643   enc_class aarch64_enc_b(label lbl) %{
 3644     C2_MacroAssembler _masm(&cbuf);
 3645     Label *L = $lbl$$label;
 3646     __ b(*L);
 3647   %}
 3648 
 3649   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3650     C2_MacroAssembler _masm(&cbuf);
 3651     Label *L = $lbl$$label;
 3652     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3653   %}
 3654 
 3655   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3656     C2_MacroAssembler _masm(&cbuf);
 3657     Label *L = $lbl$$label;
 3658     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3659   %}
 3660 
 3661   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3662   %{
 3663      Register sub_reg = as_Register($sub$$reg);
 3664      Register super_reg = as_Register($super$$reg);
 3665      Register temp_reg = as_Register($temp$$reg);
 3666      Register result_reg = as_Register($result$$reg);
 3667 
 3668      Label miss;
 3669      C2_MacroAssembler _masm(&cbuf);
 3670      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3671                                      NULL, &miss,
 3672                                      /*set_cond_codes:*/ true);
 3673      if ($primary) {
 3674        __ mov(result_reg, zr);
 3675      }
 3676      __ bind(miss);
 3677   %}
 3678 
 3679   enc_class aarch64_enc_java_static_call(method meth) %{
 3680     C2_MacroAssembler _masm(&cbuf);
 3681 
 3682     address addr = (address)$meth$$method;
 3683     address call;
 3684     if (!_method) {
 3685       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3686       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
 3687       if (call == NULL) {
 3688         ciEnv::current()->record_failure("CodeCache is full");
 3689         return;
 3690       }
 3691     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 3692       // The NOP here is purely to ensure that eliding a call to
 3693       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 3694       __ nop();
 3695       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 3696     } else {
 3697       int method_index = resolved_method_index(cbuf);
 3698       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3699                                                   : static_call_Relocation::spec(method_index);
 3700       call = __ trampoline_call(Address(addr, rspec));
 3701       if (call == NULL) {
 3702         ciEnv::current()->record_failure("CodeCache is full");
 3703         return;
 3704       }
 3705       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 3706         // Calls of the same statically bound method can share
 3707         // a stub to the interpreter.
 3708         cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
 3709       } else {
 3710         // Emit stub for static call
 3711         address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
 3712         if (stub == NULL) {
 3713           ciEnv::current()->record_failure("CodeCache is full");
 3714           return;
 3715         }
 3716       }
 3717     }
 3718 
 3719     __ post_call_nop();
 3720 
 3721     // Only non uncommon_trap calls need to reinitialize ptrue.
 3722     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3723       __ reinitialize_ptrue();
 3724     }
 3725   %}
 3726 
 3727   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3728     C2_MacroAssembler _masm(&cbuf);
 3729     int method_index = resolved_method_index(cbuf);
 3730     address call = __ ic_call((address)$meth$$method, method_index);
 3731     if (call == NULL) {
 3732       ciEnv::current()->record_failure("CodeCache is full");
 3733       return;
 3734     }
 3735     __ post_call_nop();
 3736     if (Compile::current()->max_vector_size() > 0) {
 3737       __ reinitialize_ptrue();
 3738     }
 3739   %}
 3740 
 3741   enc_class aarch64_enc_call_epilog() %{
 3742     C2_MacroAssembler _masm(&cbuf);
 3743     if (VerifyStackAtCalls) {
 3744       // Check that stack depth is unchanged: find majik cookie on stack
 3745       __ call_Unimplemented();
 3746     }
 3747   %}
 3748 
 3749   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3750     C2_MacroAssembler _masm(&cbuf);
 3751 
 3752     // some calls to generated routines (arraycopy code) are scheduled
 3753     // by C2 as runtime calls. if so we can call them using a br (they
 3754     // will be in a reachable segment) otherwise we have to use a blr
 3755     // which loads the absolute address into a register.
 3756     address entry = (address)$meth$$method;
 3757     CodeBlob *cb = CodeCache::find_blob(entry);
 3758     if (cb) {
 3759       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3760       if (call == NULL) {
 3761         ciEnv::current()->record_failure("CodeCache is full");
 3762         return;
 3763       }
 3764       __ post_call_nop();
 3765     } else {
 3766       Label retaddr;
 3767       __ adr(rscratch2, retaddr);
 3768       __ lea(rscratch1, RuntimeAddress(entry));
 3769       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3770       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3771       __ blr(rscratch1);
 3772       __ bind(retaddr);
 3773       __ post_call_nop();
 3774       __ add(sp, sp, 2 * wordSize);
 3775     }
 3776     if (Compile::current()->max_vector_size() > 0) {
 3777       __ reinitialize_ptrue();
 3778     }
 3779   %}
 3780 
 3781   enc_class aarch64_enc_rethrow() %{
 3782     C2_MacroAssembler _masm(&cbuf);
 3783     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3784   %}
 3785 
 3786   enc_class aarch64_enc_ret() %{
 3787     C2_MacroAssembler _masm(&cbuf);
 3788 #ifdef ASSERT
 3789     if (Compile::current()->max_vector_size() > 0) {
 3790       __ verify_ptrue();
 3791     }
 3792 #endif
 3793     __ ret(lr);
 3794   %}
 3795 
 3796   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3797     C2_MacroAssembler _masm(&cbuf);
 3798     Register target_reg = as_Register($jump_target$$reg);
 3799     __ br(target_reg);
 3800   %}
 3801 
 3802   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3803     C2_MacroAssembler _masm(&cbuf);
 3804     Register target_reg = as_Register($jump_target$$reg);
 3805     // exception oop should be in r0
 3806     // ret addr has been popped into lr
 3807     // callee expects it in r3
 3808     __ mov(r3, lr);
 3809     __ br(target_reg);
 3810   %}
 3811 
 3812   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3813     C2_MacroAssembler _masm(&cbuf);
 3814     Register oop = as_Register($object$$reg);
 3815     Register box = as_Register($box$$reg);
 3816     Register disp_hdr = as_Register($tmp$$reg);
 3817     Register tmp = as_Register($tmp2$$reg);
 3818     Label cont;
 3819     Label object_has_monitor;
 3820     Label count, no_count;
 3821 
 3822     assert_different_registers(oop, box, tmp, disp_hdr);
 3823 
 3824     // Load markWord from object into displaced_header.
 3825     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3826 
 3827     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3828       __ load_klass(tmp, oop);
 3829       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3830       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3831       __ br(Assembler::NE, cont);
 3832     }
 3833 
 3834     // Check for existing monitor
 3835     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3836 
 3837     if (LockingMode == LM_MONITOR) {
 3838       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3839       __ b(cont);
 3840     } else if (LockingMode == LM_LEGACY) {
 3841       // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3842       __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3843 
 3844       // Initialize the box. (Must happen before we update the object mark!)
 3845       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3846 
 3847       // Compare object markWord with an unlocked value (tmp) and if
 3848       // equal exchange the stack address of our box with object markWord.
 3849       // On failure disp_hdr contains the possibly locked markWord.
 3850       __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3851                  /*release*/ true, /*weak*/ false, disp_hdr);
 3852       __ br(Assembler::EQ, cont);
 3853 
 3854       assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3855 
 3856       // If the compare-and-exchange succeeded, then we found an unlocked
 3857       // object, will have now locked it will continue at label cont
 3858 
 3859       // Check if the owner is self by comparing the value in the
 3860       // markWord of object (disp_hdr) with the stack pointer.
 3861       __ mov(rscratch1, sp);
 3862       __ sub(disp_hdr, disp_hdr, rscratch1);
 3863       __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3864       // If condition is true we are cont and hence we can store 0 as the
 3865       // displaced header in the box, which indicates that it is a recursive lock.
 3866       __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3867       __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3868       __ b(cont);
 3869     } else {
 3870       assert(LockingMode == LM_LIGHTWEIGHT, "must be");
 3871       __ fast_lock(oop, disp_hdr, tmp, rscratch1, no_count);
 3872       __ b(count);
 3873     }
 3874 
 3875     // Handle existing monitor.
 3876     __ bind(object_has_monitor);
 3877 
 3878     // The object's monitor m is unlocked iff m->owner == NULL,
 3879     // otherwise m->owner may contain a thread or a stack address.
 3880     //
 3881     // Try to CAS m->owner from NULL to current thread.
 3882     __ add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset())-markWord::monitor_value));
 3883     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3884                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 3885 
 3886     if (LockingMode != LM_LIGHTWEIGHT) {
 3887       // Store a non-null value into the box to avoid looking like a re-entrant
 3888       // lock. The fast-path monitor unlock code checks for
 3889       // markWord::monitor_value so use markWord::unused_mark which has the
 3890       // relevant bit set, and also matches ObjectSynchronizer::enter.
 3891       __ mov(tmp, (address)markWord::unused_mark().value());
 3892       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3893     }
 3894     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 3895 
 3896     __ cmp(rscratch1, rthread);
 3897     __ br(Assembler::NE, cont); // Check for recursive locking
 3898 
 3899     // Recursive lock case
 3900     __ increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1);
 3901     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 3902 
 3903     __ bind(cont);
 3904     // flag == EQ indicates success
 3905     // flag == NE indicates failure
 3906     __ br(Assembler::NE, no_count);
 3907 
 3908     __ bind(count);
 3909     __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
 3910 
 3911     __ bind(no_count);
 3912   %}
 3913 
 3914   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3915     C2_MacroAssembler _masm(&cbuf);
 3916     Register oop = as_Register($object$$reg);
 3917     Register box = as_Register($box$$reg);
 3918     Register disp_hdr = as_Register($tmp$$reg);
 3919     Register tmp = as_Register($tmp2$$reg);
 3920     Label cont;
 3921     Label object_has_monitor;
 3922     Label count, no_count;
 3923 
 3924     assert_different_registers(oop, box, tmp, disp_hdr);
 3925 
 3926     if (LockingMode == LM_LEGACY) {
 3927       // Find the lock address and load the displaced header from the stack.
 3928       __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3929 
 3930       // If the displaced header is 0, we have a recursive unlock.
 3931       __ cmp(disp_hdr, zr);
 3932       __ br(Assembler::EQ, cont);
 3933     }
 3934 
 3935     // Handle existing monitor.
 3936     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 3937     __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
 3938 
 3939     if (LockingMode == LM_MONITOR) {
 3940       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3941       __ b(cont);
 3942     } else if (LockingMode == LM_LEGACY) {
 3943       // Check if it is still a light weight lock, this is is true if we
 3944       // see the stack address of the basicLock in the markWord of the
 3945       // object.
 3946 
 3947       __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 3948                  /*release*/ true, /*weak*/ false, tmp);
 3949       __ b(cont);
 3950     } else {
 3951       assert(LockingMode == LM_LIGHTWEIGHT, "must be");
 3952       __ fast_unlock(oop, tmp, box, disp_hdr, no_count);
 3953       __ b(count);
 3954     }
 3955 
 3956     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3957 
 3958     // Handle existing monitor.
 3959     __ bind(object_has_monitor);
 3960     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 3961     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 3962 
 3963     if (LockingMode == LM_LIGHTWEIGHT) {
 3964       // If the owner is anonymous, we need to fix it -- in an outline stub.
 3965       Register tmp2 = disp_hdr;
 3966       __ ldr(tmp2, Address(tmp, ObjectMonitor::owner_offset()));
 3967       // We cannot use tbnz here, the target might be too far away and cannot
 3968       // be encoded.
 3969       __ tst(tmp2, (uint64_t)ObjectMonitor::ANONYMOUS_OWNER);
 3970       C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
 3971       Compile::current()->output()->add_stub(stub);
 3972       __ br(Assembler::NE, stub->entry());
 3973       __ bind(stub->continuation());
 3974     }
 3975 
 3976     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
 3977 
 3978     Label notRecursive;
 3979     __ cbz(disp_hdr, notRecursive);
 3980 
 3981     // Recursive lock
 3982     __ sub(disp_hdr, disp_hdr, 1u);
 3983     __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
 3984     __ cmp(disp_hdr, disp_hdr); // Sets flags for result
 3985     __ b(cont);
 3986 
 3987     __ bind(notRecursive);
 3988     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset()));
 3989     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset()));
 3990     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 3991     __ cmp(rscratch1, zr); // Sets flags for result
 3992     __ cbnz(rscratch1, cont);
 3993     // need a release store here
 3994     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset()));
 3995     __ stlr(zr, tmp); // set unowned
 3996 
 3997     __ bind(cont);
 3998     // flag == EQ indicates success
 3999     // flag == NE indicates failure
 4000     __ br(Assembler::NE, no_count);
 4001 
 4002     __ bind(count);
 4003     __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
 4004 
 4005     __ bind(no_count);
 4006   %}
 4007 
 4008 %}
 4009 
 4010 //----------FRAME--------------------------------------------------------------
 4011 // Definition of frame structure and management information.
 4012 //
 4013 //  S T A C K   L A Y O U T    Allocators stack-slot number
 4014 //                             |   (to get allocators register number
 4015 //  G  Owned by    |        |  v    add OptoReg::stack0())
 4016 //  r   CALLER     |        |
 4017 //  o     |        +--------+      pad to even-align allocators stack-slot
 4018 //  w     V        |  pad0  |        numbers; owned by CALLER
 4019 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 4020 //  h     ^        |   in   |  5
 4021 //        |        |  args  |  4   Holes in incoming args owned by SELF
 4022 //  |     |        |        |  3
 4023 //  |     |        +--------+
 4024 //  V     |        | old out|      Empty on Intel, window on Sparc
 4025 //        |    old |preserve|      Must be even aligned.
 4026 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 4027 //        |        |   in   |  3   area for Intel ret address
 4028 //     Owned by    |preserve|      Empty on Sparc.
 4029 //       SELF      +--------+
 4030 //        |        |  pad2  |  2   pad to align old SP
 4031 //        |        +--------+  1
 4032 //        |        | locks  |  0
 4033 //        |        +--------+----> OptoReg::stack0(), even aligned
 4034 //        |        |  pad1  | 11   pad to align new SP
 4035 //        |        +--------+
 4036 //        |        |        | 10
 4037 //        |        | spills |  9   spills
 4038 //        V        |        |  8   (pad0 slot for callee)
 4039 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 4040 //        ^        |  out   |  7
 4041 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 4042 //     Owned by    +--------+
 4043 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 4044 //        |    new |preserve|      Must be even-aligned.
 4045 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 4046 //        |        |        |
 4047 //
 4048 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 4049 //         known from SELF's arguments and the Java calling convention.
 4050 //         Region 6-7 is determined per call site.
 4051 // Note 2: If the calling convention leaves holes in the incoming argument
 4052 //         area, those holes are owned by SELF.  Holes in the outgoing area
 4053 //         are owned by the CALLEE.  Holes should not be necessary in the
 4054 //         incoming area, as the Java calling convention is completely under
 4055 //         the control of the AD file.  Doubles can be sorted and packed to
 4056 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 4057 //         varargs C calling conventions.
 4058 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 4059 //         even aligned with pad0 as needed.
 4060 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 4061 //           (the latter is true on Intel but is it false on AArch64?)
 4062 //         region 6-11 is even aligned; it may be padded out more so that
 4063 //         the region from SP to FP meets the minimum stack alignment.
 4064 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 4065 //         alignment.  Region 11, pad1, may be dynamically extended so that
 4066 //         SP meets the minimum alignment.
 4067 
 4068 frame %{
 4069   // These three registers define part of the calling convention
 4070   // between compiled code and the interpreter.
 4071 
 4072   // Inline Cache Register or Method for I2C.
 4073   inline_cache_reg(R12);
 4074 
 4075   // Number of stack slots consumed by locking an object
 4076   sync_stack_slots(2);
 4077 
 4078   // Compiled code's Frame Pointer
 4079   frame_pointer(R31);
 4080 
 4081   // Interpreter stores its frame pointer in a register which is
 4082   // stored to the stack by I2CAdaptors.
 4083   // I2CAdaptors convert from interpreted java to compiled java.
 4084   interpreter_frame_pointer(R29);
 4085 
 4086   // Stack alignment requirement
 4087   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4088 
 4089   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4090   // for calls to C.  Supports the var-args backing area for register parms.
 4091   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4092 
 4093   // The after-PROLOG location of the return address.  Location of
 4094   // return address specifies a type (REG or STACK) and a number
 4095   // representing the register number (i.e. - use a register name) or
 4096   // stack slot.
 4097   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4098   // Otherwise, it is above the locks and verification slot and alignment word
 4099   // TODO this may well be correct but need to check why that - 2 is there
 4100   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4101   // which folds in the space used for monitors
 4102   return_addr(STACK - 2 +
 4103               align_up((Compile::current()->in_preserve_stack_slots() +
 4104                         Compile::current()->fixed_slots()),
 4105                        stack_alignment_in_slots()));
 4106 
 4107   // Location of compiled Java return values.  Same as C for now.
 4108   return_value
 4109   %{
 4110     // TODO do we allow ideal_reg == Op_RegN???
 4111     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4112            "only return normal values");
 4113 
 4114     static const int lo[Op_RegL + 1] = { // enum name
 4115       0,                                 // Op_Node
 4116       0,                                 // Op_Set
 4117       R0_num,                            // Op_RegN
 4118       R0_num,                            // Op_RegI
 4119       R0_num,                            // Op_RegP
 4120       V0_num,                            // Op_RegF
 4121       V0_num,                            // Op_RegD
 4122       R0_num                             // Op_RegL
 4123     };
 4124 
 4125     static const int hi[Op_RegL + 1] = { // enum name
 4126       0,                                 // Op_Node
 4127       0,                                 // Op_Set
 4128       OptoReg::Bad,                      // Op_RegN
 4129       OptoReg::Bad,                      // Op_RegI
 4130       R0_H_num,                          // Op_RegP
 4131       OptoReg::Bad,                      // Op_RegF
 4132       V0_H_num,                          // Op_RegD
 4133       R0_H_num                           // Op_RegL
 4134     };
 4135 
 4136     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4137   %}
 4138 %}
 4139 
 4140 //----------ATTRIBUTES---------------------------------------------------------
 4141 //----------Operand Attributes-------------------------------------------------
 4142 op_attrib op_cost(1);        // Required cost attribute
 4143 
 4144 //----------Instruction Attributes---------------------------------------------
 4145 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4146 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4147 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4148                                 // a non-matching short branch variant
 4149                                 // of some long branch?
 4150 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4151                                 // be a power of 2) specifies the
 4152                                 // alignment that some part of the
 4153                                 // instruction (not necessarily the
 4154                                 // start) requires.  If > 1, a
 4155                                 // compute_padding() function must be
 4156                                 // provided for the instruction
 4157 
 4158 //----------OPERANDS-----------------------------------------------------------
 4159 // Operand definitions must precede instruction definitions for correct parsing
 4160 // in the ADLC because operands constitute user defined types which are used in
 4161 // instruction definitions.
 4162 
 4163 //----------Simple Operands----------------------------------------------------
 4164 
 4165 // Integer operands 32 bit
 4166 // 32 bit immediate
 4167 operand immI()
 4168 %{
 4169   match(ConI);
 4170 
 4171   op_cost(0);
 4172   format %{ %}
 4173   interface(CONST_INTER);
 4174 %}
 4175 
 4176 // 32 bit zero
 4177 operand immI0()
 4178 %{
 4179   predicate(n->get_int() == 0);
 4180   match(ConI);
 4181 
 4182   op_cost(0);
 4183   format %{ %}
 4184   interface(CONST_INTER);
 4185 %}
 4186 
 4187 // 32 bit unit increment
 4188 operand immI_1()
 4189 %{
 4190   predicate(n->get_int() == 1);
 4191   match(ConI);
 4192 
 4193   op_cost(0);
 4194   format %{ %}
 4195   interface(CONST_INTER);
 4196 %}
 4197 
 4198 // 32 bit unit decrement
 4199 operand immI_M1()
 4200 %{
 4201   predicate(n->get_int() == -1);
 4202   match(ConI);
 4203 
 4204   op_cost(0);
 4205   format %{ %}
 4206   interface(CONST_INTER);
 4207 %}
 4208 
 4209 // Shift values for add/sub extension shift
 4210 operand immIExt()
 4211 %{
 4212   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4213   match(ConI);
 4214 
 4215   op_cost(0);
 4216   format %{ %}
 4217   interface(CONST_INTER);
 4218 %}
 4219 
 4220 operand immI_gt_1()
 4221 %{
 4222   predicate(n->get_int() > 1);
 4223   match(ConI);
 4224 
 4225   op_cost(0);
 4226   format %{ %}
 4227   interface(CONST_INTER);
 4228 %}
 4229 
 4230 operand immI_le_4()
 4231 %{
 4232   predicate(n->get_int() <= 4);
 4233   match(ConI);
 4234 
 4235   op_cost(0);
 4236   format %{ %}
 4237   interface(CONST_INTER);
 4238 %}
 4239 
 4240 operand immI_16()
 4241 %{
 4242   predicate(n->get_int() == 16);
 4243   match(ConI);
 4244 
 4245   op_cost(0);
 4246   format %{ %}
 4247   interface(CONST_INTER);
 4248 %}
 4249 
 4250 operand immI_24()
 4251 %{
 4252   predicate(n->get_int() == 24);
 4253   match(ConI);
 4254 
 4255   op_cost(0);
 4256   format %{ %}
 4257   interface(CONST_INTER);
 4258 %}
 4259 
 4260 operand immI_32()
 4261 %{
 4262   predicate(n->get_int() == 32);
 4263   match(ConI);
 4264 
 4265   op_cost(0);
 4266   format %{ %}
 4267   interface(CONST_INTER);
 4268 %}
 4269 
 4270 operand immI_48()
 4271 %{
 4272   predicate(n->get_int() == 48);
 4273   match(ConI);
 4274 
 4275   op_cost(0);
 4276   format %{ %}
 4277   interface(CONST_INTER);
 4278 %}
 4279 
 4280 operand immI_56()
 4281 %{
 4282   predicate(n->get_int() == 56);
 4283   match(ConI);
 4284 
 4285   op_cost(0);
 4286   format %{ %}
 4287   interface(CONST_INTER);
 4288 %}
 4289 
 4290 operand immI_63()
 4291 %{
 4292   predicate(n->get_int() == 63);
 4293   match(ConI);
 4294 
 4295   op_cost(0);
 4296   format %{ %}
 4297   interface(CONST_INTER);
 4298 %}
 4299 
 4300 operand immI_64()
 4301 %{
 4302   predicate(n->get_int() == 64);
 4303   match(ConI);
 4304 
 4305   op_cost(0);
 4306   format %{ %}
 4307   interface(CONST_INTER);
 4308 %}
 4309 
 4310 operand immI_255()
 4311 %{
 4312   predicate(n->get_int() == 255);
 4313   match(ConI);
 4314 
 4315   op_cost(0);
 4316   format %{ %}
 4317   interface(CONST_INTER);
 4318 %}
 4319 
 4320 operand immI_65535()
 4321 %{
 4322   predicate(n->get_int() == 65535);
 4323   match(ConI);
 4324 
 4325   op_cost(0);
 4326   format %{ %}
 4327   interface(CONST_INTER);
 4328 %}
 4329 
 4330 operand immI_positive()
 4331 %{
 4332   predicate(n->get_int() > 0);
 4333   match(ConI);
 4334 
 4335   op_cost(0);
 4336   format %{ %}
 4337   interface(CONST_INTER);
 4338 %}
 4339 
 4340 // BoolTest condition for signed compare
 4341 operand immI_cmp_cond()
 4342 %{
 4343   predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
 4344   match(ConI);
 4345 
 4346   op_cost(0);
 4347   format %{ %}
 4348   interface(CONST_INTER);
 4349 %}
 4350 
 4351 // BoolTest condition for unsigned compare
 4352 operand immI_cmpU_cond()
 4353 %{
 4354   predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
 4355   match(ConI);
 4356 
 4357   op_cost(0);
 4358   format %{ %}
 4359   interface(CONST_INTER);
 4360 %}
 4361 
 4362 operand immL_255()
 4363 %{
 4364   predicate(n->get_long() == 255L);
 4365   match(ConL);
 4366 
 4367   op_cost(0);
 4368   format %{ %}
 4369   interface(CONST_INTER);
 4370 %}
 4371 
 4372 operand immL_65535()
 4373 %{
 4374   predicate(n->get_long() == 65535L);
 4375   match(ConL);
 4376 
 4377   op_cost(0);
 4378   format %{ %}
 4379   interface(CONST_INTER);
 4380 %}
 4381 
 4382 operand immL_4294967295()
 4383 %{
 4384   predicate(n->get_long() == 4294967295L);
 4385   match(ConL);
 4386 
 4387   op_cost(0);
 4388   format %{ %}
 4389   interface(CONST_INTER);
 4390 %}
 4391 
 4392 operand immL_bitmask()
 4393 %{
 4394   predicate((n->get_long() != 0)
 4395             && ((n->get_long() & 0xc000000000000000l) == 0)
 4396             && is_power_of_2(n->get_long() + 1));
 4397   match(ConL);
 4398 
 4399   op_cost(0);
 4400   format %{ %}
 4401   interface(CONST_INTER);
 4402 %}
 4403 
 4404 operand immI_bitmask()
 4405 %{
 4406   predicate((n->get_int() != 0)
 4407             && ((n->get_int() & 0xc0000000) == 0)
 4408             && is_power_of_2(n->get_int() + 1));
 4409   match(ConI);
 4410 
 4411   op_cost(0);
 4412   format %{ %}
 4413   interface(CONST_INTER);
 4414 %}
 4415 
 4416 operand immL_positive_bitmaskI()
 4417 %{
 4418   predicate((n->get_long() != 0)
 4419             && ((julong)n->get_long() < 0x80000000ULL)
 4420             && is_power_of_2(n->get_long() + 1));
 4421   match(ConL);
 4422 
 4423   op_cost(0);
 4424   format %{ %}
 4425   interface(CONST_INTER);
 4426 %}
 4427 
 4428 // Scale values for scaled offset addressing modes (up to long but not quad)
 4429 operand immIScale()
 4430 %{
 4431   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4432   match(ConI);
 4433 
 4434   op_cost(0);
 4435   format %{ %}
 4436   interface(CONST_INTER);
 4437 %}
 4438 
 4439 // 26 bit signed offset -- for pc-relative branches
 4440 operand immI26()
 4441 %{
 4442   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4443   match(ConI);
 4444 
 4445   op_cost(0);
 4446   format %{ %}
 4447   interface(CONST_INTER);
 4448 %}
 4449 
 4450 // 19 bit signed offset -- for pc-relative loads
 4451 operand immI19()
 4452 %{
 4453   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4454   match(ConI);
 4455 
 4456   op_cost(0);
 4457   format %{ %}
 4458   interface(CONST_INTER);
 4459 %}
 4460 
 4461 // 5 bit signed integer
 4462 operand immI5()
 4463 %{
 4464   predicate(Assembler::is_simm(n->get_int(), 5));
 4465   match(ConI);
 4466 
 4467   op_cost(0);
 4468   format %{ %}
 4469   interface(CONST_INTER);
 4470 %}
 4471 
 4472 // 7 bit unsigned integer
 4473 operand immIU7()
 4474 %{
 4475   predicate(Assembler::is_uimm(n->get_int(), 7));
 4476   match(ConI);
 4477 
 4478   op_cost(0);
 4479   format %{ %}
 4480   interface(CONST_INTER);
 4481 %}
 4482 
 4483 // 12 bit unsigned offset -- for base plus immediate loads
 4484 operand immIU12()
 4485 %{
 4486   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4487   match(ConI);
 4488 
 4489   op_cost(0);
 4490   format %{ %}
 4491   interface(CONST_INTER);
 4492 %}
 4493 
 4494 operand immLU12()
 4495 %{
 4496   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4497   match(ConL);
 4498 
 4499   op_cost(0);
 4500   format %{ %}
 4501   interface(CONST_INTER);
 4502 %}
 4503 
 4504 // Offset for scaled or unscaled immediate loads and stores
 4505 operand immIOffset()
 4506 %{
 4507   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4508   match(ConI);
 4509 
 4510   op_cost(0);
 4511   format %{ %}
 4512   interface(CONST_INTER);
 4513 %}
 4514 
 4515 operand immIOffset1()
 4516 %{
 4517   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4518   match(ConI);
 4519 
 4520   op_cost(0);
 4521   format %{ %}
 4522   interface(CONST_INTER);
 4523 %}
 4524 
 4525 operand immIOffset2()
 4526 %{
 4527   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4528   match(ConI);
 4529 
 4530   op_cost(0);
 4531   format %{ %}
 4532   interface(CONST_INTER);
 4533 %}
 4534 
 4535 operand immIOffset4()
 4536 %{
 4537   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4538   match(ConI);
 4539 
 4540   op_cost(0);
 4541   format %{ %}
 4542   interface(CONST_INTER);
 4543 %}
 4544 
 4545 operand immIOffset8()
 4546 %{
 4547   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4548   match(ConI);
 4549 
 4550   op_cost(0);
 4551   format %{ %}
 4552   interface(CONST_INTER);
 4553 %}
 4554 
 4555 operand immIOffset16()
 4556 %{
 4557   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4558   match(ConI);
 4559 
 4560   op_cost(0);
 4561   format %{ %}
 4562   interface(CONST_INTER);
 4563 %}
 4564 
 4565 operand immLoffset()
 4566 %{
 4567   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4568   match(ConL);
 4569 
 4570   op_cost(0);
 4571   format %{ %}
 4572   interface(CONST_INTER);
 4573 %}
 4574 
 4575 operand immLoffset1()
 4576 %{
 4577   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4578   match(ConL);
 4579 
 4580   op_cost(0);
 4581   format %{ %}
 4582   interface(CONST_INTER);
 4583 %}
 4584 
 4585 operand immLoffset2()
 4586 %{
 4587   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4588   match(ConL);
 4589 
 4590   op_cost(0);
 4591   format %{ %}
 4592   interface(CONST_INTER);
 4593 %}
 4594 
 4595 operand immLoffset4()
 4596 %{
 4597   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4598   match(ConL);
 4599 
 4600   op_cost(0);
 4601   format %{ %}
 4602   interface(CONST_INTER);
 4603 %}
 4604 
 4605 operand immLoffset8()
 4606 %{
 4607   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4608   match(ConL);
 4609 
 4610   op_cost(0);
 4611   format %{ %}
 4612   interface(CONST_INTER);
 4613 %}
 4614 
 4615 operand immLoffset16()
 4616 %{
 4617   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4618   match(ConL);
 4619 
 4620   op_cost(0);
 4621   format %{ %}
 4622   interface(CONST_INTER);
 4623 %}
 4624 
 4625 // 5 bit signed long integer
 4626 operand immL5()
 4627 %{
 4628   predicate(Assembler::is_simm(n->get_long(), 5));
 4629   match(ConL);
 4630 
 4631   op_cost(0);
 4632   format %{ %}
 4633   interface(CONST_INTER);
 4634 %}
 4635 
 4636 // 7 bit unsigned long integer
 4637 operand immLU7()
 4638 %{
 4639   predicate(Assembler::is_uimm(n->get_long(), 7));
 4640   match(ConL);
 4641 
 4642   op_cost(0);
 4643   format %{ %}
 4644   interface(CONST_INTER);
 4645 %}
 4646 
 4647 // 8 bit signed value.
 4648 operand immI8()
 4649 %{
 4650   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4651   match(ConI);
 4652 
 4653   op_cost(0);
 4654   format %{ %}
 4655   interface(CONST_INTER);
 4656 %}
 4657 
 4658 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4659 operand immI8_shift8()
 4660 %{
 4661   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4662             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4663   match(ConI);
 4664 
 4665   op_cost(0);
 4666   format %{ %}
 4667   interface(CONST_INTER);
 4668 %}
 4669 
 4670 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4671 operand immL8_shift8()
 4672 %{
 4673   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4674             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4675   match(ConL);
 4676 
 4677   op_cost(0);
 4678   format %{ %}
 4679   interface(CONST_INTER);
 4680 %}
 4681 
 4682 // 8 bit integer valid for vector add sub immediate
 4683 operand immBAddSubV()
 4684 %{
 4685   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4686   match(ConI);
 4687 
 4688   op_cost(0);
 4689   format %{ %}
 4690   interface(CONST_INTER);
 4691 %}
 4692 
 4693 // 32 bit integer valid for add sub immediate
 4694 operand immIAddSub()
 4695 %{
 4696   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4697   match(ConI);
 4698   op_cost(0);
 4699   format %{ %}
 4700   interface(CONST_INTER);
 4701 %}
 4702 
 4703 // 32 bit integer valid for vector add sub immediate
 4704 operand immIAddSubV()
 4705 %{
 4706   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4707   match(ConI);
 4708 
 4709   op_cost(0);
 4710   format %{ %}
 4711   interface(CONST_INTER);
 4712 %}
 4713 
 4714 // 32 bit unsigned integer valid for logical immediate
 4715 
 4716 operand immBLog()
 4717 %{
 4718   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4719   match(ConI);
 4720 
 4721   op_cost(0);
 4722   format %{ %}
 4723   interface(CONST_INTER);
 4724 %}
 4725 
 4726 operand immSLog()
 4727 %{
 4728   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4729   match(ConI);
 4730 
 4731   op_cost(0);
 4732   format %{ %}
 4733   interface(CONST_INTER);
 4734 %}
 4735 
 4736 operand immILog()
 4737 %{
 4738   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4739   match(ConI);
 4740 
 4741   op_cost(0);
 4742   format %{ %}
 4743   interface(CONST_INTER);
 4744 %}
 4745 
 4746 // Integer operands 64 bit
 4747 // 64 bit immediate
 4748 operand immL()
 4749 %{
 4750   match(ConL);
 4751 
 4752   op_cost(0);
 4753   format %{ %}
 4754   interface(CONST_INTER);
 4755 %}
 4756 
 4757 // 64 bit zero
 4758 operand immL0()
 4759 %{
 4760   predicate(n->get_long() == 0);
 4761   match(ConL);
 4762 
 4763   op_cost(0);
 4764   format %{ %}
 4765   interface(CONST_INTER);
 4766 %}
 4767 
 4768 // 64 bit unit increment
 4769 operand immL_1()
 4770 %{
 4771   predicate(n->get_long() == 1);
 4772   match(ConL);
 4773 
 4774   op_cost(0);
 4775   format %{ %}
 4776   interface(CONST_INTER);
 4777 %}
 4778 
 4779 // 64 bit unit decrement
 4780 operand immL_M1()
 4781 %{
 4782   predicate(n->get_long() == -1);
 4783   match(ConL);
 4784 
 4785   op_cost(0);
 4786   format %{ %}
 4787   interface(CONST_INTER);
 4788 %}
 4789 
 4790 // 32 bit offset of pc in thread anchor
 4791 
 4792 operand immL_pc_off()
 4793 %{
 4794   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4795                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4796   match(ConL);
 4797 
 4798   op_cost(0);
 4799   format %{ %}
 4800   interface(CONST_INTER);
 4801 %}
 4802 
 4803 // 64 bit integer valid for add sub immediate
 4804 operand immLAddSub()
 4805 %{
 4806   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4807   match(ConL);
 4808   op_cost(0);
 4809   format %{ %}
 4810   interface(CONST_INTER);
 4811 %}
 4812 
 4813 // 64 bit integer valid for addv subv immediate
 4814 operand immLAddSubV()
 4815 %{
 4816   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4817   match(ConL);
 4818 
 4819   op_cost(0);
 4820   format %{ %}
 4821   interface(CONST_INTER);
 4822 %}
 4823 
 4824 // 64 bit integer valid for logical immediate
 4825 operand immLLog()
 4826 %{
 4827   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4828   match(ConL);
 4829   op_cost(0);
 4830   format %{ %}
 4831   interface(CONST_INTER);
 4832 %}
 4833 
 4834 // Long Immediate: low 32-bit mask
 4835 operand immL_32bits()
 4836 %{
 4837   predicate(n->get_long() == 0xFFFFFFFFL);
 4838   match(ConL);
 4839   op_cost(0);
 4840   format %{ %}
 4841   interface(CONST_INTER);
 4842 %}
 4843 
 4844 // Pointer operands
 4845 // Pointer Immediate
 4846 operand immP()
 4847 %{
 4848   match(ConP);
 4849 
 4850   op_cost(0);
 4851   format %{ %}
 4852   interface(CONST_INTER);
 4853 %}
 4854 
 4855 // NULL Pointer Immediate
 4856 operand immP0()
 4857 %{
 4858   predicate(n->get_ptr() == 0);
 4859   match(ConP);
 4860 
 4861   op_cost(0);
 4862   format %{ %}
 4863   interface(CONST_INTER);
 4864 %}
 4865 
 4866 // Pointer Immediate One
 4867 // this is used in object initialization (initial object header)
 4868 operand immP_1()
 4869 %{
 4870   predicate(n->get_ptr() == 1);
 4871   match(ConP);
 4872 
 4873   op_cost(0);
 4874   format %{ %}
 4875   interface(CONST_INTER);
 4876 %}
 4877 
 4878 // Card Table Byte Map Base
 4879 operand immByteMapBase()
 4880 %{
 4881   // Get base of card map
 4882   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4883             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4884   match(ConP);
 4885 
 4886   op_cost(0);
 4887   format %{ %}
 4888   interface(CONST_INTER);
 4889 %}
 4890 
 4891 // Pointer Immediate Minus One
 4892 // this is used when we want to write the current PC to the thread anchor
 4893 operand immP_M1()
 4894 %{
 4895   predicate(n->get_ptr() == -1);
 4896   match(ConP);
 4897 
 4898   op_cost(0);
 4899   format %{ %}
 4900   interface(CONST_INTER);
 4901 %}
 4902 
 4903 // Pointer Immediate Minus Two
 4904 // this is used when we want to write the current PC to the thread anchor
 4905 operand immP_M2()
 4906 %{
 4907   predicate(n->get_ptr() == -2);
 4908   match(ConP);
 4909 
 4910   op_cost(0);
 4911   format %{ %}
 4912   interface(CONST_INTER);
 4913 %}
 4914 
 4915 // Float and Double operands
 4916 // Double Immediate
 4917 operand immD()
 4918 %{
 4919   match(ConD);
 4920   op_cost(0);
 4921   format %{ %}
 4922   interface(CONST_INTER);
 4923 %}
 4924 
 4925 // Double Immediate: +0.0d
 4926 operand immD0()
 4927 %{
 4928   predicate(jlong_cast(n->getd()) == 0);
 4929   match(ConD);
 4930 
 4931   op_cost(0);
 4932   format %{ %}
 4933   interface(CONST_INTER);
 4934 %}
 4935 
 4936 // constant 'double +0.0'.
 4937 operand immDPacked()
 4938 %{
 4939   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4940   match(ConD);
 4941   op_cost(0);
 4942   format %{ %}
 4943   interface(CONST_INTER);
 4944 %}
 4945 
 4946 // Float Immediate
 4947 operand immF()
 4948 %{
 4949   match(ConF);
 4950   op_cost(0);
 4951   format %{ %}
 4952   interface(CONST_INTER);
 4953 %}
 4954 
 4955 // Float Immediate: +0.0f.
 4956 operand immF0()
 4957 %{
 4958   predicate(jint_cast(n->getf()) == 0);
 4959   match(ConF);
 4960 
 4961   op_cost(0);
 4962   format %{ %}
 4963   interface(CONST_INTER);
 4964 %}
 4965 
 4966 //
 4967 operand immFPacked()
 4968 %{
 4969   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4970   match(ConF);
 4971   op_cost(0);
 4972   format %{ %}
 4973   interface(CONST_INTER);
 4974 %}
 4975 
 4976 // Narrow pointer operands
 4977 // Narrow Pointer Immediate
 4978 operand immN()
 4979 %{
 4980   match(ConN);
 4981 
 4982   op_cost(0);
 4983   format %{ %}
 4984   interface(CONST_INTER);
 4985 %}
 4986 
 4987 // Narrow NULL Pointer Immediate
 4988 operand immN0()
 4989 %{
 4990   predicate(n->get_narrowcon() == 0);
 4991   match(ConN);
 4992 
 4993   op_cost(0);
 4994   format %{ %}
 4995   interface(CONST_INTER);
 4996 %}
 4997 
 4998 operand immNKlass()
 4999 %{
 5000   match(ConNKlass);
 5001 
 5002   op_cost(0);
 5003   format %{ %}
 5004   interface(CONST_INTER);
 5005 %}
 5006 
 5007 // Integer 32 bit Register Operands
 5008 // Integer 32 bitRegister (excludes SP)
 5009 operand iRegI()
 5010 %{
 5011   constraint(ALLOC_IN_RC(any_reg32));
 5012   match(RegI);
 5013   match(iRegINoSp);
 5014   op_cost(0);
 5015   format %{ %}
 5016   interface(REG_INTER);
 5017 %}
 5018 
 5019 // Integer 32 bit Register not Special
 5020 operand iRegINoSp()
 5021 %{
 5022   constraint(ALLOC_IN_RC(no_special_reg32));
 5023   match(RegI);
 5024   op_cost(0);
 5025   format %{ %}
 5026   interface(REG_INTER);
 5027 %}
 5028 
 5029 // Integer 64 bit Register Operands
 5030 // Integer 64 bit Register (includes SP)
 5031 operand iRegL()
 5032 %{
 5033   constraint(ALLOC_IN_RC(any_reg));
 5034   match(RegL);
 5035   match(iRegLNoSp);
 5036   op_cost(0);
 5037   format %{ %}
 5038   interface(REG_INTER);
 5039 %}
 5040 
 5041 // Integer 64 bit Register not Special
 5042 operand iRegLNoSp()
 5043 %{
 5044   constraint(ALLOC_IN_RC(no_special_reg));
 5045   match(RegL);
 5046   match(iRegL_R0);
 5047   format %{ %}
 5048   interface(REG_INTER);
 5049 %}
 5050 
 5051 // Pointer Register Operands
 5052 // Pointer Register
 5053 operand iRegP()
 5054 %{
 5055   constraint(ALLOC_IN_RC(ptr_reg));
 5056   match(RegP);
 5057   match(iRegPNoSp);
 5058   match(iRegP_R0);
 5059   //match(iRegP_R2);
 5060   //match(iRegP_R4);
 5061   match(iRegP_R5);
 5062   match(thread_RegP);
 5063   op_cost(0);
 5064   format %{ %}
 5065   interface(REG_INTER);
 5066 %}
 5067 
 5068 // Pointer 64 bit Register not Special
 5069 operand iRegPNoSp()
 5070 %{
 5071   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 5072   match(RegP);
 5073   // match(iRegP);
 5074   // match(iRegP_R0);
 5075   // match(iRegP_R2);
 5076   // match(iRegP_R4);
 5077   // match(iRegP_R5);
 5078   // match(thread_RegP);
 5079   op_cost(0);
 5080   format %{ %}
 5081   interface(REG_INTER);
 5082 %}
 5083 
 5084 // Pointer 64 bit Register R0 only
 5085 operand iRegP_R0()
 5086 %{
 5087   constraint(ALLOC_IN_RC(r0_reg));
 5088   match(RegP);
 5089   // match(iRegP);
 5090   match(iRegPNoSp);
 5091   op_cost(0);
 5092   format %{ %}
 5093   interface(REG_INTER);
 5094 %}
 5095 
 5096 // Pointer 64 bit Register R1 only
 5097 operand iRegP_R1()
 5098 %{
 5099   constraint(ALLOC_IN_RC(r1_reg));
 5100   match(RegP);
 5101   // match(iRegP);
 5102   match(iRegPNoSp);
 5103   op_cost(0);
 5104   format %{ %}
 5105   interface(REG_INTER);
 5106 %}
 5107 
 5108 // Pointer 64 bit Register R2 only
 5109 operand iRegP_R2()
 5110 %{
 5111   constraint(ALLOC_IN_RC(r2_reg));
 5112   match(RegP);
 5113   // match(iRegP);
 5114   match(iRegPNoSp);
 5115   op_cost(0);
 5116   format %{ %}
 5117   interface(REG_INTER);
 5118 %}
 5119 
 5120 // Pointer 64 bit Register R3 only
 5121 operand iRegP_R3()
 5122 %{
 5123   constraint(ALLOC_IN_RC(r3_reg));
 5124   match(RegP);
 5125   // match(iRegP);
 5126   match(iRegPNoSp);
 5127   op_cost(0);
 5128   format %{ %}
 5129   interface(REG_INTER);
 5130 %}
 5131 
 5132 // Pointer 64 bit Register R4 only
 5133 operand iRegP_R4()
 5134 %{
 5135   constraint(ALLOC_IN_RC(r4_reg));
 5136   match(RegP);
 5137   // match(iRegP);
 5138   match(iRegPNoSp);
 5139   op_cost(0);
 5140   format %{ %}
 5141   interface(REG_INTER);
 5142 %}
 5143 
 5144 // Pointer 64 bit Register R5 only
 5145 operand iRegP_R5()
 5146 %{
 5147   constraint(ALLOC_IN_RC(r5_reg));
 5148   match(RegP);
 5149   // match(iRegP);
 5150   match(iRegPNoSp);
 5151   op_cost(0);
 5152   format %{ %}
 5153   interface(REG_INTER);
 5154 %}
 5155 
 5156 // Pointer 64 bit Register R10 only
 5157 operand iRegP_R10()
 5158 %{
 5159   constraint(ALLOC_IN_RC(r10_reg));
 5160   match(RegP);
 5161   // match(iRegP);
 5162   match(iRegPNoSp);
 5163   op_cost(0);
 5164   format %{ %}
 5165   interface(REG_INTER);
 5166 %}
 5167 
 5168 // Long 64 bit Register R0 only
 5169 operand iRegL_R0()
 5170 %{
 5171   constraint(ALLOC_IN_RC(r0_reg));
 5172   match(RegL);
 5173   match(iRegLNoSp);
 5174   op_cost(0);
 5175   format %{ %}
 5176   interface(REG_INTER);
 5177 %}
 5178 
 5179 // Long 64 bit Register R2 only
 5180 operand iRegL_R2()
 5181 %{
 5182   constraint(ALLOC_IN_RC(r2_reg));
 5183   match(RegL);
 5184   match(iRegLNoSp);
 5185   op_cost(0);
 5186   format %{ %}
 5187   interface(REG_INTER);
 5188 %}
 5189 
 5190 // Long 64 bit Register R3 only
 5191 operand iRegL_R3()
 5192 %{
 5193   constraint(ALLOC_IN_RC(r3_reg));
 5194   match(RegL);
 5195   match(iRegLNoSp);
 5196   op_cost(0);
 5197   format %{ %}
 5198   interface(REG_INTER);
 5199 %}
 5200 
 5201 // Long 64 bit Register R11 only
 5202 operand iRegL_R11()
 5203 %{
 5204   constraint(ALLOC_IN_RC(r11_reg));
 5205   match(RegL);
 5206   match(iRegLNoSp);
 5207   op_cost(0);
 5208   format %{ %}
 5209   interface(REG_INTER);
 5210 %}
 5211 
 5212 // Pointer 64 bit Register FP only
 5213 operand iRegP_FP()
 5214 %{
 5215   constraint(ALLOC_IN_RC(fp_reg));
 5216   match(RegP);
 5217   // match(iRegP);
 5218   op_cost(0);
 5219   format %{ %}
 5220   interface(REG_INTER);
 5221 %}
 5222 
 5223 // Register R0 only
 5224 operand iRegI_R0()
 5225 %{
 5226   constraint(ALLOC_IN_RC(int_r0_reg));
 5227   match(RegI);
 5228   match(iRegINoSp);
 5229   op_cost(0);
 5230   format %{ %}
 5231   interface(REG_INTER);
 5232 %}
 5233 
 5234 // Register R2 only
 5235 operand iRegI_R2()
 5236 %{
 5237   constraint(ALLOC_IN_RC(int_r2_reg));
 5238   match(RegI);
 5239   match(iRegINoSp);
 5240   op_cost(0);
 5241   format %{ %}
 5242   interface(REG_INTER);
 5243 %}
 5244 
 5245 // Register R3 only
 5246 operand iRegI_R3()
 5247 %{
 5248   constraint(ALLOC_IN_RC(int_r3_reg));
 5249   match(RegI);
 5250   match(iRegINoSp);
 5251   op_cost(0);
 5252   format %{ %}
 5253   interface(REG_INTER);
 5254 %}
 5255 
 5256 
 5257 // Register R4 only
 5258 operand iRegI_R4()
 5259 %{
 5260   constraint(ALLOC_IN_RC(int_r4_reg));
 5261   match(RegI);
 5262   match(iRegINoSp);
 5263   op_cost(0);
 5264   format %{ %}
 5265   interface(REG_INTER);
 5266 %}
 5267 
 5268 
 5269 // Pointer Register Operands
 5270 // Narrow Pointer Register
 5271 operand iRegN()
 5272 %{
 5273   constraint(ALLOC_IN_RC(any_reg32));
 5274   match(RegN);
 5275   match(iRegNNoSp);
 5276   op_cost(0);
 5277   format %{ %}
 5278   interface(REG_INTER);
 5279 %}
 5280 
 5281 operand iRegN_R0()
 5282 %{
 5283   constraint(ALLOC_IN_RC(r0_reg));
 5284   match(iRegN);
 5285   op_cost(0);
 5286   format %{ %}
 5287   interface(REG_INTER);
 5288 %}
 5289 
 5290 operand iRegN_R2()
 5291 %{
 5292   constraint(ALLOC_IN_RC(r2_reg));
 5293   match(iRegN);
 5294   op_cost(0);
 5295   format %{ %}
 5296   interface(REG_INTER);
 5297 %}
 5298 
 5299 operand iRegN_R3()
 5300 %{
 5301   constraint(ALLOC_IN_RC(r3_reg));
 5302   match(iRegN);
 5303   op_cost(0);
 5304   format %{ %}
 5305   interface(REG_INTER);
 5306 %}
 5307 
 5308 // Integer 64 bit Register not Special
 5309 operand iRegNNoSp()
 5310 %{
 5311   constraint(ALLOC_IN_RC(no_special_reg32));
 5312   match(RegN);
 5313   op_cost(0);
 5314   format %{ %}
 5315   interface(REG_INTER);
 5316 %}
 5317 
 5318 // Float Register
 5319 // Float register operands
 5320 operand vRegF()
 5321 %{
 5322   constraint(ALLOC_IN_RC(float_reg));
 5323   match(RegF);
 5324 
 5325   op_cost(0);
 5326   format %{ %}
 5327   interface(REG_INTER);
 5328 %}
 5329 
 5330 // Double Register
 5331 // Double register operands
 5332 operand vRegD()
 5333 %{
 5334   constraint(ALLOC_IN_RC(double_reg));
 5335   match(RegD);
 5336 
 5337   op_cost(0);
 5338   format %{ %}
 5339   interface(REG_INTER);
 5340 %}
 5341 
 5342 // Generic vector class. This will be used for
 5343 // all vector operands, including NEON and SVE.
 5344 operand vReg()
 5345 %{
 5346   constraint(ALLOC_IN_RC(dynamic));
 5347   match(VecA);
 5348   match(VecD);
 5349   match(VecX);
 5350 
 5351   op_cost(0);
 5352   format %{ %}
 5353   interface(REG_INTER);
 5354 %}
 5355 
 5356 operand vecA()
 5357 %{
 5358   constraint(ALLOC_IN_RC(vectora_reg));
 5359   match(VecA);
 5360 
 5361   op_cost(0);
 5362   format %{ %}
 5363   interface(REG_INTER);
 5364 %}
 5365 
 5366 operand vecD()
 5367 %{
 5368   constraint(ALLOC_IN_RC(vectord_reg));
 5369   match(VecD);
 5370 
 5371   op_cost(0);
 5372   format %{ %}
 5373   interface(REG_INTER);
 5374 %}
 5375 
 5376 operand vecX()
 5377 %{
 5378   constraint(ALLOC_IN_RC(vectorx_reg));
 5379   match(VecX);
 5380 
 5381   op_cost(0);
 5382   format %{ %}
 5383   interface(REG_INTER);
 5384 %}
 5385 
 5386 operand vRegD_V0()
 5387 %{
 5388   constraint(ALLOC_IN_RC(v0_reg));
 5389   match(RegD);
 5390   op_cost(0);
 5391   format %{ %}
 5392   interface(REG_INTER);
 5393 %}
 5394 
 5395 operand vRegD_V1()
 5396 %{
 5397   constraint(ALLOC_IN_RC(v1_reg));
 5398   match(RegD);
 5399   op_cost(0);
 5400   format %{ %}
 5401   interface(REG_INTER);
 5402 %}
 5403 
 5404 operand vRegD_V2()
 5405 %{
 5406   constraint(ALLOC_IN_RC(v2_reg));
 5407   match(RegD);
 5408   op_cost(0);
 5409   format %{ %}
 5410   interface(REG_INTER);
 5411 %}
 5412 
 5413 operand vRegD_V3()
 5414 %{
 5415   constraint(ALLOC_IN_RC(v3_reg));
 5416   match(RegD);
 5417   op_cost(0);
 5418   format %{ %}
 5419   interface(REG_INTER);
 5420 %}
 5421 
 5422 operand vRegD_V4()
 5423 %{
 5424   constraint(ALLOC_IN_RC(v4_reg));
 5425   match(RegD);
 5426   op_cost(0);
 5427   format %{ %}
 5428   interface(REG_INTER);
 5429 %}
 5430 
 5431 operand vRegD_V5()
 5432 %{
 5433   constraint(ALLOC_IN_RC(v5_reg));
 5434   match(RegD);
 5435   op_cost(0);
 5436   format %{ %}
 5437   interface(REG_INTER);
 5438 %}
 5439 
 5440 operand vRegD_V6()
 5441 %{
 5442   constraint(ALLOC_IN_RC(v6_reg));
 5443   match(RegD);
 5444   op_cost(0);
 5445   format %{ %}
 5446   interface(REG_INTER);
 5447 %}
 5448 
 5449 operand vRegD_V7()
 5450 %{
 5451   constraint(ALLOC_IN_RC(v7_reg));
 5452   match(RegD);
 5453   op_cost(0);
 5454   format %{ %}
 5455   interface(REG_INTER);
 5456 %}
 5457 
 5458 operand vRegD_V8()
 5459 %{
 5460   constraint(ALLOC_IN_RC(v8_reg));
 5461   match(RegD);
 5462   op_cost(0);
 5463   format %{ %}
 5464   interface(REG_INTER);
 5465 %}
 5466 
 5467 operand vRegD_V9()
 5468 %{
 5469   constraint(ALLOC_IN_RC(v9_reg));
 5470   match(RegD);
 5471   op_cost(0);
 5472   format %{ %}
 5473   interface(REG_INTER);
 5474 %}
 5475 
 5476 operand vRegD_V10()
 5477 %{
 5478   constraint(ALLOC_IN_RC(v10_reg));
 5479   match(RegD);
 5480   op_cost(0);
 5481   format %{ %}
 5482   interface(REG_INTER);
 5483 %}
 5484 
 5485 operand vRegD_V11()
 5486 %{
 5487   constraint(ALLOC_IN_RC(v11_reg));
 5488   match(RegD);
 5489   op_cost(0);
 5490   format %{ %}
 5491   interface(REG_INTER);
 5492 %}
 5493 
 5494 operand vRegD_V12()
 5495 %{
 5496   constraint(ALLOC_IN_RC(v12_reg));
 5497   match(RegD);
 5498   op_cost(0);
 5499   format %{ %}
 5500   interface(REG_INTER);
 5501 %}
 5502 
 5503 operand vRegD_V13()
 5504 %{
 5505   constraint(ALLOC_IN_RC(v13_reg));
 5506   match(RegD);
 5507   op_cost(0);
 5508   format %{ %}
 5509   interface(REG_INTER);
 5510 %}
 5511 
 5512 operand vRegD_V14()
 5513 %{
 5514   constraint(ALLOC_IN_RC(v14_reg));
 5515   match(RegD);
 5516   op_cost(0);
 5517   format %{ %}
 5518   interface(REG_INTER);
 5519 %}
 5520 
 5521 operand vRegD_V15()
 5522 %{
 5523   constraint(ALLOC_IN_RC(v15_reg));
 5524   match(RegD);
 5525   op_cost(0);
 5526   format %{ %}
 5527   interface(REG_INTER);
 5528 %}
 5529 
 5530 operand vRegD_V16()
 5531 %{
 5532   constraint(ALLOC_IN_RC(v16_reg));
 5533   match(RegD);
 5534   op_cost(0);
 5535   format %{ %}
 5536   interface(REG_INTER);
 5537 %}
 5538 
 5539 operand vRegD_V17()
 5540 %{
 5541   constraint(ALLOC_IN_RC(v17_reg));
 5542   match(RegD);
 5543   op_cost(0);
 5544   format %{ %}
 5545   interface(REG_INTER);
 5546 %}
 5547 
 5548 operand vRegD_V18()
 5549 %{
 5550   constraint(ALLOC_IN_RC(v18_reg));
 5551   match(RegD);
 5552   op_cost(0);
 5553   format %{ %}
 5554   interface(REG_INTER);
 5555 %}
 5556 
 5557 operand vRegD_V19()
 5558 %{
 5559   constraint(ALLOC_IN_RC(v19_reg));
 5560   match(RegD);
 5561   op_cost(0);
 5562   format %{ %}
 5563   interface(REG_INTER);
 5564 %}
 5565 
 5566 operand vRegD_V20()
 5567 %{
 5568   constraint(ALLOC_IN_RC(v20_reg));
 5569   match(RegD);
 5570   op_cost(0);
 5571   format %{ %}
 5572   interface(REG_INTER);
 5573 %}
 5574 
 5575 operand vRegD_V21()
 5576 %{
 5577   constraint(ALLOC_IN_RC(v21_reg));
 5578   match(RegD);
 5579   op_cost(0);
 5580   format %{ %}
 5581   interface(REG_INTER);
 5582 %}
 5583 
 5584 operand vRegD_V22()
 5585 %{
 5586   constraint(ALLOC_IN_RC(v22_reg));
 5587   match(RegD);
 5588   op_cost(0);
 5589   format %{ %}
 5590   interface(REG_INTER);
 5591 %}
 5592 
 5593 operand vRegD_V23()
 5594 %{
 5595   constraint(ALLOC_IN_RC(v23_reg));
 5596   match(RegD);
 5597   op_cost(0);
 5598   format %{ %}
 5599   interface(REG_INTER);
 5600 %}
 5601 
 5602 operand vRegD_V24()
 5603 %{
 5604   constraint(ALLOC_IN_RC(v24_reg));
 5605   match(RegD);
 5606   op_cost(0);
 5607   format %{ %}
 5608   interface(REG_INTER);
 5609 %}
 5610 
 5611 operand vRegD_V25()
 5612 %{
 5613   constraint(ALLOC_IN_RC(v25_reg));
 5614   match(RegD);
 5615   op_cost(0);
 5616   format %{ %}
 5617   interface(REG_INTER);
 5618 %}
 5619 
 5620 operand vRegD_V26()
 5621 %{
 5622   constraint(ALLOC_IN_RC(v26_reg));
 5623   match(RegD);
 5624   op_cost(0);
 5625   format %{ %}
 5626   interface(REG_INTER);
 5627 %}
 5628 
 5629 operand vRegD_V27()
 5630 %{
 5631   constraint(ALLOC_IN_RC(v27_reg));
 5632   match(RegD);
 5633   op_cost(0);
 5634   format %{ %}
 5635   interface(REG_INTER);
 5636 %}
 5637 
 5638 operand vRegD_V28()
 5639 %{
 5640   constraint(ALLOC_IN_RC(v28_reg));
 5641   match(RegD);
 5642   op_cost(0);
 5643   format %{ %}
 5644   interface(REG_INTER);
 5645 %}
 5646 
 5647 operand vRegD_V29()
 5648 %{
 5649   constraint(ALLOC_IN_RC(v29_reg));
 5650   match(RegD);
 5651   op_cost(0);
 5652   format %{ %}
 5653   interface(REG_INTER);
 5654 %}
 5655 
 5656 operand vRegD_V30()
 5657 %{
 5658   constraint(ALLOC_IN_RC(v30_reg));
 5659   match(RegD);
 5660   op_cost(0);
 5661   format %{ %}
 5662   interface(REG_INTER);
 5663 %}
 5664 
 5665 operand vRegD_V31()
 5666 %{
 5667   constraint(ALLOC_IN_RC(v31_reg));
 5668   match(RegD);
 5669   op_cost(0);
 5670   format %{ %}
 5671   interface(REG_INTER);
 5672 %}
 5673 
 5674 operand pReg()
 5675 %{
 5676   constraint(ALLOC_IN_RC(pr_reg));
 5677   match(RegVectMask);
 5678   match(pRegGov);
 5679   op_cost(0);
 5680   format %{ %}
 5681   interface(REG_INTER);
 5682 %}
 5683 
 5684 operand pRegGov()
 5685 %{
 5686   constraint(ALLOC_IN_RC(gov_pr));
 5687   match(RegVectMask);
 5688   match(pReg);
 5689   op_cost(0);
 5690   format %{ %}
 5691   interface(REG_INTER);
 5692 %}
 5693 
 5694 operand pRegGov_P0()
 5695 %{
 5696   constraint(ALLOC_IN_RC(p0_reg));
 5697   match(RegVectMask);
 5698   op_cost(0);
 5699   format %{ %}
 5700   interface(REG_INTER);
 5701 %}
 5702 
 5703 operand pRegGov_P1()
 5704 %{
 5705   constraint(ALLOC_IN_RC(p1_reg));
 5706   match(RegVectMask);
 5707   op_cost(0);
 5708   format %{ %}
 5709   interface(REG_INTER);
 5710 %}
 5711 
 5712 // Flags register, used as output of signed compare instructions
 5713 
 5714 // note that on AArch64 we also use this register as the output for
 5715 // for floating point compare instructions (CmpF CmpD). this ensures
 5716 // that ordered inequality tests use GT, GE, LT or LE none of which
 5717 // pass through cases where the result is unordered i.e. one or both
 5718 // inputs to the compare is a NaN. this means that the ideal code can
 5719 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5720 // (where the comparison should always fail). EQ and NE tests are
 5721 // always generated in ideal code so that unordered folds into the NE
 5722 // case, matching the behaviour of AArch64 NE.
 5723 //
 5724 // This differs from x86 where the outputs of FP compares use a
 5725 // special FP flags registers and where compares based on this
 5726 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5727 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5728 // to explicitly handle the unordered case in branches. x86 also has
 5729 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5730 
 5731 operand rFlagsReg()
 5732 %{
 5733   constraint(ALLOC_IN_RC(int_flags));
 5734   match(RegFlags);
 5735 
 5736   op_cost(0);
 5737   format %{ "RFLAGS" %}
 5738   interface(REG_INTER);
 5739 %}
 5740 
 5741 // Flags register, used as output of unsigned compare instructions
 5742 operand rFlagsRegU()
 5743 %{
 5744   constraint(ALLOC_IN_RC(int_flags));
 5745   match(RegFlags);
 5746 
 5747   op_cost(0);
 5748   format %{ "RFLAGSU" %}
 5749   interface(REG_INTER);
 5750 %}
 5751 
 5752 // Special Registers
 5753 
 5754 // Method Register
 5755 operand inline_cache_RegP(iRegP reg)
 5756 %{
 5757   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5758   match(reg);
 5759   match(iRegPNoSp);
 5760   op_cost(0);
 5761   format %{ %}
 5762   interface(REG_INTER);
 5763 %}
 5764 
 5765 // Thread Register
 5766 operand thread_RegP(iRegP reg)
 5767 %{
 5768   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5769   match(reg);
 5770   op_cost(0);
 5771   format %{ %}
 5772   interface(REG_INTER);
 5773 %}
 5774 
 5775 operand lr_RegP(iRegP reg)
 5776 %{
 5777   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5778   match(reg);
 5779   op_cost(0);
 5780   format %{ %}
 5781   interface(REG_INTER);
 5782 %}
 5783 
 5784 //----------Memory Operands----------------------------------------------------
 5785 
 5786 operand indirect(iRegP reg)
 5787 %{
 5788   constraint(ALLOC_IN_RC(ptr_reg));
 5789   match(reg);
 5790   op_cost(0);
 5791   format %{ "[$reg]" %}
 5792   interface(MEMORY_INTER) %{
 5793     base($reg);
 5794     index(0xffffffff);
 5795     scale(0x0);
 5796     disp(0x0);
 5797   %}
 5798 %}
 5799 
 5800 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5801 %{
 5802   constraint(ALLOC_IN_RC(ptr_reg));
 5803   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5804   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5805   op_cost(0);
 5806   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5807   interface(MEMORY_INTER) %{
 5808     base($reg);
 5809     index($ireg);
 5810     scale($scale);
 5811     disp(0x0);
 5812   %}
 5813 %}
 5814 
 5815 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5816 %{
 5817   constraint(ALLOC_IN_RC(ptr_reg));
 5818   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5819   match(AddP reg (LShiftL lreg scale));
 5820   op_cost(0);
 5821   format %{ "$reg, $lreg lsl($scale)" %}
 5822   interface(MEMORY_INTER) %{
 5823     base($reg);
 5824     index($lreg);
 5825     scale($scale);
 5826     disp(0x0);
 5827   %}
 5828 %}
 5829 
 5830 operand indIndexI2L(iRegP reg, iRegI ireg)
 5831 %{
 5832   constraint(ALLOC_IN_RC(ptr_reg));
 5833   match(AddP reg (ConvI2L ireg));
 5834   op_cost(0);
 5835   format %{ "$reg, $ireg, 0, I2L" %}
 5836   interface(MEMORY_INTER) %{
 5837     base($reg);
 5838     index($ireg);
 5839     scale(0x0);
 5840     disp(0x0);
 5841   %}
 5842 %}
 5843 
 5844 operand indIndex(iRegP reg, iRegL lreg)
 5845 %{
 5846   constraint(ALLOC_IN_RC(ptr_reg));
 5847   match(AddP reg lreg);
 5848   op_cost(0);
 5849   format %{ "$reg, $lreg" %}
 5850   interface(MEMORY_INTER) %{
 5851     base($reg);
 5852     index($lreg);
 5853     scale(0x0);
 5854     disp(0x0);
 5855   %}
 5856 %}
 5857 
 5858 operand indOffI(iRegP reg, immIOffset off)
 5859 %{
 5860   constraint(ALLOC_IN_RC(ptr_reg));
 5861   match(AddP reg off);
 5862   op_cost(0);
 5863   format %{ "[$reg, $off]" %}
 5864   interface(MEMORY_INTER) %{
 5865     base($reg);
 5866     index(0xffffffff);
 5867     scale(0x0);
 5868     disp($off);
 5869   %}
 5870 %}
 5871 
 5872 operand indOffI1(iRegP reg, immIOffset1 off)
 5873 %{
 5874   constraint(ALLOC_IN_RC(ptr_reg));
 5875   match(AddP reg off);
 5876   op_cost(0);
 5877   format %{ "[$reg, $off]" %}
 5878   interface(MEMORY_INTER) %{
 5879     base($reg);
 5880     index(0xffffffff);
 5881     scale(0x0);
 5882     disp($off);
 5883   %}
 5884 %}
 5885 
 5886 operand indOffI2(iRegP reg, immIOffset2 off)
 5887 %{
 5888   constraint(ALLOC_IN_RC(ptr_reg));
 5889   match(AddP reg off);
 5890   op_cost(0);
 5891   format %{ "[$reg, $off]" %}
 5892   interface(MEMORY_INTER) %{
 5893     base($reg);
 5894     index(0xffffffff);
 5895     scale(0x0);
 5896     disp($off);
 5897   %}
 5898 %}
 5899 
 5900 operand indOffI4(iRegP reg, immIOffset4 off)
 5901 %{
 5902   constraint(ALLOC_IN_RC(ptr_reg));
 5903   match(AddP reg off);
 5904   op_cost(0);
 5905   format %{ "[$reg, $off]" %}
 5906   interface(MEMORY_INTER) %{
 5907     base($reg);
 5908     index(0xffffffff);
 5909     scale(0x0);
 5910     disp($off);
 5911   %}
 5912 %}
 5913 
 5914 operand indOffI8(iRegP reg, immIOffset8 off)
 5915 %{
 5916   constraint(ALLOC_IN_RC(ptr_reg));
 5917   match(AddP reg off);
 5918   op_cost(0);
 5919   format %{ "[$reg, $off]" %}
 5920   interface(MEMORY_INTER) %{
 5921     base($reg);
 5922     index(0xffffffff);
 5923     scale(0x0);
 5924     disp($off);
 5925   %}
 5926 %}
 5927 
 5928 operand indOffI16(iRegP reg, immIOffset16 off)
 5929 %{
 5930   constraint(ALLOC_IN_RC(ptr_reg));
 5931   match(AddP reg off);
 5932   op_cost(0);
 5933   format %{ "[$reg, $off]" %}
 5934   interface(MEMORY_INTER) %{
 5935     base($reg);
 5936     index(0xffffffff);
 5937     scale(0x0);
 5938     disp($off);
 5939   %}
 5940 %}
 5941 
 5942 operand indOffL(iRegP reg, immLoffset off)
 5943 %{
 5944   constraint(ALLOC_IN_RC(ptr_reg));
 5945   match(AddP reg off);
 5946   op_cost(0);
 5947   format %{ "[$reg, $off]" %}
 5948   interface(MEMORY_INTER) %{
 5949     base($reg);
 5950     index(0xffffffff);
 5951     scale(0x0);
 5952     disp($off);
 5953   %}
 5954 %}
 5955 
 5956 operand indOffL1(iRegP reg, immLoffset1 off)
 5957 %{
 5958   constraint(ALLOC_IN_RC(ptr_reg));
 5959   match(AddP reg off);
 5960   op_cost(0);
 5961   format %{ "[$reg, $off]" %}
 5962   interface(MEMORY_INTER) %{
 5963     base($reg);
 5964     index(0xffffffff);
 5965     scale(0x0);
 5966     disp($off);
 5967   %}
 5968 %}
 5969 
 5970 operand indOffL2(iRegP reg, immLoffset2 off)
 5971 %{
 5972   constraint(ALLOC_IN_RC(ptr_reg));
 5973   match(AddP reg off);
 5974   op_cost(0);
 5975   format %{ "[$reg, $off]" %}
 5976   interface(MEMORY_INTER) %{
 5977     base($reg);
 5978     index(0xffffffff);
 5979     scale(0x0);
 5980     disp($off);
 5981   %}
 5982 %}
 5983 
 5984 operand indOffL4(iRegP reg, immLoffset4 off)
 5985 %{
 5986   constraint(ALLOC_IN_RC(ptr_reg));
 5987   match(AddP reg off);
 5988   op_cost(0);
 5989   format %{ "[$reg, $off]" %}
 5990   interface(MEMORY_INTER) %{
 5991     base($reg);
 5992     index(0xffffffff);
 5993     scale(0x0);
 5994     disp($off);
 5995   %}
 5996 %}
 5997 
 5998 operand indOffL8(iRegP reg, immLoffset8 off)
 5999 %{
 6000   constraint(ALLOC_IN_RC(ptr_reg));
 6001   match(AddP reg off);
 6002   op_cost(0);
 6003   format %{ "[$reg, $off]" %}
 6004   interface(MEMORY_INTER) %{
 6005     base($reg);
 6006     index(0xffffffff);
 6007     scale(0x0);
 6008     disp($off);
 6009   %}
 6010 %}
 6011 
 6012 operand indOffL16(iRegP reg, immLoffset16 off)
 6013 %{
 6014   constraint(ALLOC_IN_RC(ptr_reg));
 6015   match(AddP reg off);
 6016   op_cost(0);
 6017   format %{ "[$reg, $off]" %}
 6018   interface(MEMORY_INTER) %{
 6019     base($reg);
 6020     index(0xffffffff);
 6021     scale(0x0);
 6022     disp($off);
 6023   %}
 6024 %}
 6025 
 6026 operand indirectN(iRegN reg)
 6027 %{
 6028   predicate(CompressedOops::shift() == 0);
 6029   constraint(ALLOC_IN_RC(ptr_reg));
 6030   match(DecodeN reg);
 6031   op_cost(0);
 6032   format %{ "[$reg]\t# narrow" %}
 6033   interface(MEMORY_INTER) %{
 6034     base($reg);
 6035     index(0xffffffff);
 6036     scale(0x0);
 6037     disp(0x0);
 6038   %}
 6039 %}
 6040 
 6041 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 6042 %{
 6043   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 6044   constraint(ALLOC_IN_RC(ptr_reg));
 6045   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 6046   op_cost(0);
 6047   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 6048   interface(MEMORY_INTER) %{
 6049     base($reg);
 6050     index($ireg);
 6051     scale($scale);
 6052     disp(0x0);
 6053   %}
 6054 %}
 6055 
 6056 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 6057 %{
 6058   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 6059   constraint(ALLOC_IN_RC(ptr_reg));
 6060   match(AddP (DecodeN reg) (LShiftL lreg scale));
 6061   op_cost(0);
 6062   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 6063   interface(MEMORY_INTER) %{
 6064     base($reg);
 6065     index($lreg);
 6066     scale($scale);
 6067     disp(0x0);
 6068   %}
 6069 %}
 6070 
 6071 operand indIndexI2LN(iRegN reg, iRegI ireg)
 6072 %{
 6073   predicate(CompressedOops::shift() == 0);
 6074   constraint(ALLOC_IN_RC(ptr_reg));
 6075   match(AddP (DecodeN reg) (ConvI2L ireg));
 6076   op_cost(0);
 6077   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 6078   interface(MEMORY_INTER) %{
 6079     base($reg);
 6080     index($ireg);
 6081     scale(0x0);
 6082     disp(0x0);
 6083   %}
 6084 %}
 6085 
 6086 operand indIndexN(iRegN reg, iRegL lreg)
 6087 %{
 6088   predicate(CompressedOops::shift() == 0);
 6089   constraint(ALLOC_IN_RC(ptr_reg));
 6090   match(AddP (DecodeN reg) lreg);
 6091   op_cost(0);
 6092   format %{ "$reg, $lreg\t# narrow" %}
 6093   interface(MEMORY_INTER) %{
 6094     base($reg);
 6095     index($lreg);
 6096     scale(0x0);
 6097     disp(0x0);
 6098   %}
 6099 %}
 6100 
 6101 operand indOffIN(iRegN reg, immIOffset off)
 6102 %{
 6103   predicate(CompressedOops::shift() == 0);
 6104   constraint(ALLOC_IN_RC(ptr_reg));
 6105   match(AddP (DecodeN reg) off);
 6106   op_cost(0);
 6107   format %{ "[$reg, $off]\t# narrow" %}
 6108   interface(MEMORY_INTER) %{
 6109     base($reg);
 6110     index(0xffffffff);
 6111     scale(0x0);
 6112     disp($off);
 6113   %}
 6114 %}
 6115 
 6116 operand indOffLN(iRegN reg, immLoffset off)
 6117 %{
 6118   predicate(CompressedOops::shift() == 0);
 6119   constraint(ALLOC_IN_RC(ptr_reg));
 6120   match(AddP (DecodeN reg) off);
 6121   op_cost(0);
 6122   format %{ "[$reg, $off]\t# narrow" %}
 6123   interface(MEMORY_INTER) %{
 6124     base($reg);
 6125     index(0xffffffff);
 6126     scale(0x0);
 6127     disp($off);
 6128   %}
 6129 %}
 6130 
 6131 
 6132 
 6133 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 6134 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 6135 %{
 6136   constraint(ALLOC_IN_RC(ptr_reg));
 6137   match(AddP reg off);
 6138   op_cost(0);
 6139   format %{ "[$reg, $off]" %}
 6140   interface(MEMORY_INTER) %{
 6141     base($reg);
 6142     index(0xffffffff);
 6143     scale(0x0);
 6144     disp($off);
 6145   %}
 6146 %}
 6147 
 6148 //----------Special Memory Operands--------------------------------------------
 6149 // Stack Slot Operand - This operand is used for loading and storing temporary
 6150 //                      values on the stack where a match requires a value to
 6151 //                      flow through memory.
 6152 operand stackSlotP(sRegP reg)
 6153 %{
 6154   constraint(ALLOC_IN_RC(stack_slots));
 6155   op_cost(100);
 6156   // No match rule because this operand is only generated in matching
 6157   // match(RegP);
 6158   format %{ "[$reg]" %}
 6159   interface(MEMORY_INTER) %{
 6160     base(0x1e);  // RSP
 6161     index(0x0);  // No Index
 6162     scale(0x0);  // No Scale
 6163     disp($reg);  // Stack Offset
 6164   %}
 6165 %}
 6166 
 6167 operand stackSlotI(sRegI reg)
 6168 %{
 6169   constraint(ALLOC_IN_RC(stack_slots));
 6170   // No match rule because this operand is only generated in matching
 6171   // match(RegI);
 6172   format %{ "[$reg]" %}
 6173   interface(MEMORY_INTER) %{
 6174     base(0x1e);  // RSP
 6175     index(0x0);  // No Index
 6176     scale(0x0);  // No Scale
 6177     disp($reg);  // Stack Offset
 6178   %}
 6179 %}
 6180 
 6181 operand stackSlotF(sRegF reg)
 6182 %{
 6183   constraint(ALLOC_IN_RC(stack_slots));
 6184   // No match rule because this operand is only generated in matching
 6185   // match(RegF);
 6186   format %{ "[$reg]" %}
 6187   interface(MEMORY_INTER) %{
 6188     base(0x1e);  // RSP
 6189     index(0x0);  // No Index
 6190     scale(0x0);  // No Scale
 6191     disp($reg);  // Stack Offset
 6192   %}
 6193 %}
 6194 
 6195 operand stackSlotD(sRegD reg)
 6196 %{
 6197   constraint(ALLOC_IN_RC(stack_slots));
 6198   // No match rule because this operand is only generated in matching
 6199   // match(RegD);
 6200   format %{ "[$reg]" %}
 6201   interface(MEMORY_INTER) %{
 6202     base(0x1e);  // RSP
 6203     index(0x0);  // No Index
 6204     scale(0x0);  // No Scale
 6205     disp($reg);  // Stack Offset
 6206   %}
 6207 %}
 6208 
 6209 operand stackSlotL(sRegL reg)
 6210 %{
 6211   constraint(ALLOC_IN_RC(stack_slots));
 6212   // No match rule because this operand is only generated in matching
 6213   // match(RegL);
 6214   format %{ "[$reg]" %}
 6215   interface(MEMORY_INTER) %{
 6216     base(0x1e);  // RSP
 6217     index(0x0);  // No Index
 6218     scale(0x0);  // No Scale
 6219     disp($reg);  // Stack Offset
 6220   %}
 6221 %}
 6222 
 6223 // Operands for expressing Control Flow
 6224 // NOTE: Label is a predefined operand which should not be redefined in
 6225 //       the AD file. It is generically handled within the ADLC.
 6226 
 6227 //----------Conditional Branch Operands----------------------------------------
 6228 // Comparison Op  - This is the operation of the comparison, and is limited to
 6229 //                  the following set of codes:
 6230 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6231 //
 6232 // Other attributes of the comparison, such as unsignedness, are specified
 6233 // by the comparison instruction that sets a condition code flags register.
 6234 // That result is represented by a flags operand whose subtype is appropriate
 6235 // to the unsignedness (etc.) of the comparison.
 6236 //
 6237 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6238 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6239 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6240 
 6241 // used for signed integral comparisons and fp comparisons
 6242 
 6243 operand cmpOp()
 6244 %{
 6245   match(Bool);
 6246 
 6247   format %{ "" %}
 6248   interface(COND_INTER) %{
 6249     equal(0x0, "eq");
 6250     not_equal(0x1, "ne");
 6251     less(0xb, "lt");
 6252     greater_equal(0xa, "ge");
 6253     less_equal(0xd, "le");
 6254     greater(0xc, "gt");
 6255     overflow(0x6, "vs");
 6256     no_overflow(0x7, "vc");
 6257   %}
 6258 %}
 6259 
 6260 // used for unsigned integral comparisons
 6261 
 6262 operand cmpOpU()
 6263 %{
 6264   match(Bool);
 6265 
 6266   format %{ "" %}
 6267   interface(COND_INTER) %{
 6268     equal(0x0, "eq");
 6269     not_equal(0x1, "ne");
 6270     less(0x3, "lo");
 6271     greater_equal(0x2, "hs");
 6272     less_equal(0x9, "ls");
 6273     greater(0x8, "hi");
 6274     overflow(0x6, "vs");
 6275     no_overflow(0x7, "vc");
 6276   %}
 6277 %}
 6278 
 6279 // used for certain integral comparisons which can be
 6280 // converted to cbxx or tbxx instructions
 6281 
 6282 operand cmpOpEqNe()
 6283 %{
 6284   match(Bool);
 6285   op_cost(0);
 6286   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6287             || n->as_Bool()->_test._test == BoolTest::eq);
 6288 
 6289   format %{ "" %}
 6290   interface(COND_INTER) %{
 6291     equal(0x0, "eq");
 6292     not_equal(0x1, "ne");
 6293     less(0xb, "lt");
 6294     greater_equal(0xa, "ge");
 6295     less_equal(0xd, "le");
 6296     greater(0xc, "gt");
 6297     overflow(0x6, "vs");
 6298     no_overflow(0x7, "vc");
 6299   %}
 6300 %}
 6301 
 6302 // used for certain integral comparisons which can be
 6303 // converted to cbxx or tbxx instructions
 6304 
 6305 operand cmpOpLtGe()
 6306 %{
 6307   match(Bool);
 6308   op_cost(0);
 6309 
 6310   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6311             || n->as_Bool()->_test._test == BoolTest::ge);
 6312 
 6313   format %{ "" %}
 6314   interface(COND_INTER) %{
 6315     equal(0x0, "eq");
 6316     not_equal(0x1, "ne");
 6317     less(0xb, "lt");
 6318     greater_equal(0xa, "ge");
 6319     less_equal(0xd, "le");
 6320     greater(0xc, "gt");
 6321     overflow(0x6, "vs");
 6322     no_overflow(0x7, "vc");
 6323   %}
 6324 %}
 6325 
 6326 // used for certain unsigned integral comparisons which can be
 6327 // converted to cbxx or tbxx instructions
 6328 
 6329 operand cmpOpUEqNeLtGe()
 6330 %{
 6331   match(Bool);
 6332   op_cost(0);
 6333 
 6334   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6335             || n->as_Bool()->_test._test == BoolTest::ne
 6336             || n->as_Bool()->_test._test == BoolTest::lt
 6337             || n->as_Bool()->_test._test == BoolTest::ge);
 6338 
 6339   format %{ "" %}
 6340   interface(COND_INTER) %{
 6341     equal(0x0, "eq");
 6342     not_equal(0x1, "ne");
 6343     less(0xb, "lt");
 6344     greater_equal(0xa, "ge");
 6345     less_equal(0xd, "le");
 6346     greater(0xc, "gt");
 6347     overflow(0x6, "vs");
 6348     no_overflow(0x7, "vc");
 6349   %}
 6350 %}
 6351 
 6352 // Special operand allowing long args to int ops to be truncated for free
 6353 
 6354 operand iRegL2I(iRegL reg) %{
 6355 
 6356   op_cost(0);
 6357 
 6358   match(ConvL2I reg);
 6359 
 6360   format %{ "l2i($reg)" %}
 6361 
 6362   interface(REG_INTER)
 6363 %}
 6364 
 6365 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6366 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6367 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6368 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6369 
 6370 //----------OPERAND CLASSES----------------------------------------------------
 6371 // Operand Classes are groups of operands that are used as to simplify
 6372 // instruction definitions by not requiring the AD writer to specify
 6373 // separate instructions for every form of operand when the
 6374 // instruction accepts multiple operand types with the same basic
 6375 // encoding and format. The classic case of this is memory operands.
 6376 
 6377 // memory is used to define read/write location for load/store
 6378 // instruction defs. we can turn a memory op into an Address
 6379 
 6380 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6381                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6382 
 6383 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6384                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6385 
 6386 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6387                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6388 
 6389 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6390                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6391 
 6392 // All of the memory operands. For the pipeline description.
 6393 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6394                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6395                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6396 
 6397 
 6398 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6399 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6400 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6401 // can be elided because the 32-bit instruction will just employ the
 6402 // lower 32 bits anyway.
 6403 //
 6404 // n.b. this does not elide all L2I conversions. if the truncated
 6405 // value is consumed by more than one operation then the ConvL2I
 6406 // cannot be bundled into the consuming nodes so an l2i gets planted
 6407 // (actually a movw $dst $src) and the downstream instructions consume
 6408 // the result of the l2i as an iRegI input. That's a shame since the
 6409 // movw is actually redundant but its not too costly.
 6410 
 6411 opclass iRegIorL2I(iRegI, iRegL2I);
 6412 
 6413 //----------PIPELINE-----------------------------------------------------------
 6414 // Rules which define the behavior of the target architectures pipeline.
 6415 
 6416 // For specific pipelines, eg A53, define the stages of that pipeline
 6417 //pipe_desc(ISS, EX1, EX2, WR);
 6418 #define ISS S0
 6419 #define EX1 S1
 6420 #define EX2 S2
 6421 #define WR  S3
 6422 
 6423 // Integer ALU reg operation
 6424 pipeline %{
 6425 
 6426 attributes %{
 6427   // ARM instructions are of fixed length
 6428   fixed_size_instructions;        // Fixed size instructions TODO does
 6429   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6430   // ARM instructions come in 32-bit word units
 6431   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6432   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6433   instruction_fetch_units = 1;       // of 64 bytes
 6434 
 6435   // List of nop instructions
 6436   nops( MachNop );
 6437 %}
 6438 
 6439 // We don't use an actual pipeline model so don't care about resources
 6440 // or description. we do use pipeline classes to introduce fixed
 6441 // latencies
 6442 
 6443 //----------RESOURCES----------------------------------------------------------
 6444 // Resources are the functional units available to the machine
 6445 
 6446 resources( INS0, INS1, INS01 = INS0 | INS1,
 6447            ALU0, ALU1, ALU = ALU0 | ALU1,
 6448            MAC,
 6449            DIV,
 6450            BRANCH,
 6451            LDST,
 6452            NEON_FP);
 6453 
 6454 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6455 // Pipeline Description specifies the stages in the machine's pipeline
 6456 
 6457 // Define the pipeline as a generic 6 stage pipeline
 6458 pipe_desc(S0, S1, S2, S3, S4, S5);
 6459 
 6460 //----------PIPELINE CLASSES---------------------------------------------------
 6461 // Pipeline Classes describe the stages in which input and output are
 6462 // referenced by the hardware pipeline.
 6463 
 6464 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6465 %{
 6466   single_instruction;
 6467   src1   : S1(read);
 6468   src2   : S2(read);
 6469   dst    : S5(write);
 6470   INS01  : ISS;
 6471   NEON_FP : S5;
 6472 %}
 6473 
 6474 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6475 %{
 6476   single_instruction;
 6477   src1   : S1(read);
 6478   src2   : S2(read);
 6479   dst    : S5(write);
 6480   INS01  : ISS;
 6481   NEON_FP : S5;
 6482 %}
 6483 
 6484 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6485 %{
 6486   single_instruction;
 6487   src    : S1(read);
 6488   dst    : S5(write);
 6489   INS01  : ISS;
 6490   NEON_FP : S5;
 6491 %}
 6492 
 6493 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6494 %{
 6495   single_instruction;
 6496   src    : S1(read);
 6497   dst    : S5(write);
 6498   INS01  : ISS;
 6499   NEON_FP : S5;
 6500 %}
 6501 
 6502 pipe_class fp_d2f(vRegF dst, vRegD src)
 6503 %{
 6504   single_instruction;
 6505   src    : S1(read);
 6506   dst    : S5(write);
 6507   INS01  : ISS;
 6508   NEON_FP : S5;
 6509 %}
 6510 
 6511 pipe_class fp_f2d(vRegD dst, vRegF src)
 6512 %{
 6513   single_instruction;
 6514   src    : S1(read);
 6515   dst    : S5(write);
 6516   INS01  : ISS;
 6517   NEON_FP : S5;
 6518 %}
 6519 
 6520 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6521 %{
 6522   single_instruction;
 6523   src    : S1(read);
 6524   dst    : S5(write);
 6525   INS01  : ISS;
 6526   NEON_FP : S5;
 6527 %}
 6528 
 6529 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6530 %{
 6531   single_instruction;
 6532   src    : S1(read);
 6533   dst    : S5(write);
 6534   INS01  : ISS;
 6535   NEON_FP : S5;
 6536 %}
 6537 
 6538 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6539 %{
 6540   single_instruction;
 6541   src    : S1(read);
 6542   dst    : S5(write);
 6543   INS01  : ISS;
 6544   NEON_FP : S5;
 6545 %}
 6546 
 6547 pipe_class fp_l2f(vRegF dst, iRegL src)
 6548 %{
 6549   single_instruction;
 6550   src    : S1(read);
 6551   dst    : S5(write);
 6552   INS01  : ISS;
 6553   NEON_FP : S5;
 6554 %}
 6555 
 6556 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6557 %{
 6558   single_instruction;
 6559   src    : S1(read);
 6560   dst    : S5(write);
 6561   INS01  : ISS;
 6562   NEON_FP : S5;
 6563 %}
 6564 
 6565 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6566 %{
 6567   single_instruction;
 6568   src    : S1(read);
 6569   dst    : S5(write);
 6570   INS01  : ISS;
 6571   NEON_FP : S5;
 6572 %}
 6573 
 6574 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6575 %{
 6576   single_instruction;
 6577   src    : S1(read);
 6578   dst    : S5(write);
 6579   INS01  : ISS;
 6580   NEON_FP : S5;
 6581 %}
 6582 
 6583 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6584 %{
 6585   single_instruction;
 6586   src    : S1(read);
 6587   dst    : S5(write);
 6588   INS01  : ISS;
 6589   NEON_FP : S5;
 6590 %}
 6591 
 6592 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6593 %{
 6594   single_instruction;
 6595   src1   : S1(read);
 6596   src2   : S2(read);
 6597   dst    : S5(write);
 6598   INS0   : ISS;
 6599   NEON_FP : S5;
 6600 %}
 6601 
 6602 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6603 %{
 6604   single_instruction;
 6605   src1   : S1(read);
 6606   src2   : S2(read);
 6607   dst    : S5(write);
 6608   INS0   : ISS;
 6609   NEON_FP : S5;
 6610 %}
 6611 
 6612 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6613 %{
 6614   single_instruction;
 6615   cr     : S1(read);
 6616   src1   : S1(read);
 6617   src2   : S1(read);
 6618   dst    : S3(write);
 6619   INS01  : ISS;
 6620   NEON_FP : S3;
 6621 %}
 6622 
 6623 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6624 %{
 6625   single_instruction;
 6626   cr     : S1(read);
 6627   src1   : S1(read);
 6628   src2   : S1(read);
 6629   dst    : S3(write);
 6630   INS01  : ISS;
 6631   NEON_FP : S3;
 6632 %}
 6633 
 6634 pipe_class fp_imm_s(vRegF dst)
 6635 %{
 6636   single_instruction;
 6637   dst    : S3(write);
 6638   INS01  : ISS;
 6639   NEON_FP : S3;
 6640 %}
 6641 
 6642 pipe_class fp_imm_d(vRegD dst)
 6643 %{
 6644   single_instruction;
 6645   dst    : S3(write);
 6646   INS01  : ISS;
 6647   NEON_FP : S3;
 6648 %}
 6649 
 6650 pipe_class fp_load_constant_s(vRegF dst)
 6651 %{
 6652   single_instruction;
 6653   dst    : S4(write);
 6654   INS01  : ISS;
 6655   NEON_FP : S4;
 6656 %}
 6657 
 6658 pipe_class fp_load_constant_d(vRegD dst)
 6659 %{
 6660   single_instruction;
 6661   dst    : S4(write);
 6662   INS01  : ISS;
 6663   NEON_FP : S4;
 6664 %}
 6665 
 6666 //------- Integer ALU operations --------------------------
 6667 
 6668 // Integer ALU reg-reg operation
 6669 // Operands needed in EX1, result generated in EX2
 6670 // Eg.  ADD     x0, x1, x2
 6671 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6672 %{
 6673   single_instruction;
 6674   dst    : EX2(write);
 6675   src1   : EX1(read);
 6676   src2   : EX1(read);
 6677   INS01  : ISS; // Dual issue as instruction 0 or 1
 6678   ALU    : EX2;
 6679 %}
 6680 
 6681 // Integer ALU reg-reg operation with constant shift
 6682 // Shifted register must be available in LATE_ISS instead of EX1
 6683 // Eg.  ADD     x0, x1, x2, LSL #2
 6684 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6685 %{
 6686   single_instruction;
 6687   dst    : EX2(write);
 6688   src1   : EX1(read);
 6689   src2   : ISS(read);
 6690   INS01  : ISS;
 6691   ALU    : EX2;
 6692 %}
 6693 
 6694 // Integer ALU reg operation with constant shift
 6695 // Eg.  LSL     x0, x1, #shift
 6696 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6697 %{
 6698   single_instruction;
 6699   dst    : EX2(write);
 6700   src1   : ISS(read);
 6701   INS01  : ISS;
 6702   ALU    : EX2;
 6703 %}
 6704 
 6705 // Integer ALU reg-reg operation with variable shift
 6706 // Both operands must be available in LATE_ISS instead of EX1
 6707 // Result is available in EX1 instead of EX2
 6708 // Eg.  LSLV    x0, x1, x2
 6709 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6710 %{
 6711   single_instruction;
 6712   dst    : EX1(write);
 6713   src1   : ISS(read);
 6714   src2   : ISS(read);
 6715   INS01  : ISS;
 6716   ALU    : EX1;
 6717 %}
 6718 
 6719 // Integer ALU reg-reg operation with extract
 6720 // As for _vshift above, but result generated in EX2
 6721 // Eg.  EXTR    x0, x1, x2, #N
 6722 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6723 %{
 6724   single_instruction;
 6725   dst    : EX2(write);
 6726   src1   : ISS(read);
 6727   src2   : ISS(read);
 6728   INS1   : ISS; // Can only dual issue as Instruction 1
 6729   ALU    : EX1;
 6730 %}
 6731 
 6732 // Integer ALU reg operation
 6733 // Eg.  NEG     x0, x1
 6734 pipe_class ialu_reg(iRegI dst, iRegI src)
 6735 %{
 6736   single_instruction;
 6737   dst    : EX2(write);
 6738   src    : EX1(read);
 6739   INS01  : ISS;
 6740   ALU    : EX2;
 6741 %}
 6742 
 6743 // Integer ALU reg mmediate operation
 6744 // Eg.  ADD     x0, x1, #N
 6745 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6746 %{
 6747   single_instruction;
 6748   dst    : EX2(write);
 6749   src1   : EX1(read);
 6750   INS01  : ISS;
 6751   ALU    : EX2;
 6752 %}
 6753 
 6754 // Integer ALU immediate operation (no source operands)
 6755 // Eg.  MOV     x0, #N
 6756 pipe_class ialu_imm(iRegI dst)
 6757 %{
 6758   single_instruction;
 6759   dst    : EX1(write);
 6760   INS01  : ISS;
 6761   ALU    : EX1;
 6762 %}
 6763 
 6764 //------- Compare operation -------------------------------
 6765 
 6766 // Compare reg-reg
 6767 // Eg.  CMP     x0, x1
 6768 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6769 %{
 6770   single_instruction;
 6771 //  fixed_latency(16);
 6772   cr     : EX2(write);
 6773   op1    : EX1(read);
 6774   op2    : EX1(read);
 6775   INS01  : ISS;
 6776   ALU    : EX2;
 6777 %}
 6778 
 6779 // Compare reg-reg
 6780 // Eg.  CMP     x0, #N
 6781 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6782 %{
 6783   single_instruction;
 6784 //  fixed_latency(16);
 6785   cr     : EX2(write);
 6786   op1    : EX1(read);
 6787   INS01  : ISS;
 6788   ALU    : EX2;
 6789 %}
 6790 
 6791 //------- Conditional instructions ------------------------
 6792 
 6793 // Conditional no operands
 6794 // Eg.  CSINC   x0, zr, zr, <cond>
 6795 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6796 %{
 6797   single_instruction;
 6798   cr     : EX1(read);
 6799   dst    : EX2(write);
 6800   INS01  : ISS;
 6801   ALU    : EX2;
 6802 %}
 6803 
 6804 // Conditional 2 operand
 6805 // EG.  CSEL    X0, X1, X2, <cond>
 6806 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6807 %{
 6808   single_instruction;
 6809   cr     : EX1(read);
 6810   src1   : EX1(read);
 6811   src2   : EX1(read);
 6812   dst    : EX2(write);
 6813   INS01  : ISS;
 6814   ALU    : EX2;
 6815 %}
 6816 
 6817 // Conditional 2 operand
 6818 // EG.  CSEL    X0, X1, X2, <cond>
 6819 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6820 %{
 6821   single_instruction;
 6822   cr     : EX1(read);
 6823   src    : EX1(read);
 6824   dst    : EX2(write);
 6825   INS01  : ISS;
 6826   ALU    : EX2;
 6827 %}
 6828 
 6829 //------- Multiply pipeline operations --------------------
 6830 
 6831 // Multiply reg-reg
 6832 // Eg.  MUL     w0, w1, w2
 6833 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6834 %{
 6835   single_instruction;
 6836   dst    : WR(write);
 6837   src1   : ISS(read);
 6838   src2   : ISS(read);
 6839   INS01  : ISS;
 6840   MAC    : WR;
 6841 %}
 6842 
 6843 // Multiply accumulate
 6844 // Eg.  MADD    w0, w1, w2, w3
 6845 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6846 %{
 6847   single_instruction;
 6848   dst    : WR(write);
 6849   src1   : ISS(read);
 6850   src2   : ISS(read);
 6851   src3   : ISS(read);
 6852   INS01  : ISS;
 6853   MAC    : WR;
 6854 %}
 6855 
 6856 // Eg.  MUL     w0, w1, w2
 6857 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6858 %{
 6859   single_instruction;
 6860   fixed_latency(3); // Maximum latency for 64 bit mul
 6861   dst    : WR(write);
 6862   src1   : ISS(read);
 6863   src2   : ISS(read);
 6864   INS01  : ISS;
 6865   MAC    : WR;
 6866 %}
 6867 
 6868 // Multiply accumulate
 6869 // Eg.  MADD    w0, w1, w2, w3
 6870 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6871 %{
 6872   single_instruction;
 6873   fixed_latency(3); // Maximum latency for 64 bit mul
 6874   dst    : WR(write);
 6875   src1   : ISS(read);
 6876   src2   : ISS(read);
 6877   src3   : ISS(read);
 6878   INS01  : ISS;
 6879   MAC    : WR;
 6880 %}
 6881 
 6882 //------- Divide pipeline operations --------------------
 6883 
 6884 // Eg.  SDIV    w0, w1, w2
 6885 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6886 %{
 6887   single_instruction;
 6888   fixed_latency(8); // Maximum latency for 32 bit divide
 6889   dst    : WR(write);
 6890   src1   : ISS(read);
 6891   src2   : ISS(read);
 6892   INS0   : ISS; // Can only dual issue as instruction 0
 6893   DIV    : WR;
 6894 %}
 6895 
 6896 // Eg.  SDIV    x0, x1, x2
 6897 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6898 %{
 6899   single_instruction;
 6900   fixed_latency(16); // Maximum latency for 64 bit divide
 6901   dst    : WR(write);
 6902   src1   : ISS(read);
 6903   src2   : ISS(read);
 6904   INS0   : ISS; // Can only dual issue as instruction 0
 6905   DIV    : WR;
 6906 %}
 6907 
 6908 //------- Load pipeline operations ------------------------
 6909 
 6910 // Load - prefetch
 6911 // Eg.  PFRM    <mem>
 6912 pipe_class iload_prefetch(memory mem)
 6913 %{
 6914   single_instruction;
 6915   mem    : ISS(read);
 6916   INS01  : ISS;
 6917   LDST   : WR;
 6918 %}
 6919 
 6920 // Load - reg, mem
 6921 // Eg.  LDR     x0, <mem>
 6922 pipe_class iload_reg_mem(iRegI dst, memory mem)
 6923 %{
 6924   single_instruction;
 6925   dst    : WR(write);
 6926   mem    : ISS(read);
 6927   INS01  : ISS;
 6928   LDST   : WR;
 6929 %}
 6930 
 6931 // Load - reg, reg
 6932 // Eg.  LDR     x0, [sp, x1]
 6933 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 6934 %{
 6935   single_instruction;
 6936   dst    : WR(write);
 6937   src    : ISS(read);
 6938   INS01  : ISS;
 6939   LDST   : WR;
 6940 %}
 6941 
 6942 //------- Store pipeline operations -----------------------
 6943 
 6944 // Store - zr, mem
 6945 // Eg.  STR     zr, <mem>
 6946 pipe_class istore_mem(memory mem)
 6947 %{
 6948   single_instruction;
 6949   mem    : ISS(read);
 6950   INS01  : ISS;
 6951   LDST   : WR;
 6952 %}
 6953 
 6954 // Store - reg, mem
 6955 // Eg.  STR     x0, <mem>
 6956 pipe_class istore_reg_mem(iRegI src, memory mem)
 6957 %{
 6958   single_instruction;
 6959   mem    : ISS(read);
 6960   src    : EX2(read);
 6961   INS01  : ISS;
 6962   LDST   : WR;
 6963 %}
 6964 
 6965 // Store - reg, reg
 6966 // Eg. STR      x0, [sp, x1]
 6967 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 6968 %{
 6969   single_instruction;
 6970   dst    : ISS(read);
 6971   src    : EX2(read);
 6972   INS01  : ISS;
 6973   LDST   : WR;
 6974 %}
 6975 
 6976 //------- Store pipeline operations -----------------------
 6977 
 6978 // Branch
 6979 pipe_class pipe_branch()
 6980 %{
 6981   single_instruction;
 6982   INS01  : ISS;
 6983   BRANCH : EX1;
 6984 %}
 6985 
 6986 // Conditional branch
 6987 pipe_class pipe_branch_cond(rFlagsReg cr)
 6988 %{
 6989   single_instruction;
 6990   cr     : EX1(read);
 6991   INS01  : ISS;
 6992   BRANCH : EX1;
 6993 %}
 6994 
 6995 // Compare & Branch
 6996 // EG.  CBZ/CBNZ
 6997 pipe_class pipe_cmp_branch(iRegI op1)
 6998 %{
 6999   single_instruction;
 7000   op1    : EX1(read);
 7001   INS01  : ISS;
 7002   BRANCH : EX1;
 7003 %}
 7004 
 7005 //------- Synchronisation operations ----------------------
 7006 
 7007 // Any operation requiring serialization.
 7008 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 7009 pipe_class pipe_serial()
 7010 %{
 7011   single_instruction;
 7012   force_serialization;
 7013   fixed_latency(16);
 7014   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7015   LDST   : WR;
 7016 %}
 7017 
 7018 // Generic big/slow expanded idiom - also serialized
 7019 pipe_class pipe_slow()
 7020 %{
 7021   instruction_count(10);
 7022   multiple_bundles;
 7023   force_serialization;
 7024   fixed_latency(16);
 7025   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7026   LDST   : WR;
 7027 %}
 7028 
 7029 // Empty pipeline class
 7030 pipe_class pipe_class_empty()
 7031 %{
 7032   single_instruction;
 7033   fixed_latency(0);
 7034 %}
 7035 
 7036 // Default pipeline class.
 7037 pipe_class pipe_class_default()
 7038 %{
 7039   single_instruction;
 7040   fixed_latency(2);
 7041 %}
 7042 
 7043 // Pipeline class for compares.
 7044 pipe_class pipe_class_compare()
 7045 %{
 7046   single_instruction;
 7047   fixed_latency(16);
 7048 %}
 7049 
 7050 // Pipeline class for memory operations.
 7051 pipe_class pipe_class_memory()
 7052 %{
 7053   single_instruction;
 7054   fixed_latency(16);
 7055 %}
 7056 
 7057 // Pipeline class for call.
 7058 pipe_class pipe_class_call()
 7059 %{
 7060   single_instruction;
 7061   fixed_latency(100);
 7062 %}
 7063 
 7064 // Define the class for the Nop node.
 7065 define %{
 7066    MachNop = pipe_class_empty;
 7067 %}
 7068 
 7069 %}
 7070 //----------INSTRUCTIONS-------------------------------------------------------
 7071 //
 7072 // match      -- States which machine-independent subtree may be replaced
 7073 //               by this instruction.
 7074 // ins_cost   -- The estimated cost of this instruction is used by instruction
 7075 //               selection to identify a minimum cost tree of machine
 7076 //               instructions that matches a tree of machine-independent
 7077 //               instructions.
 7078 // format     -- A string providing the disassembly for this instruction.
 7079 //               The value of an instruction's operand may be inserted
 7080 //               by referring to it with a '$' prefix.
 7081 // opcode     -- Three instruction opcodes may be provided.  These are referred
 7082 //               to within an encode class as $primary, $secondary, and $tertiary
 7083 //               rrspectively.  The primary opcode is commonly used to
 7084 //               indicate the type of machine instruction, while secondary
 7085 //               and tertiary are often used for prefix options or addressing
 7086 //               modes.
 7087 // ins_encode -- A list of encode classes with parameters. The encode class
 7088 //               name must have been defined in an 'enc_class' specification
 7089 //               in the encode section of the architecture description.
 7090 
 7091 // ============================================================================
 7092 // Memory (Load/Store) Instructions
 7093 
 7094 // Load Instructions
 7095 
 7096 // Load Byte (8 bit signed)
 7097 instruct loadB(iRegINoSp dst, memory1 mem)
 7098 %{
 7099   match(Set dst (LoadB mem));
 7100   predicate(!needs_acquiring_load(n));
 7101 
 7102   ins_cost(4 * INSN_COST);
 7103   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 7104 
 7105   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 7106 
 7107   ins_pipe(iload_reg_mem);
 7108 %}
 7109 
 7110 // Load Byte (8 bit signed) into long
 7111 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 7112 %{
 7113   match(Set dst (ConvI2L (LoadB mem)));
 7114   predicate(!needs_acquiring_load(n->in(1)));
 7115 
 7116   ins_cost(4 * INSN_COST);
 7117   format %{ "ldrsb  $dst, $mem\t# byte" %}
 7118 
 7119   ins_encode(aarch64_enc_ldrsb(dst, mem));
 7120 
 7121   ins_pipe(iload_reg_mem);
 7122 %}
 7123 
 7124 // Load Byte (8 bit unsigned)
 7125 instruct loadUB(iRegINoSp dst, memory1 mem)
 7126 %{
 7127   match(Set dst (LoadUB mem));
 7128   predicate(!needs_acquiring_load(n));
 7129 
 7130   ins_cost(4 * INSN_COST);
 7131   format %{ "ldrbw  $dst, $mem\t# byte" %}
 7132 
 7133   ins_encode(aarch64_enc_ldrb(dst, mem));
 7134 
 7135   ins_pipe(iload_reg_mem);
 7136 %}
 7137 
 7138 // Load Byte (8 bit unsigned) into long
 7139 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7140 %{
 7141   match(Set dst (ConvI2L (LoadUB mem)));
 7142   predicate(!needs_acquiring_load(n->in(1)));
 7143 
 7144   ins_cost(4 * INSN_COST);
 7145   format %{ "ldrb  $dst, $mem\t# byte" %}
 7146 
 7147   ins_encode(aarch64_enc_ldrb(dst, mem));
 7148 
 7149   ins_pipe(iload_reg_mem);
 7150 %}
 7151 
 7152 // Load Short (16 bit signed)
 7153 instruct loadS(iRegINoSp dst, memory2 mem)
 7154 %{
 7155   match(Set dst (LoadS mem));
 7156   predicate(!needs_acquiring_load(n));
 7157 
 7158   ins_cost(4 * INSN_COST);
 7159   format %{ "ldrshw  $dst, $mem\t# short" %}
 7160 
 7161   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7162 
 7163   ins_pipe(iload_reg_mem);
 7164 %}
 7165 
 7166 // Load Short (16 bit signed) into long
 7167 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7168 %{
 7169   match(Set dst (ConvI2L (LoadS mem)));
 7170   predicate(!needs_acquiring_load(n->in(1)));
 7171 
 7172   ins_cost(4 * INSN_COST);
 7173   format %{ "ldrsh  $dst, $mem\t# short" %}
 7174 
 7175   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7176 
 7177   ins_pipe(iload_reg_mem);
 7178 %}
 7179 
 7180 // Load Char (16 bit unsigned)
 7181 instruct loadUS(iRegINoSp dst, memory2 mem)
 7182 %{
 7183   match(Set dst (LoadUS mem));
 7184   predicate(!needs_acquiring_load(n));
 7185 
 7186   ins_cost(4 * INSN_COST);
 7187   format %{ "ldrh  $dst, $mem\t# short" %}
 7188 
 7189   ins_encode(aarch64_enc_ldrh(dst, mem));
 7190 
 7191   ins_pipe(iload_reg_mem);
 7192 %}
 7193 
 7194 // Load Short/Char (16 bit unsigned) into long
 7195 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7196 %{
 7197   match(Set dst (ConvI2L (LoadUS mem)));
 7198   predicate(!needs_acquiring_load(n->in(1)));
 7199 
 7200   ins_cost(4 * INSN_COST);
 7201   format %{ "ldrh  $dst, $mem\t# short" %}
 7202 
 7203   ins_encode(aarch64_enc_ldrh(dst, mem));
 7204 
 7205   ins_pipe(iload_reg_mem);
 7206 %}
 7207 
 7208 // Load Integer (32 bit signed)
 7209 instruct loadI(iRegINoSp dst, memory4 mem)
 7210 %{
 7211   match(Set dst (LoadI mem));
 7212   predicate(!needs_acquiring_load(n));
 7213 
 7214   ins_cost(4 * INSN_COST);
 7215   format %{ "ldrw  $dst, $mem\t# int" %}
 7216 
 7217   ins_encode(aarch64_enc_ldrw(dst, mem));
 7218 
 7219   ins_pipe(iload_reg_mem);
 7220 %}
 7221 
 7222 // Load Integer (32 bit signed) into long
 7223 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7224 %{
 7225   match(Set dst (ConvI2L (LoadI mem)));
 7226   predicate(!needs_acquiring_load(n->in(1)));
 7227 
 7228   ins_cost(4 * INSN_COST);
 7229   format %{ "ldrsw  $dst, $mem\t# int" %}
 7230 
 7231   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7232 
 7233   ins_pipe(iload_reg_mem);
 7234 %}
 7235 
 7236 // Load Integer (32 bit unsigned) into long
 7237 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7238 %{
 7239   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7240   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7241 
 7242   ins_cost(4 * INSN_COST);
 7243   format %{ "ldrw  $dst, $mem\t# int" %}
 7244 
 7245   ins_encode(aarch64_enc_ldrw(dst, mem));
 7246 
 7247   ins_pipe(iload_reg_mem);
 7248 %}
 7249 
 7250 // Load Long (64 bit signed)
 7251 instruct loadL(iRegLNoSp dst, memory8 mem)
 7252 %{
 7253   match(Set dst (LoadL mem));
 7254   predicate(!needs_acquiring_load(n));
 7255 
 7256   ins_cost(4 * INSN_COST);
 7257   format %{ "ldr  $dst, $mem\t# int" %}
 7258 
 7259   ins_encode(aarch64_enc_ldr(dst, mem));
 7260 
 7261   ins_pipe(iload_reg_mem);
 7262 %}
 7263 
 7264 // Load Range
 7265 instruct loadRange(iRegINoSp dst, memory4 mem)
 7266 %{
 7267   match(Set dst (LoadRange mem));
 7268 
 7269   ins_cost(4 * INSN_COST);
 7270   format %{ "ldrw  $dst, $mem\t# range" %}
 7271 
 7272   ins_encode(aarch64_enc_ldrw(dst, mem));
 7273 
 7274   ins_pipe(iload_reg_mem);
 7275 %}
 7276 
 7277 // Load Pointer
 7278 instruct loadP(iRegPNoSp dst, memory8 mem)
 7279 %{
 7280   match(Set dst (LoadP mem));
 7281   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7282 
 7283   ins_cost(4 * INSN_COST);
 7284   format %{ "ldr  $dst, $mem\t# ptr" %}
 7285 
 7286   ins_encode(aarch64_enc_ldr(dst, mem));
 7287 
 7288   ins_pipe(iload_reg_mem);
 7289 %}
 7290 
 7291 // Load Compressed Pointer
 7292 instruct loadN(iRegNNoSp dst, memory4 mem)
 7293 %{
 7294   match(Set dst (LoadN mem));
 7295   predicate(!needs_acquiring_load(n));
 7296 
 7297   ins_cost(4 * INSN_COST);
 7298   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7299 
 7300   ins_encode(aarch64_enc_ldrw(dst, mem));
 7301 
 7302   ins_pipe(iload_reg_mem);
 7303 %}
 7304 
 7305 // Load Klass Pointer
 7306 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7307 %{
 7308   match(Set dst (LoadKlass mem));
 7309   predicate(!needs_acquiring_load(n));
 7310 
 7311   ins_cost(4 * INSN_COST);
 7312   format %{ "ldr  $dst, $mem\t# class" %}
 7313 
 7314   ins_encode(aarch64_enc_ldr(dst, mem));
 7315 
 7316   ins_pipe(iload_reg_mem);
 7317 %}
 7318 
 7319 // Load Narrow Klass Pointer
 7320 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7321 %{
 7322   match(Set dst (LoadNKlass mem));
 7323   predicate(!needs_acquiring_load(n));
 7324 
 7325   ins_cost(4 * INSN_COST);
 7326   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7327 
 7328   ins_encode(aarch64_enc_ldrw(dst, mem));
 7329 
 7330   ins_pipe(iload_reg_mem);
 7331 %}
 7332 
 7333 // Load Float
 7334 instruct loadF(vRegF dst, memory4 mem)
 7335 %{
 7336   match(Set dst (LoadF mem));
 7337   predicate(!needs_acquiring_load(n));
 7338 
 7339   ins_cost(4 * INSN_COST);
 7340   format %{ "ldrs  $dst, $mem\t# float" %}
 7341 
 7342   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7343 
 7344   ins_pipe(pipe_class_memory);
 7345 %}
 7346 
 7347 // Load Double
 7348 instruct loadD(vRegD dst, memory8 mem)
 7349 %{
 7350   match(Set dst (LoadD mem));
 7351   predicate(!needs_acquiring_load(n));
 7352 
 7353   ins_cost(4 * INSN_COST);
 7354   format %{ "ldrd  $dst, $mem\t# double" %}
 7355 
 7356   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7357 
 7358   ins_pipe(pipe_class_memory);
 7359 %}
 7360 
 7361 
 7362 // Load Int Constant
 7363 instruct loadConI(iRegINoSp dst, immI src)
 7364 %{
 7365   match(Set dst src);
 7366 
 7367   ins_cost(INSN_COST);
 7368   format %{ "mov $dst, $src\t# int" %}
 7369 
 7370   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7371 
 7372   ins_pipe(ialu_imm);
 7373 %}
 7374 
 7375 // Load Long Constant
 7376 instruct loadConL(iRegLNoSp dst, immL src)
 7377 %{
 7378   match(Set dst src);
 7379 
 7380   ins_cost(INSN_COST);
 7381   format %{ "mov $dst, $src\t# long" %}
 7382 
 7383   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7384 
 7385   ins_pipe(ialu_imm);
 7386 %}
 7387 
 7388 // Load Pointer Constant
 7389 
 7390 instruct loadConP(iRegPNoSp dst, immP con)
 7391 %{
 7392   match(Set dst con);
 7393 
 7394   ins_cost(INSN_COST * 4);
 7395   format %{
 7396     "mov  $dst, $con\t# ptr\n\t"
 7397   %}
 7398 
 7399   ins_encode(aarch64_enc_mov_p(dst, con));
 7400 
 7401   ins_pipe(ialu_imm);
 7402 %}
 7403 
 7404 // Load Null Pointer Constant
 7405 
 7406 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7407 %{
 7408   match(Set dst con);
 7409 
 7410   ins_cost(INSN_COST);
 7411   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7412 
 7413   ins_encode(aarch64_enc_mov_p0(dst, con));
 7414 
 7415   ins_pipe(ialu_imm);
 7416 %}
 7417 
 7418 // Load Pointer Constant One
 7419 
 7420 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7421 %{
 7422   match(Set dst con);
 7423 
 7424   ins_cost(INSN_COST);
 7425   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7426 
 7427   ins_encode(aarch64_enc_mov_p1(dst, con));
 7428 
 7429   ins_pipe(ialu_imm);
 7430 %}
 7431 
 7432 // Load Byte Map Base Constant
 7433 
 7434 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7435 %{
 7436   match(Set dst con);
 7437 
 7438   ins_cost(INSN_COST);
 7439   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7440 
 7441   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7442 
 7443   ins_pipe(ialu_imm);
 7444 %}
 7445 
 7446 // Load Narrow Pointer Constant
 7447 
 7448 instruct loadConN(iRegNNoSp dst, immN con)
 7449 %{
 7450   match(Set dst con);
 7451 
 7452   ins_cost(INSN_COST * 4);
 7453   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7454 
 7455   ins_encode(aarch64_enc_mov_n(dst, con));
 7456 
 7457   ins_pipe(ialu_imm);
 7458 %}
 7459 
 7460 // Load Narrow Null Pointer Constant
 7461 
 7462 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7463 %{
 7464   match(Set dst con);
 7465 
 7466   ins_cost(INSN_COST);
 7467   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7468 
 7469   ins_encode(aarch64_enc_mov_n0(dst, con));
 7470 
 7471   ins_pipe(ialu_imm);
 7472 %}
 7473 
 7474 // Load Narrow Klass Constant
 7475 
 7476 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7477 %{
 7478   match(Set dst con);
 7479 
 7480   ins_cost(INSN_COST);
 7481   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7482 
 7483   ins_encode(aarch64_enc_mov_nk(dst, con));
 7484 
 7485   ins_pipe(ialu_imm);
 7486 %}
 7487 
 7488 // Load Packed Float Constant
 7489 
 7490 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7491   match(Set dst con);
 7492   ins_cost(INSN_COST * 4);
 7493   format %{ "fmovs  $dst, $con"%}
 7494   ins_encode %{
 7495     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7496   %}
 7497 
 7498   ins_pipe(fp_imm_s);
 7499 %}
 7500 
 7501 // Load Float Constant
 7502 
 7503 instruct loadConF(vRegF dst, immF con) %{
 7504   match(Set dst con);
 7505 
 7506   ins_cost(INSN_COST * 4);
 7507 
 7508   format %{
 7509     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7510   %}
 7511 
 7512   ins_encode %{
 7513     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7514   %}
 7515 
 7516   ins_pipe(fp_load_constant_s);
 7517 %}
 7518 
 7519 // Load Packed Double Constant
 7520 
 7521 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7522   match(Set dst con);
 7523   ins_cost(INSN_COST);
 7524   format %{ "fmovd  $dst, $con"%}
 7525   ins_encode %{
 7526     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7527   %}
 7528 
 7529   ins_pipe(fp_imm_d);
 7530 %}
 7531 
 7532 // Load Double Constant
 7533 
 7534 instruct loadConD(vRegD dst, immD con) %{
 7535   match(Set dst con);
 7536 
 7537   ins_cost(INSN_COST * 5);
 7538   format %{
 7539     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7540   %}
 7541 
 7542   ins_encode %{
 7543     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7544   %}
 7545 
 7546   ins_pipe(fp_load_constant_d);
 7547 %}
 7548 
 7549 // Store Instructions
 7550 
 7551 // Store CMS card-mark Immediate
 7552 instruct storeimmCM0(immI0 zero, memory1 mem)
 7553 %{
 7554   match(Set mem (StoreCM mem zero));
 7555 
 7556   ins_cost(INSN_COST);
 7557   format %{ "storestore (elided)\n\t"
 7558             "strb zr, $mem\t# byte" %}
 7559 
 7560   ins_encode(aarch64_enc_strb0(mem));
 7561 
 7562   ins_pipe(istore_mem);
 7563 %}
 7564 
 7565 // Store CMS card-mark Immediate with intervening StoreStore
 7566 // needed when using CMS with no conditional card marking
 7567 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7568 %{
 7569   match(Set mem (StoreCM mem zero));
 7570 
 7571   ins_cost(INSN_COST * 2);
 7572   format %{ "storestore\n\t"
 7573             "dmb ishst"
 7574             "\n\tstrb zr, $mem\t# byte" %}
 7575 
 7576   ins_encode(aarch64_enc_strb0_ordered(mem));
 7577 
 7578   ins_pipe(istore_mem);
 7579 %}
 7580 
 7581 // Store Byte
 7582 instruct storeB(iRegIorL2I src, memory1 mem)
 7583 %{
 7584   match(Set mem (StoreB mem src));
 7585   predicate(!needs_releasing_store(n));
 7586 
 7587   ins_cost(INSN_COST);
 7588   format %{ "strb  $src, $mem\t# byte" %}
 7589 
 7590   ins_encode(aarch64_enc_strb(src, mem));
 7591 
 7592   ins_pipe(istore_reg_mem);
 7593 %}
 7594 
 7595 
 7596 instruct storeimmB0(immI0 zero, memory1 mem)
 7597 %{
 7598   match(Set mem (StoreB mem zero));
 7599   predicate(!needs_releasing_store(n));
 7600 
 7601   ins_cost(INSN_COST);
 7602   format %{ "strb rscractch2, $mem\t# byte" %}
 7603 
 7604   ins_encode(aarch64_enc_strb0(mem));
 7605 
 7606   ins_pipe(istore_mem);
 7607 %}
 7608 
 7609 // Store Char/Short
 7610 instruct storeC(iRegIorL2I src, memory2 mem)
 7611 %{
 7612   match(Set mem (StoreC mem src));
 7613   predicate(!needs_releasing_store(n));
 7614 
 7615   ins_cost(INSN_COST);
 7616   format %{ "strh  $src, $mem\t# short" %}
 7617 
 7618   ins_encode(aarch64_enc_strh(src, mem));
 7619 
 7620   ins_pipe(istore_reg_mem);
 7621 %}
 7622 
 7623 instruct storeimmC0(immI0 zero, memory2 mem)
 7624 %{
 7625   match(Set mem (StoreC mem zero));
 7626   predicate(!needs_releasing_store(n));
 7627 
 7628   ins_cost(INSN_COST);
 7629   format %{ "strh  zr, $mem\t# short" %}
 7630 
 7631   ins_encode(aarch64_enc_strh0(mem));
 7632 
 7633   ins_pipe(istore_mem);
 7634 %}
 7635 
 7636 // Store Integer
 7637 
 7638 instruct storeI(iRegIorL2I src, memory4 mem)
 7639 %{
 7640   match(Set mem(StoreI mem src));
 7641   predicate(!needs_releasing_store(n));
 7642 
 7643   ins_cost(INSN_COST);
 7644   format %{ "strw  $src, $mem\t# int" %}
 7645 
 7646   ins_encode(aarch64_enc_strw(src, mem));
 7647 
 7648   ins_pipe(istore_reg_mem);
 7649 %}
 7650 
 7651 instruct storeimmI0(immI0 zero, memory4 mem)
 7652 %{
 7653   match(Set mem(StoreI mem zero));
 7654   predicate(!needs_releasing_store(n));
 7655 
 7656   ins_cost(INSN_COST);
 7657   format %{ "strw  zr, $mem\t# int" %}
 7658 
 7659   ins_encode(aarch64_enc_strw0(mem));
 7660 
 7661   ins_pipe(istore_mem);
 7662 %}
 7663 
 7664 // Store Long (64 bit signed)
 7665 instruct storeL(iRegL src, memory8 mem)
 7666 %{
 7667   match(Set mem (StoreL mem src));
 7668   predicate(!needs_releasing_store(n));
 7669 
 7670   ins_cost(INSN_COST);
 7671   format %{ "str  $src, $mem\t# int" %}
 7672 
 7673   ins_encode(aarch64_enc_str(src, mem));
 7674 
 7675   ins_pipe(istore_reg_mem);
 7676 %}
 7677 
 7678 // Store Long (64 bit signed)
 7679 instruct storeimmL0(immL0 zero, memory8 mem)
 7680 %{
 7681   match(Set mem (StoreL mem zero));
 7682   predicate(!needs_releasing_store(n));
 7683 
 7684   ins_cost(INSN_COST);
 7685   format %{ "str  zr, $mem\t# int" %}
 7686 
 7687   ins_encode(aarch64_enc_str0(mem));
 7688 
 7689   ins_pipe(istore_mem);
 7690 %}
 7691 
 7692 // Store Pointer
 7693 instruct storeP(iRegP src, memory8 mem)
 7694 %{
 7695   match(Set mem (StoreP mem src));
 7696   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7697 
 7698   ins_cost(INSN_COST);
 7699   format %{ "str  $src, $mem\t# ptr" %}
 7700 
 7701   ins_encode(aarch64_enc_str(src, mem));
 7702 
 7703   ins_pipe(istore_reg_mem);
 7704 %}
 7705 
 7706 // Store Pointer
 7707 instruct storeimmP0(immP0 zero, memory8 mem)
 7708 %{
 7709   match(Set mem (StoreP mem zero));
 7710   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7711 
 7712   ins_cost(INSN_COST);
 7713   format %{ "str zr, $mem\t# ptr" %}
 7714 
 7715   ins_encode(aarch64_enc_str0(mem));
 7716 
 7717   ins_pipe(istore_mem);
 7718 %}
 7719 
 7720 // Store Compressed Pointer
 7721 instruct storeN(iRegN src, memory4 mem)
 7722 %{
 7723   match(Set mem (StoreN mem src));
 7724   predicate(!needs_releasing_store(n));
 7725 
 7726   ins_cost(INSN_COST);
 7727   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7728 
 7729   ins_encode(aarch64_enc_strw(src, mem));
 7730 
 7731   ins_pipe(istore_reg_mem);
 7732 %}
 7733 
 7734 instruct storeImmN0(immN0 zero, memory4 mem)
 7735 %{
 7736   match(Set mem (StoreN mem zero));
 7737   predicate(!needs_releasing_store(n));
 7738 
 7739   ins_cost(INSN_COST);
 7740   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7741 
 7742   ins_encode(aarch64_enc_strw0(mem));
 7743 
 7744   ins_pipe(istore_mem);
 7745 %}
 7746 
 7747 // Store Float
 7748 instruct storeF(vRegF src, memory4 mem)
 7749 %{
 7750   match(Set mem (StoreF mem src));
 7751   predicate(!needs_releasing_store(n));
 7752 
 7753   ins_cost(INSN_COST);
 7754   format %{ "strs  $src, $mem\t# float" %}
 7755 
 7756   ins_encode( aarch64_enc_strs(src, mem) );
 7757 
 7758   ins_pipe(pipe_class_memory);
 7759 %}
 7760 
 7761 // TODO
 7762 // implement storeImmF0 and storeFImmPacked
 7763 
 7764 // Store Double
 7765 instruct storeD(vRegD src, memory8 mem)
 7766 %{
 7767   match(Set mem (StoreD mem src));
 7768   predicate(!needs_releasing_store(n));
 7769 
 7770   ins_cost(INSN_COST);
 7771   format %{ "strd  $src, $mem\t# double" %}
 7772 
 7773   ins_encode( aarch64_enc_strd(src, mem) );
 7774 
 7775   ins_pipe(pipe_class_memory);
 7776 %}
 7777 
 7778 // Store Compressed Klass Pointer
 7779 instruct storeNKlass(iRegN src, memory4 mem)
 7780 %{
 7781   predicate(!needs_releasing_store(n));
 7782   match(Set mem (StoreNKlass mem src));
 7783 
 7784   ins_cost(INSN_COST);
 7785   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7786 
 7787   ins_encode(aarch64_enc_strw(src, mem));
 7788 
 7789   ins_pipe(istore_reg_mem);
 7790 %}
 7791 
 7792 // TODO
 7793 // implement storeImmD0 and storeDImmPacked
 7794 
 7795 // prefetch instructions
 7796 // Must be safe to execute with invalid address (cannot fault).
 7797 
 7798 instruct prefetchalloc( memory8 mem ) %{
 7799   match(PrefetchAllocation mem);
 7800 
 7801   ins_cost(INSN_COST);
 7802   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7803 
 7804   ins_encode( aarch64_enc_prefetchw(mem) );
 7805 
 7806   ins_pipe(iload_prefetch);
 7807 %}
 7808 
 7809 //  ---------------- volatile loads and stores ----------------
 7810 
 7811 // Load Byte (8 bit signed)
 7812 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7813 %{
 7814   match(Set dst (LoadB mem));
 7815 
 7816   ins_cost(VOLATILE_REF_COST);
 7817   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7818 
 7819   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7820 
 7821   ins_pipe(pipe_serial);
 7822 %}
 7823 
 7824 // Load Byte (8 bit signed) into long
 7825 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7826 %{
 7827   match(Set dst (ConvI2L (LoadB mem)));
 7828 
 7829   ins_cost(VOLATILE_REF_COST);
 7830   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7831 
 7832   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7833 
 7834   ins_pipe(pipe_serial);
 7835 %}
 7836 
 7837 // Load Byte (8 bit unsigned)
 7838 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7839 %{
 7840   match(Set dst (LoadUB mem));
 7841 
 7842   ins_cost(VOLATILE_REF_COST);
 7843   format %{ "ldarb  $dst, $mem\t# byte" %}
 7844 
 7845   ins_encode(aarch64_enc_ldarb(dst, mem));
 7846 
 7847   ins_pipe(pipe_serial);
 7848 %}
 7849 
 7850 // Load Byte (8 bit unsigned) into long
 7851 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7852 %{
 7853   match(Set dst (ConvI2L (LoadUB mem)));
 7854 
 7855   ins_cost(VOLATILE_REF_COST);
 7856   format %{ "ldarb  $dst, $mem\t# byte" %}
 7857 
 7858   ins_encode(aarch64_enc_ldarb(dst, mem));
 7859 
 7860   ins_pipe(pipe_serial);
 7861 %}
 7862 
 7863 // Load Short (16 bit signed)
 7864 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7865 %{
 7866   match(Set dst (LoadS mem));
 7867 
 7868   ins_cost(VOLATILE_REF_COST);
 7869   format %{ "ldarshw  $dst, $mem\t# short" %}
 7870 
 7871   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7872 
 7873   ins_pipe(pipe_serial);
 7874 %}
 7875 
 7876 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7877 %{
 7878   match(Set dst (LoadUS mem));
 7879 
 7880   ins_cost(VOLATILE_REF_COST);
 7881   format %{ "ldarhw  $dst, $mem\t# short" %}
 7882 
 7883   ins_encode(aarch64_enc_ldarhw(dst, mem));
 7884 
 7885   ins_pipe(pipe_serial);
 7886 %}
 7887 
 7888 // Load Short/Char (16 bit unsigned) into long
 7889 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7890 %{
 7891   match(Set dst (ConvI2L (LoadUS mem)));
 7892 
 7893   ins_cost(VOLATILE_REF_COST);
 7894   format %{ "ldarh  $dst, $mem\t# short" %}
 7895 
 7896   ins_encode(aarch64_enc_ldarh(dst, mem));
 7897 
 7898   ins_pipe(pipe_serial);
 7899 %}
 7900 
 7901 // Load Short/Char (16 bit signed) into long
 7902 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7903 %{
 7904   match(Set dst (ConvI2L (LoadS mem)));
 7905 
 7906   ins_cost(VOLATILE_REF_COST);
 7907   format %{ "ldarh  $dst, $mem\t# short" %}
 7908 
 7909   ins_encode(aarch64_enc_ldarsh(dst, mem));
 7910 
 7911   ins_pipe(pipe_serial);
 7912 %}
 7913 
 7914 // Load Integer (32 bit signed)
 7915 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7916 %{
 7917   match(Set dst (LoadI mem));
 7918 
 7919   ins_cost(VOLATILE_REF_COST);
 7920   format %{ "ldarw  $dst, $mem\t# int" %}
 7921 
 7922   ins_encode(aarch64_enc_ldarw(dst, mem));
 7923 
 7924   ins_pipe(pipe_serial);
 7925 %}
 7926 
 7927 // Load Integer (32 bit unsigned) into long
 7928 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 7929 %{
 7930   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7931 
 7932   ins_cost(VOLATILE_REF_COST);
 7933   format %{ "ldarw  $dst, $mem\t# int" %}
 7934 
 7935   ins_encode(aarch64_enc_ldarw(dst, mem));
 7936 
 7937   ins_pipe(pipe_serial);
 7938 %}
 7939 
 7940 // Load Long (64 bit signed)
 7941 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7942 %{
 7943   match(Set dst (LoadL mem));
 7944 
 7945   ins_cost(VOLATILE_REF_COST);
 7946   format %{ "ldar  $dst, $mem\t# int" %}
 7947 
 7948   ins_encode(aarch64_enc_ldar(dst, mem));
 7949 
 7950   ins_pipe(pipe_serial);
 7951 %}
 7952 
 7953 // Load Pointer
 7954 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 7955 %{
 7956   match(Set dst (LoadP mem));
 7957   predicate(n->as_Load()->barrier_data() == 0);
 7958 
 7959   ins_cost(VOLATILE_REF_COST);
 7960   format %{ "ldar  $dst, $mem\t# ptr" %}
 7961 
 7962   ins_encode(aarch64_enc_ldar(dst, mem));
 7963 
 7964   ins_pipe(pipe_serial);
 7965 %}
 7966 
 7967 // Load Compressed Pointer
 7968 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 7969 %{
 7970   match(Set dst (LoadN mem));
 7971 
 7972   ins_cost(VOLATILE_REF_COST);
 7973   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 7974 
 7975   ins_encode(aarch64_enc_ldarw(dst, mem));
 7976 
 7977   ins_pipe(pipe_serial);
 7978 %}
 7979 
 7980 // Load Float
 7981 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 7982 %{
 7983   match(Set dst (LoadF mem));
 7984 
 7985   ins_cost(VOLATILE_REF_COST);
 7986   format %{ "ldars  $dst, $mem\t# float" %}
 7987 
 7988   ins_encode( aarch64_enc_fldars(dst, mem) );
 7989 
 7990   ins_pipe(pipe_serial);
 7991 %}
 7992 
 7993 // Load Double
 7994 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 7995 %{
 7996   match(Set dst (LoadD mem));
 7997 
 7998   ins_cost(VOLATILE_REF_COST);
 7999   format %{ "ldard  $dst, $mem\t# double" %}
 8000 
 8001   ins_encode( aarch64_enc_fldard(dst, mem) );
 8002 
 8003   ins_pipe(pipe_serial);
 8004 %}
 8005 
 8006 // Store Byte
 8007 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8008 %{
 8009   match(Set mem (StoreB mem src));
 8010 
 8011   ins_cost(VOLATILE_REF_COST);
 8012   format %{ "stlrb  $src, $mem\t# byte" %}
 8013 
 8014   ins_encode(aarch64_enc_stlrb(src, mem));
 8015 
 8016   ins_pipe(pipe_class_memory);
 8017 %}
 8018 
 8019 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8020 %{
 8021   match(Set mem (StoreB mem zero));
 8022 
 8023   ins_cost(VOLATILE_REF_COST);
 8024   format %{ "stlrb  zr, $mem\t# byte" %}
 8025 
 8026   ins_encode(aarch64_enc_stlrb0(mem));
 8027 
 8028   ins_pipe(pipe_class_memory);
 8029 %}
 8030 
 8031 // Store Char/Short
 8032 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8033 %{
 8034   match(Set mem (StoreC mem src));
 8035 
 8036   ins_cost(VOLATILE_REF_COST);
 8037   format %{ "stlrh  $src, $mem\t# short" %}
 8038 
 8039   ins_encode(aarch64_enc_stlrh(src, mem));
 8040 
 8041   ins_pipe(pipe_class_memory);
 8042 %}
 8043 
 8044 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8045 %{
 8046   match(Set mem (StoreC mem zero));
 8047 
 8048   ins_cost(VOLATILE_REF_COST);
 8049   format %{ "stlrh  zr, $mem\t# short" %}
 8050 
 8051   ins_encode(aarch64_enc_stlrh0(mem));
 8052 
 8053   ins_pipe(pipe_class_memory);
 8054 %}
 8055 
 8056 // Store Integer
 8057 
 8058 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8059 %{
 8060   match(Set mem(StoreI mem src));
 8061 
 8062   ins_cost(VOLATILE_REF_COST);
 8063   format %{ "stlrw  $src, $mem\t# int" %}
 8064 
 8065   ins_encode(aarch64_enc_stlrw(src, mem));
 8066 
 8067   ins_pipe(pipe_class_memory);
 8068 %}
 8069 
 8070 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8071 %{
 8072   match(Set mem(StoreI mem zero));
 8073 
 8074   ins_cost(VOLATILE_REF_COST);
 8075   format %{ "stlrw  zr, $mem\t# int" %}
 8076 
 8077   ins_encode(aarch64_enc_stlrw0(mem));
 8078 
 8079   ins_pipe(pipe_class_memory);
 8080 %}
 8081 
 8082 // Store Long (64 bit signed)
 8083 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 8084 %{
 8085   match(Set mem (StoreL mem src));
 8086 
 8087   ins_cost(VOLATILE_REF_COST);
 8088   format %{ "stlr  $src, $mem\t# int" %}
 8089 
 8090   ins_encode(aarch64_enc_stlr(src, mem));
 8091 
 8092   ins_pipe(pipe_class_memory);
 8093 %}
 8094 
 8095 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 8096 %{
 8097   match(Set mem (StoreL mem zero));
 8098 
 8099   ins_cost(VOLATILE_REF_COST);
 8100   format %{ "stlr  zr, $mem\t# int" %}
 8101 
 8102   ins_encode(aarch64_enc_stlr0(mem));
 8103 
 8104   ins_pipe(pipe_class_memory);
 8105 %}
 8106 
 8107 // Store Pointer
 8108 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 8109 %{
 8110   match(Set mem (StoreP mem src));
 8111   predicate(n->as_Store()->barrier_data() == 0);
 8112 
 8113   ins_cost(VOLATILE_REF_COST);
 8114   format %{ "stlr  $src, $mem\t# ptr" %}
 8115 
 8116   ins_encode(aarch64_enc_stlr(src, mem));
 8117 
 8118   ins_pipe(pipe_class_memory);
 8119 %}
 8120 
 8121 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 8122 %{
 8123   match(Set mem (StoreP mem zero));
 8124   predicate(n->as_Store()->barrier_data() == 0);
 8125 
 8126   ins_cost(VOLATILE_REF_COST);
 8127   format %{ "stlr  zr, $mem\t# ptr" %}
 8128 
 8129   ins_encode(aarch64_enc_stlr0(mem));
 8130 
 8131   ins_pipe(pipe_class_memory);
 8132 %}
 8133 
 8134 // Store Compressed Pointer
 8135 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8136 %{
 8137   match(Set mem (StoreN mem src));
 8138 
 8139   ins_cost(VOLATILE_REF_COST);
 8140   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8141 
 8142   ins_encode(aarch64_enc_stlrw(src, mem));
 8143 
 8144   ins_pipe(pipe_class_memory);
 8145 %}
 8146 
 8147 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 8148 %{
 8149   match(Set mem (StoreN mem zero));
 8150 
 8151   ins_cost(VOLATILE_REF_COST);
 8152   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 8153 
 8154   ins_encode(aarch64_enc_stlrw0(mem));
 8155 
 8156   ins_pipe(pipe_class_memory);
 8157 %}
 8158 
 8159 // Store Float
 8160 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8161 %{
 8162   match(Set mem (StoreF mem src));
 8163 
 8164   ins_cost(VOLATILE_REF_COST);
 8165   format %{ "stlrs  $src, $mem\t# float" %}
 8166 
 8167   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8168 
 8169   ins_pipe(pipe_class_memory);
 8170 %}
 8171 
 8172 // TODO
 8173 // implement storeImmF0 and storeFImmPacked
 8174 
 8175 // Store Double
 8176 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8177 %{
 8178   match(Set mem (StoreD mem src));
 8179 
 8180   ins_cost(VOLATILE_REF_COST);
 8181   format %{ "stlrd  $src, $mem\t# double" %}
 8182 
 8183   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8184 
 8185   ins_pipe(pipe_class_memory);
 8186 %}
 8187 
 8188 //  ---------------- end of volatile loads and stores ----------------
 8189 
 8190 instruct cacheWB(indirect addr)
 8191 %{
 8192   predicate(VM_Version::supports_data_cache_line_flush());
 8193   match(CacheWB addr);
 8194 
 8195   ins_cost(100);
 8196   format %{"cache wb $addr" %}
 8197   ins_encode %{
 8198     assert($addr->index_position() < 0, "should be");
 8199     assert($addr$$disp == 0, "should be");
 8200     __ cache_wb(Address($addr$$base$$Register, 0));
 8201   %}
 8202   ins_pipe(pipe_slow); // XXX
 8203 %}
 8204 
 8205 instruct cacheWBPreSync()
 8206 %{
 8207   predicate(VM_Version::supports_data_cache_line_flush());
 8208   match(CacheWBPreSync);
 8209 
 8210   ins_cost(100);
 8211   format %{"cache wb presync" %}
 8212   ins_encode %{
 8213     __ cache_wbsync(true);
 8214   %}
 8215   ins_pipe(pipe_slow); // XXX
 8216 %}
 8217 
 8218 instruct cacheWBPostSync()
 8219 %{
 8220   predicate(VM_Version::supports_data_cache_line_flush());
 8221   match(CacheWBPostSync);
 8222 
 8223   ins_cost(100);
 8224   format %{"cache wb postsync" %}
 8225   ins_encode %{
 8226     __ cache_wbsync(false);
 8227   %}
 8228   ins_pipe(pipe_slow); // XXX
 8229 %}
 8230 
 8231 // ============================================================================
 8232 // BSWAP Instructions
 8233 
 8234 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8235   match(Set dst (ReverseBytesI src));
 8236 
 8237   ins_cost(INSN_COST);
 8238   format %{ "revw  $dst, $src" %}
 8239 
 8240   ins_encode %{
 8241     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8242   %}
 8243 
 8244   ins_pipe(ialu_reg);
 8245 %}
 8246 
 8247 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8248   match(Set dst (ReverseBytesL src));
 8249 
 8250   ins_cost(INSN_COST);
 8251   format %{ "rev  $dst, $src" %}
 8252 
 8253   ins_encode %{
 8254     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8255   %}
 8256 
 8257   ins_pipe(ialu_reg);
 8258 %}
 8259 
 8260 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8261   match(Set dst (ReverseBytesUS src));
 8262 
 8263   ins_cost(INSN_COST);
 8264   format %{ "rev16w  $dst, $src" %}
 8265 
 8266   ins_encode %{
 8267     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8268   %}
 8269 
 8270   ins_pipe(ialu_reg);
 8271 %}
 8272 
 8273 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8274   match(Set dst (ReverseBytesS src));
 8275 
 8276   ins_cost(INSN_COST);
 8277   format %{ "rev16w  $dst, $src\n\t"
 8278             "sbfmw $dst, $dst, #0, #15" %}
 8279 
 8280   ins_encode %{
 8281     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8282     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8283   %}
 8284 
 8285   ins_pipe(ialu_reg);
 8286 %}
 8287 
 8288 // ============================================================================
 8289 // Zero Count Instructions
 8290 
 8291 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8292   match(Set dst (CountLeadingZerosI src));
 8293 
 8294   ins_cost(INSN_COST);
 8295   format %{ "clzw  $dst, $src" %}
 8296   ins_encode %{
 8297     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8298   %}
 8299 
 8300   ins_pipe(ialu_reg);
 8301 %}
 8302 
 8303 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8304   match(Set dst (CountLeadingZerosL src));
 8305 
 8306   ins_cost(INSN_COST);
 8307   format %{ "clz   $dst, $src" %}
 8308   ins_encode %{
 8309     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8310   %}
 8311 
 8312   ins_pipe(ialu_reg);
 8313 %}
 8314 
 8315 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8316   match(Set dst (CountTrailingZerosI src));
 8317 
 8318   ins_cost(INSN_COST * 2);
 8319   format %{ "rbitw  $dst, $src\n\t"
 8320             "clzw   $dst, $dst" %}
 8321   ins_encode %{
 8322     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8323     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8324   %}
 8325 
 8326   ins_pipe(ialu_reg);
 8327 %}
 8328 
 8329 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8330   match(Set dst (CountTrailingZerosL src));
 8331 
 8332   ins_cost(INSN_COST * 2);
 8333   format %{ "rbit   $dst, $src\n\t"
 8334             "clz    $dst, $dst" %}
 8335   ins_encode %{
 8336     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8337     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8338   %}
 8339 
 8340   ins_pipe(ialu_reg);
 8341 %}
 8342 
 8343 //---------- Population Count Instructions -------------------------------------
 8344 //
 8345 
 8346 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8347   match(Set dst (PopCountI src));
 8348   effect(TEMP tmp);
 8349   ins_cost(INSN_COST * 13);
 8350 
 8351   format %{ "movw   $src, $src\n\t"
 8352             "mov    $tmp, $src\t# vector (1D)\n\t"
 8353             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8354             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8355             "mov    $dst, $tmp\t# vector (1D)" %}
 8356   ins_encode %{
 8357     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8358     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8359     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8360     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8361     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8362   %}
 8363 
 8364   ins_pipe(pipe_class_default);
 8365 %}
 8366 
 8367 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8368   match(Set dst (PopCountI (LoadI mem)));
 8369   effect(TEMP tmp);
 8370   ins_cost(INSN_COST * 13);
 8371 
 8372   format %{ "ldrs   $tmp, $mem\n\t"
 8373             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8374             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8375             "mov    $dst, $tmp\t# vector (1D)" %}
 8376   ins_encode %{
 8377     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8378     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8379               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8380     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8381     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8382     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8383   %}
 8384 
 8385   ins_pipe(pipe_class_default);
 8386 %}
 8387 
 8388 // Note: Long.bitCount(long) returns an int.
 8389 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8390   match(Set dst (PopCountL src));
 8391   effect(TEMP tmp);
 8392   ins_cost(INSN_COST * 13);
 8393 
 8394   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8395             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8396             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8397             "mov    $dst, $tmp\t# vector (1D)" %}
 8398   ins_encode %{
 8399     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8400     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8401     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8402     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8403   %}
 8404 
 8405   ins_pipe(pipe_class_default);
 8406 %}
 8407 
 8408 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8409   match(Set dst (PopCountL (LoadL mem)));
 8410   effect(TEMP tmp);
 8411   ins_cost(INSN_COST * 13);
 8412 
 8413   format %{ "ldrd   $tmp, $mem\n\t"
 8414             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8415             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8416             "mov    $dst, $tmp\t# vector (1D)" %}
 8417   ins_encode %{
 8418     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8419     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8420               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8421     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8422     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8423     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8424   %}
 8425 
 8426   ins_pipe(pipe_class_default);
 8427 %}
 8428 
 8429 // ============================================================================
 8430 // MemBar Instruction
 8431 
 8432 instruct load_fence() %{
 8433   match(LoadFence);
 8434   ins_cost(VOLATILE_REF_COST);
 8435 
 8436   format %{ "load_fence" %}
 8437 
 8438   ins_encode %{
 8439     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8440   %}
 8441   ins_pipe(pipe_serial);
 8442 %}
 8443 
 8444 instruct unnecessary_membar_acquire() %{
 8445   predicate(unnecessary_acquire(n));
 8446   match(MemBarAcquire);
 8447   ins_cost(0);
 8448 
 8449   format %{ "membar_acquire (elided)" %}
 8450 
 8451   ins_encode %{
 8452     __ block_comment("membar_acquire (elided)");
 8453   %}
 8454 
 8455   ins_pipe(pipe_class_empty);
 8456 %}
 8457 
 8458 instruct membar_acquire() %{
 8459   match(MemBarAcquire);
 8460   ins_cost(VOLATILE_REF_COST);
 8461 
 8462   format %{ "membar_acquire\n\t"
 8463             "dmb ish" %}
 8464 
 8465   ins_encode %{
 8466     __ block_comment("membar_acquire");
 8467     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8468   %}
 8469 
 8470   ins_pipe(pipe_serial);
 8471 %}
 8472 
 8473 
 8474 instruct membar_acquire_lock() %{
 8475   match(MemBarAcquireLock);
 8476   ins_cost(VOLATILE_REF_COST);
 8477 
 8478   format %{ "membar_acquire_lock (elided)" %}
 8479 
 8480   ins_encode %{
 8481     __ block_comment("membar_acquire_lock (elided)");
 8482   %}
 8483 
 8484   ins_pipe(pipe_serial);
 8485 %}
 8486 
 8487 instruct store_fence() %{
 8488   match(StoreFence);
 8489   ins_cost(VOLATILE_REF_COST);
 8490 
 8491   format %{ "store_fence" %}
 8492 
 8493   ins_encode %{
 8494     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8495   %}
 8496   ins_pipe(pipe_serial);
 8497 %}
 8498 
 8499 instruct unnecessary_membar_release() %{
 8500   predicate(unnecessary_release(n));
 8501   match(MemBarRelease);
 8502   ins_cost(0);
 8503 
 8504   format %{ "membar_release (elided)" %}
 8505 
 8506   ins_encode %{
 8507     __ block_comment("membar_release (elided)");
 8508   %}
 8509   ins_pipe(pipe_serial);
 8510 %}
 8511 
 8512 instruct membar_release() %{
 8513   match(MemBarRelease);
 8514   ins_cost(VOLATILE_REF_COST);
 8515 
 8516   format %{ "membar_release\n\t"
 8517             "dmb ish" %}
 8518 
 8519   ins_encode %{
 8520     __ block_comment("membar_release");
 8521     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8522   %}
 8523   ins_pipe(pipe_serial);
 8524 %}
 8525 
 8526 instruct membar_storestore() %{
 8527   match(MemBarStoreStore);
 8528   match(StoreStoreFence);
 8529   ins_cost(VOLATILE_REF_COST);
 8530 
 8531   format %{ "MEMBAR-store-store" %}
 8532 
 8533   ins_encode %{
 8534     __ membar(Assembler::StoreStore);
 8535   %}
 8536   ins_pipe(pipe_serial);
 8537 %}
 8538 
 8539 instruct membar_release_lock() %{
 8540   match(MemBarReleaseLock);
 8541   ins_cost(VOLATILE_REF_COST);
 8542 
 8543   format %{ "membar_release_lock (elided)" %}
 8544 
 8545   ins_encode %{
 8546     __ block_comment("membar_release_lock (elided)");
 8547   %}
 8548 
 8549   ins_pipe(pipe_serial);
 8550 %}
 8551 
 8552 instruct unnecessary_membar_volatile() %{
 8553   predicate(unnecessary_volatile(n));
 8554   match(MemBarVolatile);
 8555   ins_cost(0);
 8556 
 8557   format %{ "membar_volatile (elided)" %}
 8558 
 8559   ins_encode %{
 8560     __ block_comment("membar_volatile (elided)");
 8561   %}
 8562 
 8563   ins_pipe(pipe_serial);
 8564 %}
 8565 
 8566 instruct membar_volatile() %{
 8567   match(MemBarVolatile);
 8568   ins_cost(VOLATILE_REF_COST*100);
 8569 
 8570   format %{ "membar_volatile\n\t"
 8571              "dmb ish"%}
 8572 
 8573   ins_encode %{
 8574     __ block_comment("membar_volatile");
 8575     __ membar(Assembler::StoreLoad);
 8576   %}
 8577 
 8578   ins_pipe(pipe_serial);
 8579 %}
 8580 
 8581 // ============================================================================
 8582 // Cast/Convert Instructions
 8583 
 8584 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8585   match(Set dst (CastX2P src));
 8586 
 8587   ins_cost(INSN_COST);
 8588   format %{ "mov $dst, $src\t# long -> ptr" %}
 8589 
 8590   ins_encode %{
 8591     if ($dst$$reg != $src$$reg) {
 8592       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8593     }
 8594   %}
 8595 
 8596   ins_pipe(ialu_reg);
 8597 %}
 8598 
 8599 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8600   match(Set dst (CastP2X src));
 8601 
 8602   ins_cost(INSN_COST);
 8603   format %{ "mov $dst, $src\t# ptr -> long" %}
 8604 
 8605   ins_encode %{
 8606     if ($dst$$reg != $src$$reg) {
 8607       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8608     }
 8609   %}
 8610 
 8611   ins_pipe(ialu_reg);
 8612 %}
 8613 
 8614 // Convert oop into int for vectors alignment masking
 8615 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8616   match(Set dst (ConvL2I (CastP2X src)));
 8617 
 8618   ins_cost(INSN_COST);
 8619   format %{ "movw $dst, $src\t# ptr -> int" %}
 8620   ins_encode %{
 8621     __ movw($dst$$Register, $src$$Register);
 8622   %}
 8623 
 8624   ins_pipe(ialu_reg);
 8625 %}
 8626 
 8627 // Convert compressed oop into int for vectors alignment masking
 8628 // in case of 32bit oops (heap < 4Gb).
 8629 instruct convN2I(iRegINoSp dst, iRegN src)
 8630 %{
 8631   predicate(CompressedOops::shift() == 0);
 8632   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8633 
 8634   ins_cost(INSN_COST);
 8635   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8636   ins_encode %{
 8637     __ movw($dst$$Register, $src$$Register);
 8638   %}
 8639 
 8640   ins_pipe(ialu_reg);
 8641 %}
 8642 
 8643 
 8644 // Convert oop pointer into compressed form
 8645 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8646   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8647   match(Set dst (EncodeP src));
 8648   effect(KILL cr);
 8649   ins_cost(INSN_COST * 3);
 8650   format %{ "encode_heap_oop $dst, $src" %}
 8651   ins_encode %{
 8652     Register s = $src$$Register;
 8653     Register d = $dst$$Register;
 8654     __ encode_heap_oop(d, s);
 8655   %}
 8656   ins_pipe(ialu_reg);
 8657 %}
 8658 
 8659 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8660   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8661   match(Set dst (EncodeP src));
 8662   ins_cost(INSN_COST * 3);
 8663   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8664   ins_encode %{
 8665     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8666   %}
 8667   ins_pipe(ialu_reg);
 8668 %}
 8669 
 8670 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8671   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8672             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8673   match(Set dst (DecodeN src));
 8674   ins_cost(INSN_COST * 3);
 8675   format %{ "decode_heap_oop $dst, $src" %}
 8676   ins_encode %{
 8677     Register s = $src$$Register;
 8678     Register d = $dst$$Register;
 8679     __ decode_heap_oop(d, s);
 8680   %}
 8681   ins_pipe(ialu_reg);
 8682 %}
 8683 
 8684 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8685   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8686             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8687   match(Set dst (DecodeN src));
 8688   ins_cost(INSN_COST * 3);
 8689   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8690   ins_encode %{
 8691     Register s = $src$$Register;
 8692     Register d = $dst$$Register;
 8693     __ decode_heap_oop_not_null(d, s);
 8694   %}
 8695   ins_pipe(ialu_reg);
 8696 %}
 8697 
 8698 // n.b. AArch64 implementations of encode_klass_not_null and
 8699 // decode_klass_not_null do not modify the flags register so, unlike
 8700 // Intel, we don't kill CR as a side effect here
 8701 
 8702 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8703   match(Set dst (EncodePKlass src));
 8704 
 8705   ins_cost(INSN_COST * 3);
 8706   format %{ "encode_klass_not_null $dst,$src" %}
 8707 
 8708   ins_encode %{
 8709     Register src_reg = as_Register($src$$reg);
 8710     Register dst_reg = as_Register($dst$$reg);
 8711     __ encode_klass_not_null(dst_reg, src_reg);
 8712   %}
 8713 
 8714    ins_pipe(ialu_reg);
 8715 %}
 8716 
 8717 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8718   match(Set dst (DecodeNKlass src));
 8719 
 8720   ins_cost(INSN_COST * 3);
 8721   format %{ "decode_klass_not_null $dst,$src" %}
 8722 
 8723   ins_encode %{
 8724     Register src_reg = as_Register($src$$reg);
 8725     Register dst_reg = as_Register($dst$$reg);
 8726     if (dst_reg != src_reg) {
 8727       __ decode_klass_not_null(dst_reg, src_reg);
 8728     } else {
 8729       __ decode_klass_not_null(dst_reg);
 8730     }
 8731   %}
 8732 
 8733    ins_pipe(ialu_reg);
 8734 %}
 8735 
 8736 instruct checkCastPP(iRegPNoSp dst)
 8737 %{
 8738   match(Set dst (CheckCastPP dst));
 8739 
 8740   size(0);
 8741   format %{ "# checkcastPP of $dst" %}
 8742   ins_encode(/* empty encoding */);
 8743   ins_pipe(pipe_class_empty);
 8744 %}
 8745 
 8746 instruct castPP(iRegPNoSp dst)
 8747 %{
 8748   match(Set dst (CastPP dst));
 8749 
 8750   size(0);
 8751   format %{ "# castPP of $dst" %}
 8752   ins_encode(/* empty encoding */);
 8753   ins_pipe(pipe_class_empty);
 8754 %}
 8755 
 8756 instruct castII(iRegI dst)
 8757 %{
 8758   match(Set dst (CastII dst));
 8759 
 8760   size(0);
 8761   format %{ "# castII of $dst" %}
 8762   ins_encode(/* empty encoding */);
 8763   ins_cost(0);
 8764   ins_pipe(pipe_class_empty);
 8765 %}
 8766 
 8767 instruct castLL(iRegL dst)
 8768 %{
 8769   match(Set dst (CastLL dst));
 8770 
 8771   size(0);
 8772   format %{ "# castLL of $dst" %}
 8773   ins_encode(/* empty encoding */);
 8774   ins_cost(0);
 8775   ins_pipe(pipe_class_empty);
 8776 %}
 8777 
 8778 instruct castFF(vRegF dst)
 8779 %{
 8780   match(Set dst (CastFF dst));
 8781 
 8782   size(0);
 8783   format %{ "# castFF of $dst" %}
 8784   ins_encode(/* empty encoding */);
 8785   ins_cost(0);
 8786   ins_pipe(pipe_class_empty);
 8787 %}
 8788 
 8789 instruct castDD(vRegD dst)
 8790 %{
 8791   match(Set dst (CastDD dst));
 8792 
 8793   size(0);
 8794   format %{ "# castDD of $dst" %}
 8795   ins_encode(/* empty encoding */);
 8796   ins_cost(0);
 8797   ins_pipe(pipe_class_empty);
 8798 %}
 8799 
 8800 instruct castVV(vReg dst)
 8801 %{
 8802   match(Set dst (CastVV dst));
 8803 
 8804   size(0);
 8805   format %{ "# castVV of $dst" %}
 8806   ins_encode(/* empty encoding */);
 8807   ins_cost(0);
 8808   ins_pipe(pipe_class_empty);
 8809 %}
 8810 
 8811 instruct castVVMask(pRegGov dst)
 8812 %{
 8813   match(Set dst (CastVV dst));
 8814 
 8815   size(0);
 8816   format %{ "# castVV of $dst" %}
 8817   ins_encode(/* empty encoding */);
 8818   ins_cost(0);
 8819   ins_pipe(pipe_class_empty);
 8820 %}
 8821 
 8822 // ============================================================================
 8823 // Atomic operation instructions
 8824 //
 8825 
 8826 // standard CompareAndSwapX when we are using barriers
 8827 // these have higher priority than the rules selected by a predicate
 8828 
 8829 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 8830 // can't match them
 8831 
 8832 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8833 
 8834   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8835   ins_cost(2 * VOLATILE_REF_COST);
 8836 
 8837   effect(KILL cr);
 8838 
 8839   format %{
 8840     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8841     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8842   %}
 8843 
 8844   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 8845             aarch64_enc_cset_eq(res));
 8846 
 8847   ins_pipe(pipe_slow);
 8848 %}
 8849 
 8850 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8851 
 8852   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8853   ins_cost(2 * VOLATILE_REF_COST);
 8854 
 8855   effect(KILL cr);
 8856 
 8857   format %{
 8858     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8859     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8860   %}
 8861 
 8862   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 8863             aarch64_enc_cset_eq(res));
 8864 
 8865   ins_pipe(pipe_slow);
 8866 %}
 8867 
 8868 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8869 
 8870   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8871   ins_cost(2 * VOLATILE_REF_COST);
 8872 
 8873   effect(KILL cr);
 8874 
 8875  format %{
 8876     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8877     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8878  %}
 8879 
 8880  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8881             aarch64_enc_cset_eq(res));
 8882 
 8883   ins_pipe(pipe_slow);
 8884 %}
 8885 
 8886 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8887 
 8888   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8889   ins_cost(2 * VOLATILE_REF_COST);
 8890 
 8891   effect(KILL cr);
 8892 
 8893  format %{
 8894     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8895     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8896  %}
 8897 
 8898  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8899             aarch64_enc_cset_eq(res));
 8900 
 8901   ins_pipe(pipe_slow);
 8902 %}
 8903 
 8904 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8905 
 8906   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8907   predicate(n->as_LoadStore()->barrier_data() == 0);
 8908   ins_cost(2 * VOLATILE_REF_COST);
 8909 
 8910   effect(KILL cr);
 8911 
 8912  format %{
 8913     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8914     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8915  %}
 8916 
 8917  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8918             aarch64_enc_cset_eq(res));
 8919 
 8920   ins_pipe(pipe_slow);
 8921 %}
 8922 
 8923 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8924 
 8925   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8926   ins_cost(2 * VOLATILE_REF_COST);
 8927 
 8928   effect(KILL cr);
 8929 
 8930  format %{
 8931     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8932     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8933  %}
 8934 
 8935  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8936             aarch64_enc_cset_eq(res));
 8937 
 8938   ins_pipe(pipe_slow);
 8939 %}
 8940 
 8941 // alternative CompareAndSwapX when we are eliding barriers
 8942 
 8943 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8944 
 8945   predicate(needs_acquiring_load_exclusive(n));
 8946   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8947   ins_cost(VOLATILE_REF_COST);
 8948 
 8949   effect(KILL cr);
 8950 
 8951   format %{
 8952     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8953     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8954   %}
 8955 
 8956   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 8957             aarch64_enc_cset_eq(res));
 8958 
 8959   ins_pipe(pipe_slow);
 8960 %}
 8961 
 8962 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8963 
 8964   predicate(needs_acquiring_load_exclusive(n));
 8965   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8966   ins_cost(VOLATILE_REF_COST);
 8967 
 8968   effect(KILL cr);
 8969 
 8970   format %{
 8971     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8972     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8973   %}
 8974 
 8975   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 8976             aarch64_enc_cset_eq(res));
 8977 
 8978   ins_pipe(pipe_slow);
 8979 %}
 8980 
 8981 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8982 
 8983   predicate(needs_acquiring_load_exclusive(n));
 8984   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8985   ins_cost(VOLATILE_REF_COST);
 8986 
 8987   effect(KILL cr);
 8988 
 8989  format %{
 8990     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8991     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8992  %}
 8993 
 8994  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8995             aarch64_enc_cset_eq(res));
 8996 
 8997   ins_pipe(pipe_slow);
 8998 %}
 8999 
 9000 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9001 
 9002   predicate(needs_acquiring_load_exclusive(n));
 9003   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9004   ins_cost(VOLATILE_REF_COST);
 9005 
 9006   effect(KILL cr);
 9007 
 9008  format %{
 9009     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9010     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9011  %}
 9012 
 9013  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9014             aarch64_enc_cset_eq(res));
 9015 
 9016   ins_pipe(pipe_slow);
 9017 %}
 9018 
 9019 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9020 
 9021   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9022   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9023   ins_cost(VOLATILE_REF_COST);
 9024 
 9025   effect(KILL cr);
 9026 
 9027  format %{
 9028     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9029     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9030  %}
 9031 
 9032  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9033             aarch64_enc_cset_eq(res));
 9034 
 9035   ins_pipe(pipe_slow);
 9036 %}
 9037 
 9038 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9039 
 9040   predicate(needs_acquiring_load_exclusive(n));
 9041   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9042   ins_cost(VOLATILE_REF_COST);
 9043 
 9044   effect(KILL cr);
 9045 
 9046  format %{
 9047     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9048     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9049  %}
 9050 
 9051  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9052             aarch64_enc_cset_eq(res));
 9053 
 9054   ins_pipe(pipe_slow);
 9055 %}
 9056 
 9057 
 9058 // ---------------------------------------------------------------------
 9059 
 9060 // BEGIN This section of the file is automatically generated. Do not edit --------------
 9061 
 9062 // Sundry CAS operations.  Note that release is always true,
 9063 // regardless of the memory ordering of the CAS.  This is because we
 9064 // need the volatile case to be sequentially consistent but there is
 9065 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 9066 // can't check the type of memory ordering here, so we always emit a
 9067 // STLXR.
 9068 
 9069 // This section is generated from cas.m4
 9070 
 9071 
 9072 // This pattern is generated automatically from cas.m4.
 9073 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9074 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9075   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9076   ins_cost(2 * VOLATILE_REF_COST);
 9077   effect(TEMP_DEF res, KILL cr);
 9078   format %{
 9079     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9080   %}
 9081   ins_encode %{
 9082     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9083                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9084                /*weak*/ false, $res$$Register);
 9085     __ sxtbw($res$$Register, $res$$Register);
 9086   %}
 9087   ins_pipe(pipe_slow);
 9088 %}
 9089 
 9090 // This pattern is generated automatically from cas.m4.
 9091 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9092 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9093   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9094   ins_cost(2 * VOLATILE_REF_COST);
 9095   effect(TEMP_DEF res, KILL cr);
 9096   format %{
 9097     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9098   %}
 9099   ins_encode %{
 9100     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9101                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9102                /*weak*/ false, $res$$Register);
 9103     __ sxthw($res$$Register, $res$$Register);
 9104   %}
 9105   ins_pipe(pipe_slow);
 9106 %}
 9107 
 9108 // This pattern is generated automatically from cas.m4.
 9109 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9110 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9111   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9112   ins_cost(2 * VOLATILE_REF_COST);
 9113   effect(TEMP_DEF res, KILL cr);
 9114   format %{
 9115     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9116   %}
 9117   ins_encode %{
 9118     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9119                Assembler::word, /*acquire*/ false, /*release*/ true,
 9120                /*weak*/ false, $res$$Register);
 9121   %}
 9122   ins_pipe(pipe_slow);
 9123 %}
 9124 
 9125 // This pattern is generated automatically from cas.m4.
 9126 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9127 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9128   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9129   ins_cost(2 * VOLATILE_REF_COST);
 9130   effect(TEMP_DEF res, KILL cr);
 9131   format %{
 9132     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9133   %}
 9134   ins_encode %{
 9135     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9136                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9137                /*weak*/ false, $res$$Register);
 9138   %}
 9139   ins_pipe(pipe_slow);
 9140 %}
 9141 
 9142 // This pattern is generated automatically from cas.m4.
 9143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9144 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9145   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9146   ins_cost(2 * VOLATILE_REF_COST);
 9147   effect(TEMP_DEF res, KILL cr);
 9148   format %{
 9149     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9150   %}
 9151   ins_encode %{
 9152     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9153                Assembler::word, /*acquire*/ false, /*release*/ true,
 9154                /*weak*/ false, $res$$Register);
 9155   %}
 9156   ins_pipe(pipe_slow);
 9157 %}
 9158 
 9159 // This pattern is generated automatically from cas.m4.
 9160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9161 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9162   predicate(n->as_LoadStore()->barrier_data() == 0);
 9163   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9164   ins_cost(2 * VOLATILE_REF_COST);
 9165   effect(TEMP_DEF res, KILL cr);
 9166   format %{
 9167     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9168   %}
 9169   ins_encode %{
 9170     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9171                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9172                /*weak*/ false, $res$$Register);
 9173   %}
 9174   ins_pipe(pipe_slow);
 9175 %}
 9176 
 9177 // This pattern is generated automatically from cas.m4.
 9178 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9179 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9180   predicate(needs_acquiring_load_exclusive(n));
 9181   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9182   ins_cost(VOLATILE_REF_COST);
 9183   effect(TEMP_DEF res, KILL cr);
 9184   format %{
 9185     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9186   %}
 9187   ins_encode %{
 9188     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9189                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9190                /*weak*/ false, $res$$Register);
 9191     __ sxtbw($res$$Register, $res$$Register);
 9192   %}
 9193   ins_pipe(pipe_slow);
 9194 %}
 9195 
 9196 // This pattern is generated automatically from cas.m4.
 9197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9198 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9199   predicate(needs_acquiring_load_exclusive(n));
 9200   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9201   ins_cost(VOLATILE_REF_COST);
 9202   effect(TEMP_DEF res, KILL cr);
 9203   format %{
 9204     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9205   %}
 9206   ins_encode %{
 9207     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9208                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9209                /*weak*/ false, $res$$Register);
 9210     __ sxthw($res$$Register, $res$$Register);
 9211   %}
 9212   ins_pipe(pipe_slow);
 9213 %}
 9214 
 9215 // This pattern is generated automatically from cas.m4.
 9216 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9217 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9218   predicate(needs_acquiring_load_exclusive(n));
 9219   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9220   ins_cost(VOLATILE_REF_COST);
 9221   effect(TEMP_DEF res, KILL cr);
 9222   format %{
 9223     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9224   %}
 9225   ins_encode %{
 9226     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9227                Assembler::word, /*acquire*/ true, /*release*/ true,
 9228                /*weak*/ false, $res$$Register);
 9229   %}
 9230   ins_pipe(pipe_slow);
 9231 %}
 9232 
 9233 // This pattern is generated automatically from cas.m4.
 9234 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9235 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9236   predicate(needs_acquiring_load_exclusive(n));
 9237   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9238   ins_cost(VOLATILE_REF_COST);
 9239   effect(TEMP_DEF res, KILL cr);
 9240   format %{
 9241     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9242   %}
 9243   ins_encode %{
 9244     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9245                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9246                /*weak*/ false, $res$$Register);
 9247   %}
 9248   ins_pipe(pipe_slow);
 9249 %}
 9250 
 9251 // This pattern is generated automatically from cas.m4.
 9252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9253 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9254   predicate(needs_acquiring_load_exclusive(n));
 9255   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9256   ins_cost(VOLATILE_REF_COST);
 9257   effect(TEMP_DEF res, KILL cr);
 9258   format %{
 9259     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9260   %}
 9261   ins_encode %{
 9262     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9263                Assembler::word, /*acquire*/ true, /*release*/ true,
 9264                /*weak*/ false, $res$$Register);
 9265   %}
 9266   ins_pipe(pipe_slow);
 9267 %}
 9268 
 9269 // This pattern is generated automatically from cas.m4.
 9270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9271 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9272   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9273   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9274   ins_cost(VOLATILE_REF_COST);
 9275   effect(TEMP_DEF res, KILL cr);
 9276   format %{
 9277     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9278   %}
 9279   ins_encode %{
 9280     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9281                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9282                /*weak*/ false, $res$$Register);
 9283   %}
 9284   ins_pipe(pipe_slow);
 9285 %}
 9286 
 9287 // This pattern is generated automatically from cas.m4.
 9288 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9289 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9290   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9291   ins_cost(2 * VOLATILE_REF_COST);
 9292   effect(KILL cr);
 9293   format %{
 9294     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9295     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9296   %}
 9297   ins_encode %{
 9298     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9299                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9300                /*weak*/ true, noreg);
 9301     __ csetw($res$$Register, Assembler::EQ);
 9302   %}
 9303   ins_pipe(pipe_slow);
 9304 %}
 9305 
 9306 // This pattern is generated automatically from cas.m4.
 9307 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9308 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9309   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9310   ins_cost(2 * VOLATILE_REF_COST);
 9311   effect(KILL cr);
 9312   format %{
 9313     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9314     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9315   %}
 9316   ins_encode %{
 9317     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9318                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9319                /*weak*/ true, noreg);
 9320     __ csetw($res$$Register, Assembler::EQ);
 9321   %}
 9322   ins_pipe(pipe_slow);
 9323 %}
 9324 
 9325 // This pattern is generated automatically from cas.m4.
 9326 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9327 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9328   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9329   ins_cost(2 * VOLATILE_REF_COST);
 9330   effect(KILL cr);
 9331   format %{
 9332     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9333     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9334   %}
 9335   ins_encode %{
 9336     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9337                Assembler::word, /*acquire*/ false, /*release*/ true,
 9338                /*weak*/ true, noreg);
 9339     __ csetw($res$$Register, Assembler::EQ);
 9340   %}
 9341   ins_pipe(pipe_slow);
 9342 %}
 9343 
 9344 // This pattern is generated automatically from cas.m4.
 9345 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9346 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9347   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9348   ins_cost(2 * VOLATILE_REF_COST);
 9349   effect(KILL cr);
 9350   format %{
 9351     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9352     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9353   %}
 9354   ins_encode %{
 9355     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9356                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9357                /*weak*/ true, noreg);
 9358     __ csetw($res$$Register, Assembler::EQ);
 9359   %}
 9360   ins_pipe(pipe_slow);
 9361 %}
 9362 
 9363 // This pattern is generated automatically from cas.m4.
 9364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9365 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9366   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9367   ins_cost(2 * VOLATILE_REF_COST);
 9368   effect(KILL cr);
 9369   format %{
 9370     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9371     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9372   %}
 9373   ins_encode %{
 9374     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9375                Assembler::word, /*acquire*/ false, /*release*/ true,
 9376                /*weak*/ true, noreg);
 9377     __ csetw($res$$Register, Assembler::EQ);
 9378   %}
 9379   ins_pipe(pipe_slow);
 9380 %}
 9381 
 9382 // This pattern is generated automatically from cas.m4.
 9383 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9384 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9385   predicate(n->as_LoadStore()->barrier_data() == 0);
 9386   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9387   ins_cost(2 * VOLATILE_REF_COST);
 9388   effect(KILL cr);
 9389   format %{
 9390     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9391     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9392   %}
 9393   ins_encode %{
 9394     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9395                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9396                /*weak*/ true, noreg);
 9397     __ csetw($res$$Register, Assembler::EQ);
 9398   %}
 9399   ins_pipe(pipe_slow);
 9400 %}
 9401 
 9402 // This pattern is generated automatically from cas.m4.
 9403 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9404 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9405   predicate(needs_acquiring_load_exclusive(n));
 9406   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9407   ins_cost(VOLATILE_REF_COST);
 9408   effect(KILL cr);
 9409   format %{
 9410     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9411     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9412   %}
 9413   ins_encode %{
 9414     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9415                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9416                /*weak*/ true, noreg);
 9417     __ csetw($res$$Register, Assembler::EQ);
 9418   %}
 9419   ins_pipe(pipe_slow);
 9420 %}
 9421 
 9422 // This pattern is generated automatically from cas.m4.
 9423 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9424 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9425   predicate(needs_acquiring_load_exclusive(n));
 9426   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9427   ins_cost(VOLATILE_REF_COST);
 9428   effect(KILL cr);
 9429   format %{
 9430     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9431     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9432   %}
 9433   ins_encode %{
 9434     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9435                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9436                /*weak*/ true, noreg);
 9437     __ csetw($res$$Register, Assembler::EQ);
 9438   %}
 9439   ins_pipe(pipe_slow);
 9440 %}
 9441 
 9442 // This pattern is generated automatically from cas.m4.
 9443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9444 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9445   predicate(needs_acquiring_load_exclusive(n));
 9446   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9447   ins_cost(VOLATILE_REF_COST);
 9448   effect(KILL cr);
 9449   format %{
 9450     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9451     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9452   %}
 9453   ins_encode %{
 9454     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9455                Assembler::word, /*acquire*/ true, /*release*/ true,
 9456                /*weak*/ true, noreg);
 9457     __ csetw($res$$Register, Assembler::EQ);
 9458   %}
 9459   ins_pipe(pipe_slow);
 9460 %}
 9461 
 9462 // This pattern is generated automatically from cas.m4.
 9463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9464 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9465   predicate(needs_acquiring_load_exclusive(n));
 9466   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9467   ins_cost(VOLATILE_REF_COST);
 9468   effect(KILL cr);
 9469   format %{
 9470     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9471     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9472   %}
 9473   ins_encode %{
 9474     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9475                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9476                /*weak*/ true, noreg);
 9477     __ csetw($res$$Register, Assembler::EQ);
 9478   %}
 9479   ins_pipe(pipe_slow);
 9480 %}
 9481 
 9482 // This pattern is generated automatically from cas.m4.
 9483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9484 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9485   predicate(needs_acquiring_load_exclusive(n));
 9486   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9487   ins_cost(VOLATILE_REF_COST);
 9488   effect(KILL cr);
 9489   format %{
 9490     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9491     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9492   %}
 9493   ins_encode %{
 9494     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9495                Assembler::word, /*acquire*/ true, /*release*/ true,
 9496                /*weak*/ true, noreg);
 9497     __ csetw($res$$Register, Assembler::EQ);
 9498   %}
 9499   ins_pipe(pipe_slow);
 9500 %}
 9501 
 9502 // This pattern is generated automatically from cas.m4.
 9503 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9504 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9505   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9506   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9507   ins_cost(VOLATILE_REF_COST);
 9508   effect(KILL cr);
 9509   format %{
 9510     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9511     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9512   %}
 9513   ins_encode %{
 9514     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9515                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9516                /*weak*/ true, noreg);
 9517     __ csetw($res$$Register, Assembler::EQ);
 9518   %}
 9519   ins_pipe(pipe_slow);
 9520 %}
 9521 
 9522 // END This section of the file is automatically generated. Do not edit --------------
 9523 // ---------------------------------------------------------------------
 9524 
 9525 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9526   match(Set prev (GetAndSetI mem newv));
 9527   ins_cost(2 * VOLATILE_REF_COST);
 9528   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9529   ins_encode %{
 9530     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9531   %}
 9532   ins_pipe(pipe_serial);
 9533 %}
 9534 
 9535 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9536   match(Set prev (GetAndSetL mem newv));
 9537   ins_cost(2 * VOLATILE_REF_COST);
 9538   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9539   ins_encode %{
 9540     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9541   %}
 9542   ins_pipe(pipe_serial);
 9543 %}
 9544 
 9545 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9546   match(Set prev (GetAndSetN mem newv));
 9547   ins_cost(2 * VOLATILE_REF_COST);
 9548   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9549   ins_encode %{
 9550     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9551   %}
 9552   ins_pipe(pipe_serial);
 9553 %}
 9554 
 9555 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9556   predicate(n->as_LoadStore()->barrier_data() == 0);
 9557   match(Set prev (GetAndSetP mem newv));
 9558   ins_cost(2 * VOLATILE_REF_COST);
 9559   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9560   ins_encode %{
 9561     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9562   %}
 9563   ins_pipe(pipe_serial);
 9564 %}
 9565 
 9566 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9567   predicate(needs_acquiring_load_exclusive(n));
 9568   match(Set prev (GetAndSetI mem newv));
 9569   ins_cost(VOLATILE_REF_COST);
 9570   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9571   ins_encode %{
 9572     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9573   %}
 9574   ins_pipe(pipe_serial);
 9575 %}
 9576 
 9577 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9578   predicate(needs_acquiring_load_exclusive(n));
 9579   match(Set prev (GetAndSetL mem newv));
 9580   ins_cost(VOLATILE_REF_COST);
 9581   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9582   ins_encode %{
 9583     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9584   %}
 9585   ins_pipe(pipe_serial);
 9586 %}
 9587 
 9588 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9589   predicate(needs_acquiring_load_exclusive(n));
 9590   match(Set prev (GetAndSetN mem newv));
 9591   ins_cost(VOLATILE_REF_COST);
 9592   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9593   ins_encode %{
 9594     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9595   %}
 9596   ins_pipe(pipe_serial);
 9597 %}
 9598 
 9599 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9600   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9601   match(Set prev (GetAndSetP mem newv));
 9602   ins_cost(VOLATILE_REF_COST);
 9603   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9604   ins_encode %{
 9605     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9606   %}
 9607   ins_pipe(pipe_serial);
 9608 %}
 9609 
 9610 
 9611 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9612   match(Set newval (GetAndAddL mem incr));
 9613   ins_cost(2 * VOLATILE_REF_COST + 1);
 9614   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9615   ins_encode %{
 9616     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9617   %}
 9618   ins_pipe(pipe_serial);
 9619 %}
 9620 
 9621 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9622   predicate(n->as_LoadStore()->result_not_used());
 9623   match(Set dummy (GetAndAddL mem incr));
 9624   ins_cost(2 * VOLATILE_REF_COST);
 9625   format %{ "get_and_addL [$mem], $incr" %}
 9626   ins_encode %{
 9627     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9628   %}
 9629   ins_pipe(pipe_serial);
 9630 %}
 9631 
 9632 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9633   match(Set newval (GetAndAddL mem incr));
 9634   ins_cost(2 * VOLATILE_REF_COST + 1);
 9635   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9636   ins_encode %{
 9637     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9638   %}
 9639   ins_pipe(pipe_serial);
 9640 %}
 9641 
 9642 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9643   predicate(n->as_LoadStore()->result_not_used());
 9644   match(Set dummy (GetAndAddL mem incr));
 9645   ins_cost(2 * VOLATILE_REF_COST);
 9646   format %{ "get_and_addL [$mem], $incr" %}
 9647   ins_encode %{
 9648     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9649   %}
 9650   ins_pipe(pipe_serial);
 9651 %}
 9652 
 9653 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9654   match(Set newval (GetAndAddI mem incr));
 9655   ins_cost(2 * VOLATILE_REF_COST + 1);
 9656   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9657   ins_encode %{
 9658     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9659   %}
 9660   ins_pipe(pipe_serial);
 9661 %}
 9662 
 9663 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9664   predicate(n->as_LoadStore()->result_not_used());
 9665   match(Set dummy (GetAndAddI mem incr));
 9666   ins_cost(2 * VOLATILE_REF_COST);
 9667   format %{ "get_and_addI [$mem], $incr" %}
 9668   ins_encode %{
 9669     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9670   %}
 9671   ins_pipe(pipe_serial);
 9672 %}
 9673 
 9674 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9675   match(Set newval (GetAndAddI mem incr));
 9676   ins_cost(2 * VOLATILE_REF_COST + 1);
 9677   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9678   ins_encode %{
 9679     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9680   %}
 9681   ins_pipe(pipe_serial);
 9682 %}
 9683 
 9684 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9685   predicate(n->as_LoadStore()->result_not_used());
 9686   match(Set dummy (GetAndAddI mem incr));
 9687   ins_cost(2 * VOLATILE_REF_COST);
 9688   format %{ "get_and_addI [$mem], $incr" %}
 9689   ins_encode %{
 9690     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9691   %}
 9692   ins_pipe(pipe_serial);
 9693 %}
 9694 
 9695 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9696   predicate(needs_acquiring_load_exclusive(n));
 9697   match(Set newval (GetAndAddL mem incr));
 9698   ins_cost(VOLATILE_REF_COST + 1);
 9699   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9700   ins_encode %{
 9701     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9702   %}
 9703   ins_pipe(pipe_serial);
 9704 %}
 9705 
 9706 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9707   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9708   match(Set dummy (GetAndAddL mem incr));
 9709   ins_cost(VOLATILE_REF_COST);
 9710   format %{ "get_and_addL_acq [$mem], $incr" %}
 9711   ins_encode %{
 9712     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9713   %}
 9714   ins_pipe(pipe_serial);
 9715 %}
 9716 
 9717 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9718   predicate(needs_acquiring_load_exclusive(n));
 9719   match(Set newval (GetAndAddL mem incr));
 9720   ins_cost(VOLATILE_REF_COST + 1);
 9721   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9722   ins_encode %{
 9723     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9724   %}
 9725   ins_pipe(pipe_serial);
 9726 %}
 9727 
 9728 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9729   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9730   match(Set dummy (GetAndAddL mem incr));
 9731   ins_cost(VOLATILE_REF_COST);
 9732   format %{ "get_and_addL_acq [$mem], $incr" %}
 9733   ins_encode %{
 9734     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9735   %}
 9736   ins_pipe(pipe_serial);
 9737 %}
 9738 
 9739 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9740   predicate(needs_acquiring_load_exclusive(n));
 9741   match(Set newval (GetAndAddI mem incr));
 9742   ins_cost(VOLATILE_REF_COST + 1);
 9743   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9744   ins_encode %{
 9745     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9746   %}
 9747   ins_pipe(pipe_serial);
 9748 %}
 9749 
 9750 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9751   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9752   match(Set dummy (GetAndAddI mem incr));
 9753   ins_cost(VOLATILE_REF_COST);
 9754   format %{ "get_and_addI_acq [$mem], $incr" %}
 9755   ins_encode %{
 9756     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9757   %}
 9758   ins_pipe(pipe_serial);
 9759 %}
 9760 
 9761 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9762   predicate(needs_acquiring_load_exclusive(n));
 9763   match(Set newval (GetAndAddI mem incr));
 9764   ins_cost(VOLATILE_REF_COST + 1);
 9765   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9766   ins_encode %{
 9767     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9768   %}
 9769   ins_pipe(pipe_serial);
 9770 %}
 9771 
 9772 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9773   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9774   match(Set dummy (GetAndAddI mem incr));
 9775   ins_cost(VOLATILE_REF_COST);
 9776   format %{ "get_and_addI_acq [$mem], $incr" %}
 9777   ins_encode %{
 9778     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9779   %}
 9780   ins_pipe(pipe_serial);
 9781 %}
 9782 
 9783 // Manifest a CmpU result in an integer register.
 9784 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9785 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
 9786 %{
 9787   match(Set dst (CmpU3 src1 src2));
 9788   effect(KILL flags);
 9789 
 9790   ins_cost(INSN_COST * 3);
 9791   format %{
 9792       "cmpw $src1, $src2\n\t"
 9793       "csetw $dst, ne\n\t"
 9794       "cnegw $dst, lo\t# CmpU3(reg)"
 9795   %}
 9796   ins_encode %{
 9797     __ cmpw($src1$$Register, $src2$$Register);
 9798     __ csetw($dst$$Register, Assembler::NE);
 9799     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9800   %}
 9801 
 9802   ins_pipe(pipe_class_default);
 9803 %}
 9804 
 9805 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
 9806 %{
 9807   match(Set dst (CmpU3 src1 src2));
 9808   effect(KILL flags);
 9809 
 9810   ins_cost(INSN_COST * 3);
 9811   format %{
 9812       "subsw zr, $src1, $src2\n\t"
 9813       "csetw $dst, ne\n\t"
 9814       "cnegw $dst, lo\t# CmpU3(imm)"
 9815   %}
 9816   ins_encode %{
 9817     __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
 9818     __ csetw($dst$$Register, Assembler::NE);
 9819     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9820   %}
 9821 
 9822   ins_pipe(pipe_class_default);
 9823 %}
 9824 
 9825 // Manifest a CmpUL result in an integer register.
 9826 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9827 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9828 %{
 9829   match(Set dst (CmpUL3 src1 src2));
 9830   effect(KILL flags);
 9831 
 9832   ins_cost(INSN_COST * 3);
 9833   format %{
 9834       "cmp $src1, $src2\n\t"
 9835       "csetw $dst, ne\n\t"
 9836       "cnegw $dst, lo\t# CmpUL3(reg)"
 9837   %}
 9838   ins_encode %{
 9839     __ cmp($src1$$Register, $src2$$Register);
 9840     __ csetw($dst$$Register, Assembler::NE);
 9841     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9842   %}
 9843 
 9844   ins_pipe(pipe_class_default);
 9845 %}
 9846 
 9847 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9848 %{
 9849   match(Set dst (CmpUL3 src1 src2));
 9850   effect(KILL flags);
 9851 
 9852   ins_cost(INSN_COST * 3);
 9853   format %{
 9854       "subs zr, $src1, $src2\n\t"
 9855       "csetw $dst, ne\n\t"
 9856       "cnegw $dst, lo\t# CmpUL3(imm)"
 9857   %}
 9858   ins_encode %{
 9859     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9860     __ csetw($dst$$Register, Assembler::NE);
 9861     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9862   %}
 9863 
 9864   ins_pipe(pipe_class_default);
 9865 %}
 9866 
 9867 // Manifest a CmpL result in an integer register.
 9868 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9869 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9870 %{
 9871   match(Set dst (CmpL3 src1 src2));
 9872   effect(KILL flags);
 9873 
 9874   ins_cost(INSN_COST * 3);
 9875   format %{
 9876       "cmp $src1, $src2\n\t"
 9877       "csetw $dst, ne\n\t"
 9878       "cnegw $dst, lt\t# CmpL3(reg)"
 9879   %}
 9880   ins_encode %{
 9881     __ cmp($src1$$Register, $src2$$Register);
 9882     __ csetw($dst$$Register, Assembler::NE);
 9883     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9884   %}
 9885 
 9886   ins_pipe(pipe_class_default);
 9887 %}
 9888 
 9889 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9890 %{
 9891   match(Set dst (CmpL3 src1 src2));
 9892   effect(KILL flags);
 9893 
 9894   ins_cost(INSN_COST * 3);
 9895   format %{
 9896       "subs zr, $src1, $src2\n\t"
 9897       "csetw $dst, ne\n\t"
 9898       "cnegw $dst, lt\t# CmpL3(imm)"
 9899   %}
 9900   ins_encode %{
 9901     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9902     __ csetw($dst$$Register, Assembler::NE);
 9903     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9904   %}
 9905 
 9906   ins_pipe(pipe_class_default);
 9907 %}
 9908 
 9909 // ============================================================================
 9910 // Conditional Move Instructions
 9911 
 9912 // n.b. we have identical rules for both a signed compare op (cmpOp)
 9913 // and an unsigned compare op (cmpOpU). it would be nice if we could
 9914 // define an op class which merged both inputs and use it to type the
 9915 // argument to a single rule. unfortunatelyt his fails because the
 9916 // opclass does not live up to the COND_INTER interface of its
 9917 // component operands. When the generic code tries to negate the
 9918 // operand it ends up running the generci Machoper::negate method
 9919 // which throws a ShouldNotHappen. So, we have to provide two flavours
 9920 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
 9921 
 9922 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9923   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9924 
 9925   ins_cost(INSN_COST * 2);
 9926   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
 9927 
 9928   ins_encode %{
 9929     __ cselw(as_Register($dst$$reg),
 9930              as_Register($src2$$reg),
 9931              as_Register($src1$$reg),
 9932              (Assembler::Condition)$cmp$$cmpcode);
 9933   %}
 9934 
 9935   ins_pipe(icond_reg_reg);
 9936 %}
 9937 
 9938 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9939   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9940 
 9941   ins_cost(INSN_COST * 2);
 9942   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
 9943 
 9944   ins_encode %{
 9945     __ cselw(as_Register($dst$$reg),
 9946              as_Register($src2$$reg),
 9947              as_Register($src1$$reg),
 9948              (Assembler::Condition)$cmp$$cmpcode);
 9949   %}
 9950 
 9951   ins_pipe(icond_reg_reg);
 9952 %}
 9953 
 9954 // special cases where one arg is zero
 9955 
 9956 // n.b. this is selected in preference to the rule above because it
 9957 // avoids loading constant 0 into a source register
 9958 
 9959 // TODO
 9960 // we ought only to be able to cull one of these variants as the ideal
 9961 // transforms ought always to order the zero consistently (to left/right?)
 9962 
 9963 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9964   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9965 
 9966   ins_cost(INSN_COST * 2);
 9967   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
 9968 
 9969   ins_encode %{
 9970     __ cselw(as_Register($dst$$reg),
 9971              as_Register($src$$reg),
 9972              zr,
 9973              (Assembler::Condition)$cmp$$cmpcode);
 9974   %}
 9975 
 9976   ins_pipe(icond_reg);
 9977 %}
 9978 
 9979 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9980   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9981 
 9982   ins_cost(INSN_COST * 2);
 9983   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
 9984 
 9985   ins_encode %{
 9986     __ cselw(as_Register($dst$$reg),
 9987              as_Register($src$$reg),
 9988              zr,
 9989              (Assembler::Condition)$cmp$$cmpcode);
 9990   %}
 9991 
 9992   ins_pipe(icond_reg);
 9993 %}
 9994 
 9995 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9996   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9997 
 9998   ins_cost(INSN_COST * 2);
 9999   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
10000 
10001   ins_encode %{
10002     __ cselw(as_Register($dst$$reg),
10003              zr,
10004              as_Register($src$$reg),
10005              (Assembler::Condition)$cmp$$cmpcode);
10006   %}
10007 
10008   ins_pipe(icond_reg);
10009 %}
10010 
10011 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10012   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10013 
10014   ins_cost(INSN_COST * 2);
10015   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
10016 
10017   ins_encode %{
10018     __ cselw(as_Register($dst$$reg),
10019              zr,
10020              as_Register($src$$reg),
10021              (Assembler::Condition)$cmp$$cmpcode);
10022   %}
10023 
10024   ins_pipe(icond_reg);
10025 %}
10026 
10027 // special case for creating a boolean 0 or 1
10028 
10029 // n.b. this is selected in preference to the rule above because it
10030 // avoids loading constants 0 and 1 into a source register
10031 
10032 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10033   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10034 
10035   ins_cost(INSN_COST * 2);
10036   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
10037 
10038   ins_encode %{
10039     // equivalently
10040     // cset(as_Register($dst$$reg),
10041     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10042     __ csincw(as_Register($dst$$reg),
10043              zr,
10044              zr,
10045              (Assembler::Condition)$cmp$$cmpcode);
10046   %}
10047 
10048   ins_pipe(icond_none);
10049 %}
10050 
10051 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10052   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10053 
10054   ins_cost(INSN_COST * 2);
10055   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
10056 
10057   ins_encode %{
10058     // equivalently
10059     // cset(as_Register($dst$$reg),
10060     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10061     __ csincw(as_Register($dst$$reg),
10062              zr,
10063              zr,
10064              (Assembler::Condition)$cmp$$cmpcode);
10065   %}
10066 
10067   ins_pipe(icond_none);
10068 %}
10069 
10070 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10071   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10072 
10073   ins_cost(INSN_COST * 2);
10074   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
10075 
10076   ins_encode %{
10077     __ csel(as_Register($dst$$reg),
10078             as_Register($src2$$reg),
10079             as_Register($src1$$reg),
10080             (Assembler::Condition)$cmp$$cmpcode);
10081   %}
10082 
10083   ins_pipe(icond_reg_reg);
10084 %}
10085 
10086 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10087   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10088 
10089   ins_cost(INSN_COST * 2);
10090   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
10091 
10092   ins_encode %{
10093     __ csel(as_Register($dst$$reg),
10094             as_Register($src2$$reg),
10095             as_Register($src1$$reg),
10096             (Assembler::Condition)$cmp$$cmpcode);
10097   %}
10098 
10099   ins_pipe(icond_reg_reg);
10100 %}
10101 
10102 // special cases where one arg is zero
10103 
10104 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10105   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10106 
10107   ins_cost(INSN_COST * 2);
10108   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
10109 
10110   ins_encode %{
10111     __ csel(as_Register($dst$$reg),
10112             zr,
10113             as_Register($src$$reg),
10114             (Assembler::Condition)$cmp$$cmpcode);
10115   %}
10116 
10117   ins_pipe(icond_reg);
10118 %}
10119 
10120 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10121   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10122 
10123   ins_cost(INSN_COST * 2);
10124   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10125 
10126   ins_encode %{
10127     __ csel(as_Register($dst$$reg),
10128             zr,
10129             as_Register($src$$reg),
10130             (Assembler::Condition)$cmp$$cmpcode);
10131   %}
10132 
10133   ins_pipe(icond_reg);
10134 %}
10135 
10136 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10137   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10138 
10139   ins_cost(INSN_COST * 2);
10140   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10141 
10142   ins_encode %{
10143     __ csel(as_Register($dst$$reg),
10144             as_Register($src$$reg),
10145             zr,
10146             (Assembler::Condition)$cmp$$cmpcode);
10147   %}
10148 
10149   ins_pipe(icond_reg);
10150 %}
10151 
10152 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10153   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10154 
10155   ins_cost(INSN_COST * 2);
10156   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10157 
10158   ins_encode %{
10159     __ csel(as_Register($dst$$reg),
10160             as_Register($src$$reg),
10161             zr,
10162             (Assembler::Condition)$cmp$$cmpcode);
10163   %}
10164 
10165   ins_pipe(icond_reg);
10166 %}
10167 
10168 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10169   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10170 
10171   ins_cost(INSN_COST * 2);
10172   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10173 
10174   ins_encode %{
10175     __ csel(as_Register($dst$$reg),
10176             as_Register($src2$$reg),
10177             as_Register($src1$$reg),
10178             (Assembler::Condition)$cmp$$cmpcode);
10179   %}
10180 
10181   ins_pipe(icond_reg_reg);
10182 %}
10183 
10184 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10185   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10186 
10187   ins_cost(INSN_COST * 2);
10188   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10189 
10190   ins_encode %{
10191     __ csel(as_Register($dst$$reg),
10192             as_Register($src2$$reg),
10193             as_Register($src1$$reg),
10194             (Assembler::Condition)$cmp$$cmpcode);
10195   %}
10196 
10197   ins_pipe(icond_reg_reg);
10198 %}
10199 
10200 // special cases where one arg is zero
10201 
10202 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10203   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10204 
10205   ins_cost(INSN_COST * 2);
10206   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10207 
10208   ins_encode %{
10209     __ csel(as_Register($dst$$reg),
10210             zr,
10211             as_Register($src$$reg),
10212             (Assembler::Condition)$cmp$$cmpcode);
10213   %}
10214 
10215   ins_pipe(icond_reg);
10216 %}
10217 
10218 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10219   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10220 
10221   ins_cost(INSN_COST * 2);
10222   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10223 
10224   ins_encode %{
10225     __ csel(as_Register($dst$$reg),
10226             zr,
10227             as_Register($src$$reg),
10228             (Assembler::Condition)$cmp$$cmpcode);
10229   %}
10230 
10231   ins_pipe(icond_reg);
10232 %}
10233 
10234 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10235   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10236 
10237   ins_cost(INSN_COST * 2);
10238   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10239 
10240   ins_encode %{
10241     __ csel(as_Register($dst$$reg),
10242             as_Register($src$$reg),
10243             zr,
10244             (Assembler::Condition)$cmp$$cmpcode);
10245   %}
10246 
10247   ins_pipe(icond_reg);
10248 %}
10249 
10250 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10251   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10252 
10253   ins_cost(INSN_COST * 2);
10254   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10255 
10256   ins_encode %{
10257     __ csel(as_Register($dst$$reg),
10258             as_Register($src$$reg),
10259             zr,
10260             (Assembler::Condition)$cmp$$cmpcode);
10261   %}
10262 
10263   ins_pipe(icond_reg);
10264 %}
10265 
10266 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10267   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10268 
10269   ins_cost(INSN_COST * 2);
10270   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10271 
10272   ins_encode %{
10273     __ cselw(as_Register($dst$$reg),
10274              as_Register($src2$$reg),
10275              as_Register($src1$$reg),
10276              (Assembler::Condition)$cmp$$cmpcode);
10277   %}
10278 
10279   ins_pipe(icond_reg_reg);
10280 %}
10281 
10282 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10283   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10284 
10285   ins_cost(INSN_COST * 2);
10286   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10287 
10288   ins_encode %{
10289     __ cselw(as_Register($dst$$reg),
10290              as_Register($src2$$reg),
10291              as_Register($src1$$reg),
10292              (Assembler::Condition)$cmp$$cmpcode);
10293   %}
10294 
10295   ins_pipe(icond_reg_reg);
10296 %}
10297 
10298 // special cases where one arg is zero
10299 
10300 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10301   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10302 
10303   ins_cost(INSN_COST * 2);
10304   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10305 
10306   ins_encode %{
10307     __ cselw(as_Register($dst$$reg),
10308              zr,
10309              as_Register($src$$reg),
10310              (Assembler::Condition)$cmp$$cmpcode);
10311   %}
10312 
10313   ins_pipe(icond_reg);
10314 %}
10315 
10316 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10317   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10318 
10319   ins_cost(INSN_COST * 2);
10320   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10321 
10322   ins_encode %{
10323     __ cselw(as_Register($dst$$reg),
10324              zr,
10325              as_Register($src$$reg),
10326              (Assembler::Condition)$cmp$$cmpcode);
10327   %}
10328 
10329   ins_pipe(icond_reg);
10330 %}
10331 
10332 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10333   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10334 
10335   ins_cost(INSN_COST * 2);
10336   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10337 
10338   ins_encode %{
10339     __ cselw(as_Register($dst$$reg),
10340              as_Register($src$$reg),
10341              zr,
10342              (Assembler::Condition)$cmp$$cmpcode);
10343   %}
10344 
10345   ins_pipe(icond_reg);
10346 %}
10347 
10348 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10349   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10350 
10351   ins_cost(INSN_COST * 2);
10352   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10353 
10354   ins_encode %{
10355     __ cselw(as_Register($dst$$reg),
10356              as_Register($src$$reg),
10357              zr,
10358              (Assembler::Condition)$cmp$$cmpcode);
10359   %}
10360 
10361   ins_pipe(icond_reg);
10362 %}
10363 
10364 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10365 %{
10366   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10367 
10368   ins_cost(INSN_COST * 3);
10369 
10370   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10371   ins_encode %{
10372     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10373     __ fcsels(as_FloatRegister($dst$$reg),
10374               as_FloatRegister($src2$$reg),
10375               as_FloatRegister($src1$$reg),
10376               cond);
10377   %}
10378 
10379   ins_pipe(fp_cond_reg_reg_s);
10380 %}
10381 
10382 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10383 %{
10384   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10385 
10386   ins_cost(INSN_COST * 3);
10387 
10388   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10389   ins_encode %{
10390     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10391     __ fcsels(as_FloatRegister($dst$$reg),
10392               as_FloatRegister($src2$$reg),
10393               as_FloatRegister($src1$$reg),
10394               cond);
10395   %}
10396 
10397   ins_pipe(fp_cond_reg_reg_s);
10398 %}
10399 
10400 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10401 %{
10402   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10403 
10404   ins_cost(INSN_COST * 3);
10405 
10406   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10407   ins_encode %{
10408     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10409     __ fcseld(as_FloatRegister($dst$$reg),
10410               as_FloatRegister($src2$$reg),
10411               as_FloatRegister($src1$$reg),
10412               cond);
10413   %}
10414 
10415   ins_pipe(fp_cond_reg_reg_d);
10416 %}
10417 
10418 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10419 %{
10420   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10421 
10422   ins_cost(INSN_COST * 3);
10423 
10424   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10425   ins_encode %{
10426     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10427     __ fcseld(as_FloatRegister($dst$$reg),
10428               as_FloatRegister($src2$$reg),
10429               as_FloatRegister($src1$$reg),
10430               cond);
10431   %}
10432 
10433   ins_pipe(fp_cond_reg_reg_d);
10434 %}
10435 
10436 // ============================================================================
10437 // Arithmetic Instructions
10438 //
10439 
10440 // Integer Addition
10441 
10442 // TODO
10443 // these currently employ operations which do not set CR and hence are
10444 // not flagged as killing CR but we would like to isolate the cases
10445 // where we want to set flags from those where we don't. need to work
10446 // out how to do that.
10447 
10448 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10449   match(Set dst (AddI src1 src2));
10450 
10451   ins_cost(INSN_COST);
10452   format %{ "addw  $dst, $src1, $src2" %}
10453 
10454   ins_encode %{
10455     __ addw(as_Register($dst$$reg),
10456             as_Register($src1$$reg),
10457             as_Register($src2$$reg));
10458   %}
10459 
10460   ins_pipe(ialu_reg_reg);
10461 %}
10462 
10463 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10464   match(Set dst (AddI src1 src2));
10465 
10466   ins_cost(INSN_COST);
10467   format %{ "addw $dst, $src1, $src2" %}
10468 
10469   // use opcode to indicate that this is an add not a sub
10470   opcode(0x0);
10471 
10472   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10473 
10474   ins_pipe(ialu_reg_imm);
10475 %}
10476 
10477 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10478   match(Set dst (AddI (ConvL2I src1) src2));
10479 
10480   ins_cost(INSN_COST);
10481   format %{ "addw $dst, $src1, $src2" %}
10482 
10483   // use opcode to indicate that this is an add not a sub
10484   opcode(0x0);
10485 
10486   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10487 
10488   ins_pipe(ialu_reg_imm);
10489 %}
10490 
10491 // Pointer Addition
10492 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10493   match(Set dst (AddP src1 src2));
10494 
10495   ins_cost(INSN_COST);
10496   format %{ "add $dst, $src1, $src2\t# ptr" %}
10497 
10498   ins_encode %{
10499     __ add(as_Register($dst$$reg),
10500            as_Register($src1$$reg),
10501            as_Register($src2$$reg));
10502   %}
10503 
10504   ins_pipe(ialu_reg_reg);
10505 %}
10506 
10507 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10508   match(Set dst (AddP src1 (ConvI2L src2)));
10509 
10510   ins_cost(1.9 * INSN_COST);
10511   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10512 
10513   ins_encode %{
10514     __ add(as_Register($dst$$reg),
10515            as_Register($src1$$reg),
10516            as_Register($src2$$reg), ext::sxtw);
10517   %}
10518 
10519   ins_pipe(ialu_reg_reg);
10520 %}
10521 
10522 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10523   match(Set dst (AddP src1 (LShiftL src2 scale)));
10524 
10525   ins_cost(1.9 * INSN_COST);
10526   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10527 
10528   ins_encode %{
10529     __ lea(as_Register($dst$$reg),
10530            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10531                    Address::lsl($scale$$constant)));
10532   %}
10533 
10534   ins_pipe(ialu_reg_reg_shift);
10535 %}
10536 
10537 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10538   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10539 
10540   ins_cost(1.9 * INSN_COST);
10541   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10542 
10543   ins_encode %{
10544     __ lea(as_Register($dst$$reg),
10545            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10546                    Address::sxtw($scale$$constant)));
10547   %}
10548 
10549   ins_pipe(ialu_reg_reg_shift);
10550 %}
10551 
10552 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10553   match(Set dst (LShiftL (ConvI2L src) scale));
10554 
10555   ins_cost(INSN_COST);
10556   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10557 
10558   ins_encode %{
10559     __ sbfiz(as_Register($dst$$reg),
10560           as_Register($src$$reg),
10561           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10562   %}
10563 
10564   ins_pipe(ialu_reg_shift);
10565 %}
10566 
10567 // Pointer Immediate Addition
10568 // n.b. this needs to be more expensive than using an indirect memory
10569 // operand
10570 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10571   match(Set dst (AddP src1 src2));
10572 
10573   ins_cost(INSN_COST);
10574   format %{ "add $dst, $src1, $src2\t# ptr" %}
10575 
10576   // use opcode to indicate that this is an add not a sub
10577   opcode(0x0);
10578 
10579   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10580 
10581   ins_pipe(ialu_reg_imm);
10582 %}
10583 
10584 // Long Addition
10585 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10586 
10587   match(Set dst (AddL src1 src2));
10588 
10589   ins_cost(INSN_COST);
10590   format %{ "add  $dst, $src1, $src2" %}
10591 
10592   ins_encode %{
10593     __ add(as_Register($dst$$reg),
10594            as_Register($src1$$reg),
10595            as_Register($src2$$reg));
10596   %}
10597 
10598   ins_pipe(ialu_reg_reg);
10599 %}
10600 
10601 // No constant pool entries requiredLong Immediate Addition.
10602 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10603   match(Set dst (AddL src1 src2));
10604 
10605   ins_cost(INSN_COST);
10606   format %{ "add $dst, $src1, $src2" %}
10607 
10608   // use opcode to indicate that this is an add not a sub
10609   opcode(0x0);
10610 
10611   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10612 
10613   ins_pipe(ialu_reg_imm);
10614 %}
10615 
10616 // Integer Subtraction
10617 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10618   match(Set dst (SubI src1 src2));
10619 
10620   ins_cost(INSN_COST);
10621   format %{ "subw  $dst, $src1, $src2" %}
10622 
10623   ins_encode %{
10624     __ subw(as_Register($dst$$reg),
10625             as_Register($src1$$reg),
10626             as_Register($src2$$reg));
10627   %}
10628 
10629   ins_pipe(ialu_reg_reg);
10630 %}
10631 
10632 // Immediate Subtraction
10633 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10634   match(Set dst (SubI src1 src2));
10635 
10636   ins_cost(INSN_COST);
10637   format %{ "subw $dst, $src1, $src2" %}
10638 
10639   // use opcode to indicate that this is a sub not an add
10640   opcode(0x1);
10641 
10642   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10643 
10644   ins_pipe(ialu_reg_imm);
10645 %}
10646 
10647 // Long Subtraction
10648 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10649 
10650   match(Set dst (SubL src1 src2));
10651 
10652   ins_cost(INSN_COST);
10653   format %{ "sub  $dst, $src1, $src2" %}
10654 
10655   ins_encode %{
10656     __ sub(as_Register($dst$$reg),
10657            as_Register($src1$$reg),
10658            as_Register($src2$$reg));
10659   %}
10660 
10661   ins_pipe(ialu_reg_reg);
10662 %}
10663 
10664 // No constant pool entries requiredLong Immediate Subtraction.
10665 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10666   match(Set dst (SubL src1 src2));
10667 
10668   ins_cost(INSN_COST);
10669   format %{ "sub$dst, $src1, $src2" %}
10670 
10671   // use opcode to indicate that this is a sub not an add
10672   opcode(0x1);
10673 
10674   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10675 
10676   ins_pipe(ialu_reg_imm);
10677 %}
10678 
10679 // Integer Negation (special case for sub)
10680 
10681 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10682   match(Set dst (SubI zero src));
10683 
10684   ins_cost(INSN_COST);
10685   format %{ "negw $dst, $src\t# int" %}
10686 
10687   ins_encode %{
10688     __ negw(as_Register($dst$$reg),
10689             as_Register($src$$reg));
10690   %}
10691 
10692   ins_pipe(ialu_reg);
10693 %}
10694 
10695 // Long Negation
10696 
10697 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10698   match(Set dst (SubL zero src));
10699 
10700   ins_cost(INSN_COST);
10701   format %{ "neg $dst, $src\t# long" %}
10702 
10703   ins_encode %{
10704     __ neg(as_Register($dst$$reg),
10705            as_Register($src$$reg));
10706   %}
10707 
10708   ins_pipe(ialu_reg);
10709 %}
10710 
10711 // Integer Multiply
10712 
10713 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10714   match(Set dst (MulI src1 src2));
10715 
10716   ins_cost(INSN_COST * 3);
10717   format %{ "mulw  $dst, $src1, $src2" %}
10718 
10719   ins_encode %{
10720     __ mulw(as_Register($dst$$reg),
10721             as_Register($src1$$reg),
10722             as_Register($src2$$reg));
10723   %}
10724 
10725   ins_pipe(imul_reg_reg);
10726 %}
10727 
10728 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10729   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10730 
10731   ins_cost(INSN_COST * 3);
10732   format %{ "smull  $dst, $src1, $src2" %}
10733 
10734   ins_encode %{
10735     __ smull(as_Register($dst$$reg),
10736              as_Register($src1$$reg),
10737              as_Register($src2$$reg));
10738   %}
10739 
10740   ins_pipe(imul_reg_reg);
10741 %}
10742 
10743 // Long Multiply
10744 
10745 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10746   match(Set dst (MulL src1 src2));
10747 
10748   ins_cost(INSN_COST * 5);
10749   format %{ "mul  $dst, $src1, $src2" %}
10750 
10751   ins_encode %{
10752     __ mul(as_Register($dst$$reg),
10753            as_Register($src1$$reg),
10754            as_Register($src2$$reg));
10755   %}
10756 
10757   ins_pipe(lmul_reg_reg);
10758 %}
10759 
10760 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10761 %{
10762   match(Set dst (MulHiL src1 src2));
10763 
10764   ins_cost(INSN_COST * 7);
10765   format %{ "smulh   $dst, $src1, $src2\t# mulhi" %}
10766 
10767   ins_encode %{
10768     __ smulh(as_Register($dst$$reg),
10769              as_Register($src1$$reg),
10770              as_Register($src2$$reg));
10771   %}
10772 
10773   ins_pipe(lmul_reg_reg);
10774 %}
10775 
10776 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10777 %{
10778   match(Set dst (UMulHiL src1 src2));
10779 
10780   ins_cost(INSN_COST * 7);
10781   format %{ "umulh   $dst, $src1, $src2\t# umulhi" %}
10782 
10783   ins_encode %{
10784     __ umulh(as_Register($dst$$reg),
10785              as_Register($src1$$reg),
10786              as_Register($src2$$reg));
10787   %}
10788 
10789   ins_pipe(lmul_reg_reg);
10790 %}
10791 
10792 // Combined Integer Multiply & Add/Sub
10793 
10794 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10795   match(Set dst (AddI src3 (MulI src1 src2)));
10796 
10797   ins_cost(INSN_COST * 3);
10798   format %{ "madd  $dst, $src1, $src2, $src3" %}
10799 
10800   ins_encode %{
10801     __ maddw(as_Register($dst$$reg),
10802              as_Register($src1$$reg),
10803              as_Register($src2$$reg),
10804              as_Register($src3$$reg));
10805   %}
10806 
10807   ins_pipe(imac_reg_reg);
10808 %}
10809 
10810 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10811   match(Set dst (SubI src3 (MulI src1 src2)));
10812 
10813   ins_cost(INSN_COST * 3);
10814   format %{ "msub  $dst, $src1, $src2, $src3" %}
10815 
10816   ins_encode %{
10817     __ msubw(as_Register($dst$$reg),
10818              as_Register($src1$$reg),
10819              as_Register($src2$$reg),
10820              as_Register($src3$$reg));
10821   %}
10822 
10823   ins_pipe(imac_reg_reg);
10824 %}
10825 
10826 // Combined Integer Multiply & Neg
10827 
10828 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10829   match(Set dst (MulI (SubI zero src1) src2));
10830 
10831   ins_cost(INSN_COST * 3);
10832   format %{ "mneg  $dst, $src1, $src2" %}
10833 
10834   ins_encode %{
10835     __ mnegw(as_Register($dst$$reg),
10836              as_Register($src1$$reg),
10837              as_Register($src2$$reg));
10838   %}
10839 
10840   ins_pipe(imac_reg_reg);
10841 %}
10842 
10843 // Combined Long Multiply & Add/Sub
10844 
10845 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10846   match(Set dst (AddL src3 (MulL src1 src2)));
10847 
10848   ins_cost(INSN_COST * 5);
10849   format %{ "madd  $dst, $src1, $src2, $src3" %}
10850 
10851   ins_encode %{
10852     __ madd(as_Register($dst$$reg),
10853             as_Register($src1$$reg),
10854             as_Register($src2$$reg),
10855             as_Register($src3$$reg));
10856   %}
10857 
10858   ins_pipe(lmac_reg_reg);
10859 %}
10860 
10861 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10862   match(Set dst (SubL src3 (MulL src1 src2)));
10863 
10864   ins_cost(INSN_COST * 5);
10865   format %{ "msub  $dst, $src1, $src2, $src3" %}
10866 
10867   ins_encode %{
10868     __ msub(as_Register($dst$$reg),
10869             as_Register($src1$$reg),
10870             as_Register($src2$$reg),
10871             as_Register($src3$$reg));
10872   %}
10873 
10874   ins_pipe(lmac_reg_reg);
10875 %}
10876 
10877 // Combined Long Multiply & Neg
10878 
10879 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10880   match(Set dst (MulL (SubL zero src1) src2));
10881 
10882   ins_cost(INSN_COST * 5);
10883   format %{ "mneg  $dst, $src1, $src2" %}
10884 
10885   ins_encode %{
10886     __ mneg(as_Register($dst$$reg),
10887             as_Register($src1$$reg),
10888             as_Register($src2$$reg));
10889   %}
10890 
10891   ins_pipe(lmac_reg_reg);
10892 %}
10893 
10894 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10895 
10896 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10897   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10898 
10899   ins_cost(INSN_COST * 3);
10900   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10901 
10902   ins_encode %{
10903     __ smaddl(as_Register($dst$$reg),
10904               as_Register($src1$$reg),
10905               as_Register($src2$$reg),
10906               as_Register($src3$$reg));
10907   %}
10908 
10909   ins_pipe(imac_reg_reg);
10910 %}
10911 
10912 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10913   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10914 
10915   ins_cost(INSN_COST * 3);
10916   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10917 
10918   ins_encode %{
10919     __ smsubl(as_Register($dst$$reg),
10920               as_Register($src1$$reg),
10921               as_Register($src2$$reg),
10922               as_Register($src3$$reg));
10923   %}
10924 
10925   ins_pipe(imac_reg_reg);
10926 %}
10927 
10928 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10929   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10930 
10931   ins_cost(INSN_COST * 3);
10932   format %{ "smnegl  $dst, $src1, $src2" %}
10933 
10934   ins_encode %{
10935     __ smnegl(as_Register($dst$$reg),
10936               as_Register($src1$$reg),
10937               as_Register($src2$$reg));
10938   %}
10939 
10940   ins_pipe(imac_reg_reg);
10941 %}
10942 
10943 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10944 
10945 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10946   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10947 
10948   ins_cost(INSN_COST * 5);
10949   format %{ "mulw  rscratch1, $src1, $src2\n\t"
10950             "maddw $dst, $src3, $src4, rscratch1" %}
10951 
10952   ins_encode %{
10953     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10954     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10955 
10956   ins_pipe(imac_reg_reg);
10957 %}
10958 
10959 // Integer Divide
10960 
10961 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10962   match(Set dst (DivI src1 src2));
10963 
10964   ins_cost(INSN_COST * 19);
10965   format %{ "sdivw  $dst, $src1, $src2" %}
10966 
10967   ins_encode(aarch64_enc_divw(dst, src1, src2));
10968   ins_pipe(idiv_reg_reg);
10969 %}
10970 
10971 // Long Divide
10972 
10973 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10974   match(Set dst (DivL src1 src2));
10975 
10976   ins_cost(INSN_COST * 35);
10977   format %{ "sdiv   $dst, $src1, $src2" %}
10978 
10979   ins_encode(aarch64_enc_div(dst, src1, src2));
10980   ins_pipe(ldiv_reg_reg);
10981 %}
10982 
10983 // Integer Remainder
10984 
10985 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10986   match(Set dst (ModI src1 src2));
10987 
10988   ins_cost(INSN_COST * 22);
10989   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
10990             "msubw  $dst, rscratch1, $src2, $src1" %}
10991 
10992   ins_encode(aarch64_enc_modw(dst, src1, src2));
10993   ins_pipe(idiv_reg_reg);
10994 %}
10995 
10996 // Long Remainder
10997 
10998 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10999   match(Set dst (ModL src1 src2));
11000 
11001   ins_cost(INSN_COST * 38);
11002   format %{ "sdiv   rscratch1, $src1, $src2\n"
11003             "msub   $dst, rscratch1, $src2, $src1" %}
11004 
11005   ins_encode(aarch64_enc_mod(dst, src1, src2));
11006   ins_pipe(ldiv_reg_reg);
11007 %}
11008 
11009 // Unsigned Integer Divide
11010 
11011 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11012   match(Set dst (UDivI src1 src2));
11013 
11014   ins_cost(INSN_COST * 19);
11015   format %{ "udivw  $dst, $src1, $src2" %}
11016 
11017   ins_encode %{
11018     __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
11019   %}
11020 
11021   ins_pipe(idiv_reg_reg);
11022 %}
11023 
11024 //  Unsigned Long Divide
11025 
11026 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11027   match(Set dst (UDivL src1 src2));
11028 
11029   ins_cost(INSN_COST * 35);
11030   format %{ "udiv   $dst, $src1, $src2" %}
11031 
11032   ins_encode %{
11033     __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
11034   %}
11035 
11036   ins_pipe(ldiv_reg_reg);
11037 %}
11038 
11039 // Unsigned Integer Remainder
11040 
11041 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11042   match(Set dst (UModI src1 src2));
11043 
11044   ins_cost(INSN_COST * 22);
11045   format %{ "udivw  rscratch1, $src1, $src2\n\t"
11046             "msubw  $dst, rscratch1, $src2, $src1" %}
11047 
11048   ins_encode %{
11049     __ udivw(rscratch1, $src1$$Register, $src2$$Register);
11050     __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
11051   %}
11052 
11053   ins_pipe(idiv_reg_reg);
11054 %}
11055 
11056 // Unsigned Long Remainder
11057 
11058 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11059   match(Set dst (UModL src1 src2));
11060 
11061   ins_cost(INSN_COST * 38);
11062   format %{ "udiv   rscratch1, $src1, $src2\n"
11063             "msub   $dst, rscratch1, $src2, $src1" %}
11064 
11065   ins_encode %{
11066     __ udiv(rscratch1, $src1$$Register, $src2$$Register);
11067     __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
11068   %}
11069 
11070   ins_pipe(ldiv_reg_reg);
11071 %}
11072 
11073 // Integer Shifts
11074 
11075 // Shift Left Register
11076 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11077   match(Set dst (LShiftI src1 src2));
11078 
11079   ins_cost(INSN_COST * 2);
11080   format %{ "lslvw  $dst, $src1, $src2" %}
11081 
11082   ins_encode %{
11083     __ lslvw(as_Register($dst$$reg),
11084              as_Register($src1$$reg),
11085              as_Register($src2$$reg));
11086   %}
11087 
11088   ins_pipe(ialu_reg_reg_vshift);
11089 %}
11090 
11091 // Shift Left Immediate
11092 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11093   match(Set dst (LShiftI src1 src2));
11094 
11095   ins_cost(INSN_COST);
11096   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11097 
11098   ins_encode %{
11099     __ lslw(as_Register($dst$$reg),
11100             as_Register($src1$$reg),
11101             $src2$$constant & 0x1f);
11102   %}
11103 
11104   ins_pipe(ialu_reg_shift);
11105 %}
11106 
11107 // Shift Right Logical Register
11108 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11109   match(Set dst (URShiftI src1 src2));
11110 
11111   ins_cost(INSN_COST * 2);
11112   format %{ "lsrvw  $dst, $src1, $src2" %}
11113 
11114   ins_encode %{
11115     __ lsrvw(as_Register($dst$$reg),
11116              as_Register($src1$$reg),
11117              as_Register($src2$$reg));
11118   %}
11119 
11120   ins_pipe(ialu_reg_reg_vshift);
11121 %}
11122 
11123 // Shift Right Logical Immediate
11124 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11125   match(Set dst (URShiftI src1 src2));
11126 
11127   ins_cost(INSN_COST);
11128   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11129 
11130   ins_encode %{
11131     __ lsrw(as_Register($dst$$reg),
11132             as_Register($src1$$reg),
11133             $src2$$constant & 0x1f);
11134   %}
11135 
11136   ins_pipe(ialu_reg_shift);
11137 %}
11138 
11139 // Shift Right Arithmetic Register
11140 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11141   match(Set dst (RShiftI src1 src2));
11142 
11143   ins_cost(INSN_COST * 2);
11144   format %{ "asrvw  $dst, $src1, $src2" %}
11145 
11146   ins_encode %{
11147     __ asrvw(as_Register($dst$$reg),
11148              as_Register($src1$$reg),
11149              as_Register($src2$$reg));
11150   %}
11151 
11152   ins_pipe(ialu_reg_reg_vshift);
11153 %}
11154 
11155 // Shift Right Arithmetic Immediate
11156 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11157   match(Set dst (RShiftI src1 src2));
11158 
11159   ins_cost(INSN_COST);
11160   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11161 
11162   ins_encode %{
11163     __ asrw(as_Register($dst$$reg),
11164             as_Register($src1$$reg),
11165             $src2$$constant & 0x1f);
11166   %}
11167 
11168   ins_pipe(ialu_reg_shift);
11169 %}
11170 
11171 // Combined Int Mask and Right Shift (using UBFM)
11172 // TODO
11173 
11174 // Long Shifts
11175 
11176 // Shift Left Register
11177 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11178   match(Set dst (LShiftL src1 src2));
11179 
11180   ins_cost(INSN_COST * 2);
11181   format %{ "lslv  $dst, $src1, $src2" %}
11182 
11183   ins_encode %{
11184     __ lslv(as_Register($dst$$reg),
11185             as_Register($src1$$reg),
11186             as_Register($src2$$reg));
11187   %}
11188 
11189   ins_pipe(ialu_reg_reg_vshift);
11190 %}
11191 
11192 // Shift Left Immediate
11193 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11194   match(Set dst (LShiftL src1 src2));
11195 
11196   ins_cost(INSN_COST);
11197   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11198 
11199   ins_encode %{
11200     __ lsl(as_Register($dst$$reg),
11201             as_Register($src1$$reg),
11202             $src2$$constant & 0x3f);
11203   %}
11204 
11205   ins_pipe(ialu_reg_shift);
11206 %}
11207 
11208 // Shift Right Logical Register
11209 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11210   match(Set dst (URShiftL src1 src2));
11211 
11212   ins_cost(INSN_COST * 2);
11213   format %{ "lsrv  $dst, $src1, $src2" %}
11214 
11215   ins_encode %{
11216     __ lsrv(as_Register($dst$$reg),
11217             as_Register($src1$$reg),
11218             as_Register($src2$$reg));
11219   %}
11220 
11221   ins_pipe(ialu_reg_reg_vshift);
11222 %}
11223 
11224 // Shift Right Logical Immediate
11225 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11226   match(Set dst (URShiftL src1 src2));
11227 
11228   ins_cost(INSN_COST);
11229   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11230 
11231   ins_encode %{
11232     __ lsr(as_Register($dst$$reg),
11233            as_Register($src1$$reg),
11234            $src2$$constant & 0x3f);
11235   %}
11236 
11237   ins_pipe(ialu_reg_shift);
11238 %}
11239 
11240 // A special-case pattern for card table stores.
11241 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11242   match(Set dst (URShiftL (CastP2X src1) src2));
11243 
11244   ins_cost(INSN_COST);
11245   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11246 
11247   ins_encode %{
11248     __ lsr(as_Register($dst$$reg),
11249            as_Register($src1$$reg),
11250            $src2$$constant & 0x3f);
11251   %}
11252 
11253   ins_pipe(ialu_reg_shift);
11254 %}
11255 
11256 // Shift Right Arithmetic Register
11257 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11258   match(Set dst (RShiftL src1 src2));
11259 
11260   ins_cost(INSN_COST * 2);
11261   format %{ "asrv  $dst, $src1, $src2" %}
11262 
11263   ins_encode %{
11264     __ asrv(as_Register($dst$$reg),
11265             as_Register($src1$$reg),
11266             as_Register($src2$$reg));
11267   %}
11268 
11269   ins_pipe(ialu_reg_reg_vshift);
11270 %}
11271 
11272 // Shift Right Arithmetic Immediate
11273 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11274   match(Set dst (RShiftL src1 src2));
11275 
11276   ins_cost(INSN_COST);
11277   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11278 
11279   ins_encode %{
11280     __ asr(as_Register($dst$$reg),
11281            as_Register($src1$$reg),
11282            $src2$$constant & 0x3f);
11283   %}
11284 
11285   ins_pipe(ialu_reg_shift);
11286 %}
11287 
11288 // BEGIN This section of the file is automatically generated. Do not edit --------------
11289 // This section is generated from aarch64_ad.m4
11290 
11291 // This pattern is automatically generated from aarch64_ad.m4
11292 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11293 instruct regL_not_reg(iRegLNoSp dst,
11294                          iRegL src1, immL_M1 m1,
11295                          rFlagsReg cr) %{
11296   match(Set dst (XorL src1 m1));
11297   ins_cost(INSN_COST);
11298   format %{ "eon  $dst, $src1, zr" %}
11299 
11300   ins_encode %{
11301     __ eon(as_Register($dst$$reg),
11302               as_Register($src1$$reg),
11303               zr,
11304               Assembler::LSL, 0);
11305   %}
11306 
11307   ins_pipe(ialu_reg);
11308 %}
11309 
11310 // This pattern is automatically generated from aarch64_ad.m4
11311 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11312 instruct regI_not_reg(iRegINoSp dst,
11313                          iRegIorL2I src1, immI_M1 m1,
11314                          rFlagsReg cr) %{
11315   match(Set dst (XorI src1 m1));
11316   ins_cost(INSN_COST);
11317   format %{ "eonw  $dst, $src1, zr" %}
11318 
11319   ins_encode %{
11320     __ eonw(as_Register($dst$$reg),
11321               as_Register($src1$$reg),
11322               zr,
11323               Assembler::LSL, 0);
11324   %}
11325 
11326   ins_pipe(ialu_reg);
11327 %}
11328 
11329 // This pattern is automatically generated from aarch64_ad.m4
11330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11331 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11332                               immI0 zero, iRegIorL2I src1, immI src2) %{
11333   match(Set dst (SubI zero (URShiftI src1 src2)));
11334 
11335   ins_cost(1.9 * INSN_COST);
11336   format %{ "negw  $dst, $src1, LSR $src2" %}
11337 
11338   ins_encode %{
11339     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11340             Assembler::LSR, $src2$$constant & 0x1f);
11341   %}
11342 
11343   ins_pipe(ialu_reg_shift);
11344 %}
11345 
11346 // This pattern is automatically generated from aarch64_ad.m4
11347 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11348 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11349                               immI0 zero, iRegIorL2I src1, immI src2) %{
11350   match(Set dst (SubI zero (RShiftI src1 src2)));
11351 
11352   ins_cost(1.9 * INSN_COST);
11353   format %{ "negw  $dst, $src1, ASR $src2" %}
11354 
11355   ins_encode %{
11356     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11357             Assembler::ASR, $src2$$constant & 0x1f);
11358   %}
11359 
11360   ins_pipe(ialu_reg_shift);
11361 %}
11362 
11363 // This pattern is automatically generated from aarch64_ad.m4
11364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11365 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11366                               immI0 zero, iRegIorL2I src1, immI src2) %{
11367   match(Set dst (SubI zero (LShiftI src1 src2)));
11368 
11369   ins_cost(1.9 * INSN_COST);
11370   format %{ "negw  $dst, $src1, LSL $src2" %}
11371 
11372   ins_encode %{
11373     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11374             Assembler::LSL, $src2$$constant & 0x1f);
11375   %}
11376 
11377   ins_pipe(ialu_reg_shift);
11378 %}
11379 
11380 // This pattern is automatically generated from aarch64_ad.m4
11381 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11382 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11383                               immL0 zero, iRegL src1, immI src2) %{
11384   match(Set dst (SubL zero (URShiftL src1 src2)));
11385 
11386   ins_cost(1.9 * INSN_COST);
11387   format %{ "neg  $dst, $src1, LSR $src2" %}
11388 
11389   ins_encode %{
11390     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11391             Assembler::LSR, $src2$$constant & 0x3f);
11392   %}
11393 
11394   ins_pipe(ialu_reg_shift);
11395 %}
11396 
11397 // This pattern is automatically generated from aarch64_ad.m4
11398 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11399 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11400                               immL0 zero, iRegL src1, immI src2) %{
11401   match(Set dst (SubL zero (RShiftL src1 src2)));
11402 
11403   ins_cost(1.9 * INSN_COST);
11404   format %{ "neg  $dst, $src1, ASR $src2" %}
11405 
11406   ins_encode %{
11407     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11408             Assembler::ASR, $src2$$constant & 0x3f);
11409   %}
11410 
11411   ins_pipe(ialu_reg_shift);
11412 %}
11413 
11414 // This pattern is automatically generated from aarch64_ad.m4
11415 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11416 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11417                               immL0 zero, iRegL src1, immI src2) %{
11418   match(Set dst (SubL zero (LShiftL src1 src2)));
11419 
11420   ins_cost(1.9 * INSN_COST);
11421   format %{ "neg  $dst, $src1, LSL $src2" %}
11422 
11423   ins_encode %{
11424     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11425             Assembler::LSL, $src2$$constant & 0x3f);
11426   %}
11427 
11428   ins_pipe(ialu_reg_shift);
11429 %}
11430 
11431 // This pattern is automatically generated from aarch64_ad.m4
11432 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11433 instruct AndI_reg_not_reg(iRegINoSp dst,
11434                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11435   match(Set dst (AndI src1 (XorI src2 m1)));
11436   ins_cost(INSN_COST);
11437   format %{ "bicw  $dst, $src1, $src2" %}
11438 
11439   ins_encode %{
11440     __ bicw(as_Register($dst$$reg),
11441               as_Register($src1$$reg),
11442               as_Register($src2$$reg),
11443               Assembler::LSL, 0);
11444   %}
11445 
11446   ins_pipe(ialu_reg_reg);
11447 %}
11448 
11449 // This pattern is automatically generated from aarch64_ad.m4
11450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11451 instruct AndL_reg_not_reg(iRegLNoSp dst,
11452                          iRegL src1, iRegL src2, immL_M1 m1) %{
11453   match(Set dst (AndL src1 (XorL src2 m1)));
11454   ins_cost(INSN_COST);
11455   format %{ "bic  $dst, $src1, $src2" %}
11456 
11457   ins_encode %{
11458     __ bic(as_Register($dst$$reg),
11459               as_Register($src1$$reg),
11460               as_Register($src2$$reg),
11461               Assembler::LSL, 0);
11462   %}
11463 
11464   ins_pipe(ialu_reg_reg);
11465 %}
11466 
11467 // This pattern is automatically generated from aarch64_ad.m4
11468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11469 instruct OrI_reg_not_reg(iRegINoSp dst,
11470                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11471   match(Set dst (OrI src1 (XorI src2 m1)));
11472   ins_cost(INSN_COST);
11473   format %{ "ornw  $dst, $src1, $src2" %}
11474 
11475   ins_encode %{
11476     __ ornw(as_Register($dst$$reg),
11477               as_Register($src1$$reg),
11478               as_Register($src2$$reg),
11479               Assembler::LSL, 0);
11480   %}
11481 
11482   ins_pipe(ialu_reg_reg);
11483 %}
11484 
11485 // This pattern is automatically generated from aarch64_ad.m4
11486 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11487 instruct OrL_reg_not_reg(iRegLNoSp dst,
11488                          iRegL src1, iRegL src2, immL_M1 m1) %{
11489   match(Set dst (OrL src1 (XorL src2 m1)));
11490   ins_cost(INSN_COST);
11491   format %{ "orn  $dst, $src1, $src2" %}
11492 
11493   ins_encode %{
11494     __ orn(as_Register($dst$$reg),
11495               as_Register($src1$$reg),
11496               as_Register($src2$$reg),
11497               Assembler::LSL, 0);
11498   %}
11499 
11500   ins_pipe(ialu_reg_reg);
11501 %}
11502 
11503 // This pattern is automatically generated from aarch64_ad.m4
11504 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11505 instruct XorI_reg_not_reg(iRegINoSp dst,
11506                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11507   match(Set dst (XorI m1 (XorI src2 src1)));
11508   ins_cost(INSN_COST);
11509   format %{ "eonw  $dst, $src1, $src2" %}
11510 
11511   ins_encode %{
11512     __ eonw(as_Register($dst$$reg),
11513               as_Register($src1$$reg),
11514               as_Register($src2$$reg),
11515               Assembler::LSL, 0);
11516   %}
11517 
11518   ins_pipe(ialu_reg_reg);
11519 %}
11520 
11521 // This pattern is automatically generated from aarch64_ad.m4
11522 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11523 instruct XorL_reg_not_reg(iRegLNoSp dst,
11524                          iRegL src1, iRegL src2, immL_M1 m1) %{
11525   match(Set dst (XorL m1 (XorL src2 src1)));
11526   ins_cost(INSN_COST);
11527   format %{ "eon  $dst, $src1, $src2" %}
11528 
11529   ins_encode %{
11530     __ eon(as_Register($dst$$reg),
11531               as_Register($src1$$reg),
11532               as_Register($src2$$reg),
11533               Assembler::LSL, 0);
11534   %}
11535 
11536   ins_pipe(ialu_reg_reg);
11537 %}
11538 
11539 // This pattern is automatically generated from aarch64_ad.m4
11540 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11541 // val & (-1 ^ (val >>> shift)) ==> bicw
11542 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11543                          iRegIorL2I src1, iRegIorL2I src2,
11544                          immI src3, immI_M1 src4) %{
11545   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11546   ins_cost(1.9 * INSN_COST);
11547   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11548 
11549   ins_encode %{
11550     __ bicw(as_Register($dst$$reg),
11551               as_Register($src1$$reg),
11552               as_Register($src2$$reg),
11553               Assembler::LSR,
11554               $src3$$constant & 0x1f);
11555   %}
11556 
11557   ins_pipe(ialu_reg_reg_shift);
11558 %}
11559 
11560 // This pattern is automatically generated from aarch64_ad.m4
11561 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11562 // val & (-1 ^ (val >>> shift)) ==> bic
11563 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11564                          iRegL src1, iRegL src2,
11565                          immI src3, immL_M1 src4) %{
11566   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11567   ins_cost(1.9 * INSN_COST);
11568   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11569 
11570   ins_encode %{
11571     __ bic(as_Register($dst$$reg),
11572               as_Register($src1$$reg),
11573               as_Register($src2$$reg),
11574               Assembler::LSR,
11575               $src3$$constant & 0x3f);
11576   %}
11577 
11578   ins_pipe(ialu_reg_reg_shift);
11579 %}
11580 
11581 // This pattern is automatically generated from aarch64_ad.m4
11582 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11583 // val & (-1 ^ (val >> shift)) ==> bicw
11584 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11585                          iRegIorL2I src1, iRegIorL2I src2,
11586                          immI src3, immI_M1 src4) %{
11587   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11588   ins_cost(1.9 * INSN_COST);
11589   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11590 
11591   ins_encode %{
11592     __ bicw(as_Register($dst$$reg),
11593               as_Register($src1$$reg),
11594               as_Register($src2$$reg),
11595               Assembler::ASR,
11596               $src3$$constant & 0x1f);
11597   %}
11598 
11599   ins_pipe(ialu_reg_reg_shift);
11600 %}
11601 
11602 // This pattern is automatically generated from aarch64_ad.m4
11603 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11604 // val & (-1 ^ (val >> shift)) ==> bic
11605 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11606                          iRegL src1, iRegL src2,
11607                          immI src3, immL_M1 src4) %{
11608   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11609   ins_cost(1.9 * INSN_COST);
11610   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11611 
11612   ins_encode %{
11613     __ bic(as_Register($dst$$reg),
11614               as_Register($src1$$reg),
11615               as_Register($src2$$reg),
11616               Assembler::ASR,
11617               $src3$$constant & 0x3f);
11618   %}
11619 
11620   ins_pipe(ialu_reg_reg_shift);
11621 %}
11622 
11623 // This pattern is automatically generated from aarch64_ad.m4
11624 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11625 // val & (-1 ^ (val ror shift)) ==> bicw
11626 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11627                          iRegIorL2I src1, iRegIorL2I src2,
11628                          immI src3, immI_M1 src4) %{
11629   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11630   ins_cost(1.9 * INSN_COST);
11631   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11632 
11633   ins_encode %{
11634     __ bicw(as_Register($dst$$reg),
11635               as_Register($src1$$reg),
11636               as_Register($src2$$reg),
11637               Assembler::ROR,
11638               $src3$$constant & 0x1f);
11639   %}
11640 
11641   ins_pipe(ialu_reg_reg_shift);
11642 %}
11643 
11644 // This pattern is automatically generated from aarch64_ad.m4
11645 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11646 // val & (-1 ^ (val ror shift)) ==> bic
11647 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11648                          iRegL src1, iRegL src2,
11649                          immI src3, immL_M1 src4) %{
11650   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11651   ins_cost(1.9 * INSN_COST);
11652   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11653 
11654   ins_encode %{
11655     __ bic(as_Register($dst$$reg),
11656               as_Register($src1$$reg),
11657               as_Register($src2$$reg),
11658               Assembler::ROR,
11659               $src3$$constant & 0x3f);
11660   %}
11661 
11662   ins_pipe(ialu_reg_reg_shift);
11663 %}
11664 
11665 // This pattern is automatically generated from aarch64_ad.m4
11666 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11667 // val & (-1 ^ (val << shift)) ==> bicw
11668 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11669                          iRegIorL2I src1, iRegIorL2I src2,
11670                          immI src3, immI_M1 src4) %{
11671   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11672   ins_cost(1.9 * INSN_COST);
11673   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11674 
11675   ins_encode %{
11676     __ bicw(as_Register($dst$$reg),
11677               as_Register($src1$$reg),
11678               as_Register($src2$$reg),
11679               Assembler::LSL,
11680               $src3$$constant & 0x1f);
11681   %}
11682 
11683   ins_pipe(ialu_reg_reg_shift);
11684 %}
11685 
11686 // This pattern is automatically generated from aarch64_ad.m4
11687 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11688 // val & (-1 ^ (val << shift)) ==> bic
11689 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11690                          iRegL src1, iRegL src2,
11691                          immI src3, immL_M1 src4) %{
11692   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11693   ins_cost(1.9 * INSN_COST);
11694   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11695 
11696   ins_encode %{
11697     __ bic(as_Register($dst$$reg),
11698               as_Register($src1$$reg),
11699               as_Register($src2$$reg),
11700               Assembler::LSL,
11701               $src3$$constant & 0x3f);
11702   %}
11703 
11704   ins_pipe(ialu_reg_reg_shift);
11705 %}
11706 
11707 // This pattern is automatically generated from aarch64_ad.m4
11708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11709 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11710 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11711                          iRegIorL2I src1, iRegIorL2I src2,
11712                          immI src3, immI_M1 src4) %{
11713   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11714   ins_cost(1.9 * INSN_COST);
11715   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11716 
11717   ins_encode %{
11718     __ eonw(as_Register($dst$$reg),
11719               as_Register($src1$$reg),
11720               as_Register($src2$$reg),
11721               Assembler::LSR,
11722               $src3$$constant & 0x1f);
11723   %}
11724 
11725   ins_pipe(ialu_reg_reg_shift);
11726 %}
11727 
11728 // This pattern is automatically generated from aarch64_ad.m4
11729 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11730 // val ^ (-1 ^ (val >>> shift)) ==> eon
11731 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11732                          iRegL src1, iRegL src2,
11733                          immI src3, immL_M1 src4) %{
11734   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11735   ins_cost(1.9 * INSN_COST);
11736   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11737 
11738   ins_encode %{
11739     __ eon(as_Register($dst$$reg),
11740               as_Register($src1$$reg),
11741               as_Register($src2$$reg),
11742               Assembler::LSR,
11743               $src3$$constant & 0x3f);
11744   %}
11745 
11746   ins_pipe(ialu_reg_reg_shift);
11747 %}
11748 
11749 // This pattern is automatically generated from aarch64_ad.m4
11750 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11751 // val ^ (-1 ^ (val >> shift)) ==> eonw
11752 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11753                          iRegIorL2I src1, iRegIorL2I src2,
11754                          immI src3, immI_M1 src4) %{
11755   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11756   ins_cost(1.9 * INSN_COST);
11757   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11758 
11759   ins_encode %{
11760     __ eonw(as_Register($dst$$reg),
11761               as_Register($src1$$reg),
11762               as_Register($src2$$reg),
11763               Assembler::ASR,
11764               $src3$$constant & 0x1f);
11765   %}
11766 
11767   ins_pipe(ialu_reg_reg_shift);
11768 %}
11769 
11770 // This pattern is automatically generated from aarch64_ad.m4
11771 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11772 // val ^ (-1 ^ (val >> shift)) ==> eon
11773 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11774                          iRegL src1, iRegL src2,
11775                          immI src3, immL_M1 src4) %{
11776   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11777   ins_cost(1.9 * INSN_COST);
11778   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11779 
11780   ins_encode %{
11781     __ eon(as_Register($dst$$reg),
11782               as_Register($src1$$reg),
11783               as_Register($src2$$reg),
11784               Assembler::ASR,
11785               $src3$$constant & 0x3f);
11786   %}
11787 
11788   ins_pipe(ialu_reg_reg_shift);
11789 %}
11790 
11791 // This pattern is automatically generated from aarch64_ad.m4
11792 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11793 // val ^ (-1 ^ (val ror shift)) ==> eonw
11794 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11795                          iRegIorL2I src1, iRegIorL2I src2,
11796                          immI src3, immI_M1 src4) %{
11797   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11798   ins_cost(1.9 * INSN_COST);
11799   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11800 
11801   ins_encode %{
11802     __ eonw(as_Register($dst$$reg),
11803               as_Register($src1$$reg),
11804               as_Register($src2$$reg),
11805               Assembler::ROR,
11806               $src3$$constant & 0x1f);
11807   %}
11808 
11809   ins_pipe(ialu_reg_reg_shift);
11810 %}
11811 
11812 // This pattern is automatically generated from aarch64_ad.m4
11813 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11814 // val ^ (-1 ^ (val ror shift)) ==> eon
11815 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11816                          iRegL src1, iRegL src2,
11817                          immI src3, immL_M1 src4) %{
11818   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11819   ins_cost(1.9 * INSN_COST);
11820   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11821 
11822   ins_encode %{
11823     __ eon(as_Register($dst$$reg),
11824               as_Register($src1$$reg),
11825               as_Register($src2$$reg),
11826               Assembler::ROR,
11827               $src3$$constant & 0x3f);
11828   %}
11829 
11830   ins_pipe(ialu_reg_reg_shift);
11831 %}
11832 
11833 // This pattern is automatically generated from aarch64_ad.m4
11834 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11835 // val ^ (-1 ^ (val << shift)) ==> eonw
11836 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11837                          iRegIorL2I src1, iRegIorL2I src2,
11838                          immI src3, immI_M1 src4) %{
11839   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11840   ins_cost(1.9 * INSN_COST);
11841   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11842 
11843   ins_encode %{
11844     __ eonw(as_Register($dst$$reg),
11845               as_Register($src1$$reg),
11846               as_Register($src2$$reg),
11847               Assembler::LSL,
11848               $src3$$constant & 0x1f);
11849   %}
11850 
11851   ins_pipe(ialu_reg_reg_shift);
11852 %}
11853 
11854 // This pattern is automatically generated from aarch64_ad.m4
11855 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11856 // val ^ (-1 ^ (val << shift)) ==> eon
11857 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11858                          iRegL src1, iRegL src2,
11859                          immI src3, immL_M1 src4) %{
11860   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11861   ins_cost(1.9 * INSN_COST);
11862   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11863 
11864   ins_encode %{
11865     __ eon(as_Register($dst$$reg),
11866               as_Register($src1$$reg),
11867               as_Register($src2$$reg),
11868               Assembler::LSL,
11869               $src3$$constant & 0x3f);
11870   %}
11871 
11872   ins_pipe(ialu_reg_reg_shift);
11873 %}
11874 
11875 // This pattern is automatically generated from aarch64_ad.m4
11876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11877 // val | (-1 ^ (val >>> shift)) ==> ornw
11878 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11879                          iRegIorL2I src1, iRegIorL2I src2,
11880                          immI src3, immI_M1 src4) %{
11881   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11882   ins_cost(1.9 * INSN_COST);
11883   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11884 
11885   ins_encode %{
11886     __ ornw(as_Register($dst$$reg),
11887               as_Register($src1$$reg),
11888               as_Register($src2$$reg),
11889               Assembler::LSR,
11890               $src3$$constant & 0x1f);
11891   %}
11892 
11893   ins_pipe(ialu_reg_reg_shift);
11894 %}
11895 
11896 // This pattern is automatically generated from aarch64_ad.m4
11897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11898 // val | (-1 ^ (val >>> shift)) ==> orn
11899 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11900                          iRegL src1, iRegL src2,
11901                          immI src3, immL_M1 src4) %{
11902   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11903   ins_cost(1.9 * INSN_COST);
11904   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11905 
11906   ins_encode %{
11907     __ orn(as_Register($dst$$reg),
11908               as_Register($src1$$reg),
11909               as_Register($src2$$reg),
11910               Assembler::LSR,
11911               $src3$$constant & 0x3f);
11912   %}
11913 
11914   ins_pipe(ialu_reg_reg_shift);
11915 %}
11916 
11917 // This pattern is automatically generated from aarch64_ad.m4
11918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11919 // val | (-1 ^ (val >> shift)) ==> ornw
11920 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11921                          iRegIorL2I src1, iRegIorL2I src2,
11922                          immI src3, immI_M1 src4) %{
11923   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11924   ins_cost(1.9 * INSN_COST);
11925   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11926 
11927   ins_encode %{
11928     __ ornw(as_Register($dst$$reg),
11929               as_Register($src1$$reg),
11930               as_Register($src2$$reg),
11931               Assembler::ASR,
11932               $src3$$constant & 0x1f);
11933   %}
11934 
11935   ins_pipe(ialu_reg_reg_shift);
11936 %}
11937 
11938 // This pattern is automatically generated from aarch64_ad.m4
11939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11940 // val | (-1 ^ (val >> shift)) ==> orn
11941 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11942                          iRegL src1, iRegL src2,
11943                          immI src3, immL_M1 src4) %{
11944   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11945   ins_cost(1.9 * INSN_COST);
11946   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11947 
11948   ins_encode %{
11949     __ orn(as_Register($dst$$reg),
11950               as_Register($src1$$reg),
11951               as_Register($src2$$reg),
11952               Assembler::ASR,
11953               $src3$$constant & 0x3f);
11954   %}
11955 
11956   ins_pipe(ialu_reg_reg_shift);
11957 %}
11958 
11959 // This pattern is automatically generated from aarch64_ad.m4
11960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11961 // val | (-1 ^ (val ror shift)) ==> ornw
11962 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11963                          iRegIorL2I src1, iRegIorL2I src2,
11964                          immI src3, immI_M1 src4) %{
11965   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11966   ins_cost(1.9 * INSN_COST);
11967   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11968 
11969   ins_encode %{
11970     __ ornw(as_Register($dst$$reg),
11971               as_Register($src1$$reg),
11972               as_Register($src2$$reg),
11973               Assembler::ROR,
11974               $src3$$constant & 0x1f);
11975   %}
11976 
11977   ins_pipe(ialu_reg_reg_shift);
11978 %}
11979 
11980 // This pattern is automatically generated from aarch64_ad.m4
11981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11982 // val | (-1 ^ (val ror shift)) ==> orn
11983 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11984                          iRegL src1, iRegL src2,
11985                          immI src3, immL_M1 src4) %{
11986   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11987   ins_cost(1.9 * INSN_COST);
11988   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11989 
11990   ins_encode %{
11991     __ orn(as_Register($dst$$reg),
11992               as_Register($src1$$reg),
11993               as_Register($src2$$reg),
11994               Assembler::ROR,
11995               $src3$$constant & 0x3f);
11996   %}
11997 
11998   ins_pipe(ialu_reg_reg_shift);
11999 %}
12000 
12001 // This pattern is automatically generated from aarch64_ad.m4
12002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12003 // val | (-1 ^ (val << shift)) ==> ornw
12004 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
12005                          iRegIorL2I src1, iRegIorL2I src2,
12006                          immI src3, immI_M1 src4) %{
12007   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
12008   ins_cost(1.9 * INSN_COST);
12009   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
12010 
12011   ins_encode %{
12012     __ ornw(as_Register($dst$$reg),
12013               as_Register($src1$$reg),
12014               as_Register($src2$$reg),
12015               Assembler::LSL,
12016               $src3$$constant & 0x1f);
12017   %}
12018 
12019   ins_pipe(ialu_reg_reg_shift);
12020 %}
12021 
12022 // This pattern is automatically generated from aarch64_ad.m4
12023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12024 // val | (-1 ^ (val << shift)) ==> orn
12025 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
12026                          iRegL src1, iRegL src2,
12027                          immI src3, immL_M1 src4) %{
12028   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
12029   ins_cost(1.9 * INSN_COST);
12030   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
12031 
12032   ins_encode %{
12033     __ orn(as_Register($dst$$reg),
12034               as_Register($src1$$reg),
12035               as_Register($src2$$reg),
12036               Assembler::LSL,
12037               $src3$$constant & 0x3f);
12038   %}
12039 
12040   ins_pipe(ialu_reg_reg_shift);
12041 %}
12042 
12043 // This pattern is automatically generated from aarch64_ad.m4
12044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12045 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12046                          iRegIorL2I src1, iRegIorL2I src2,
12047                          immI src3) %{
12048   match(Set dst (AndI src1 (URShiftI src2 src3)));
12049 
12050   ins_cost(1.9 * INSN_COST);
12051   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
12052 
12053   ins_encode %{
12054     __ andw(as_Register($dst$$reg),
12055               as_Register($src1$$reg),
12056               as_Register($src2$$reg),
12057               Assembler::LSR,
12058               $src3$$constant & 0x1f);
12059   %}
12060 
12061   ins_pipe(ialu_reg_reg_shift);
12062 %}
12063 
12064 // This pattern is automatically generated from aarch64_ad.m4
12065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12066 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12067                          iRegL src1, iRegL src2,
12068                          immI src3) %{
12069   match(Set dst (AndL src1 (URShiftL src2 src3)));
12070 
12071   ins_cost(1.9 * INSN_COST);
12072   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
12073 
12074   ins_encode %{
12075     __ andr(as_Register($dst$$reg),
12076               as_Register($src1$$reg),
12077               as_Register($src2$$reg),
12078               Assembler::LSR,
12079               $src3$$constant & 0x3f);
12080   %}
12081 
12082   ins_pipe(ialu_reg_reg_shift);
12083 %}
12084 
12085 // This pattern is automatically generated from aarch64_ad.m4
12086 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12087 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12088                          iRegIorL2I src1, iRegIorL2I src2,
12089                          immI src3) %{
12090   match(Set dst (AndI src1 (RShiftI src2 src3)));
12091 
12092   ins_cost(1.9 * INSN_COST);
12093   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
12094 
12095   ins_encode %{
12096     __ andw(as_Register($dst$$reg),
12097               as_Register($src1$$reg),
12098               as_Register($src2$$reg),
12099               Assembler::ASR,
12100               $src3$$constant & 0x1f);
12101   %}
12102 
12103   ins_pipe(ialu_reg_reg_shift);
12104 %}
12105 
12106 // This pattern is automatically generated from aarch64_ad.m4
12107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12108 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12109                          iRegL src1, iRegL src2,
12110                          immI src3) %{
12111   match(Set dst (AndL src1 (RShiftL src2 src3)));
12112 
12113   ins_cost(1.9 * INSN_COST);
12114   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
12115 
12116   ins_encode %{
12117     __ andr(as_Register($dst$$reg),
12118               as_Register($src1$$reg),
12119               as_Register($src2$$reg),
12120               Assembler::ASR,
12121               $src3$$constant & 0x3f);
12122   %}
12123 
12124   ins_pipe(ialu_reg_reg_shift);
12125 %}
12126 
12127 // This pattern is automatically generated from aarch64_ad.m4
12128 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12129 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12130                          iRegIorL2I src1, iRegIorL2I src2,
12131                          immI src3) %{
12132   match(Set dst (AndI src1 (LShiftI src2 src3)));
12133 
12134   ins_cost(1.9 * INSN_COST);
12135   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12136 
12137   ins_encode %{
12138     __ andw(as_Register($dst$$reg),
12139               as_Register($src1$$reg),
12140               as_Register($src2$$reg),
12141               Assembler::LSL,
12142               $src3$$constant & 0x1f);
12143   %}
12144 
12145   ins_pipe(ialu_reg_reg_shift);
12146 %}
12147 
12148 // This pattern is automatically generated from aarch64_ad.m4
12149 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12150 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12151                          iRegL src1, iRegL src2,
12152                          immI src3) %{
12153   match(Set dst (AndL src1 (LShiftL src2 src3)));
12154 
12155   ins_cost(1.9 * INSN_COST);
12156   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12157 
12158   ins_encode %{
12159     __ andr(as_Register($dst$$reg),
12160               as_Register($src1$$reg),
12161               as_Register($src2$$reg),
12162               Assembler::LSL,
12163               $src3$$constant & 0x3f);
12164   %}
12165 
12166   ins_pipe(ialu_reg_reg_shift);
12167 %}
12168 
12169 // This pattern is automatically generated from aarch64_ad.m4
12170 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12171 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12172                          iRegIorL2I src1, iRegIorL2I src2,
12173                          immI src3) %{
12174   match(Set dst (AndI src1 (RotateRight src2 src3)));
12175 
12176   ins_cost(1.9 * INSN_COST);
12177   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12178 
12179   ins_encode %{
12180     __ andw(as_Register($dst$$reg),
12181               as_Register($src1$$reg),
12182               as_Register($src2$$reg),
12183               Assembler::ROR,
12184               $src3$$constant & 0x1f);
12185   %}
12186 
12187   ins_pipe(ialu_reg_reg_shift);
12188 %}
12189 
12190 // This pattern is automatically generated from aarch64_ad.m4
12191 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12192 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12193                          iRegL src1, iRegL src2,
12194                          immI src3) %{
12195   match(Set dst (AndL src1 (RotateRight src2 src3)));
12196 
12197   ins_cost(1.9 * INSN_COST);
12198   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12199 
12200   ins_encode %{
12201     __ andr(as_Register($dst$$reg),
12202               as_Register($src1$$reg),
12203               as_Register($src2$$reg),
12204               Assembler::ROR,
12205               $src3$$constant & 0x3f);
12206   %}
12207 
12208   ins_pipe(ialu_reg_reg_shift);
12209 %}
12210 
12211 // This pattern is automatically generated from aarch64_ad.m4
12212 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12213 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12214                          iRegIorL2I src1, iRegIorL2I src2,
12215                          immI src3) %{
12216   match(Set dst (XorI src1 (URShiftI src2 src3)));
12217 
12218   ins_cost(1.9 * INSN_COST);
12219   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12220 
12221   ins_encode %{
12222     __ eorw(as_Register($dst$$reg),
12223               as_Register($src1$$reg),
12224               as_Register($src2$$reg),
12225               Assembler::LSR,
12226               $src3$$constant & 0x1f);
12227   %}
12228 
12229   ins_pipe(ialu_reg_reg_shift);
12230 %}
12231 
12232 // This pattern is automatically generated from aarch64_ad.m4
12233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12234 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12235                          iRegL src1, iRegL src2,
12236                          immI src3) %{
12237   match(Set dst (XorL src1 (URShiftL src2 src3)));
12238 
12239   ins_cost(1.9 * INSN_COST);
12240   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12241 
12242   ins_encode %{
12243     __ eor(as_Register($dst$$reg),
12244               as_Register($src1$$reg),
12245               as_Register($src2$$reg),
12246               Assembler::LSR,
12247               $src3$$constant & 0x3f);
12248   %}
12249 
12250   ins_pipe(ialu_reg_reg_shift);
12251 %}
12252 
12253 // This pattern is automatically generated from aarch64_ad.m4
12254 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12255 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12256                          iRegIorL2I src1, iRegIorL2I src2,
12257                          immI src3) %{
12258   match(Set dst (XorI src1 (RShiftI src2 src3)));
12259 
12260   ins_cost(1.9 * INSN_COST);
12261   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12262 
12263   ins_encode %{
12264     __ eorw(as_Register($dst$$reg),
12265               as_Register($src1$$reg),
12266               as_Register($src2$$reg),
12267               Assembler::ASR,
12268               $src3$$constant & 0x1f);
12269   %}
12270 
12271   ins_pipe(ialu_reg_reg_shift);
12272 %}
12273 
12274 // This pattern is automatically generated from aarch64_ad.m4
12275 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12276 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12277                          iRegL src1, iRegL src2,
12278                          immI src3) %{
12279   match(Set dst (XorL src1 (RShiftL src2 src3)));
12280 
12281   ins_cost(1.9 * INSN_COST);
12282   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12283 
12284   ins_encode %{
12285     __ eor(as_Register($dst$$reg),
12286               as_Register($src1$$reg),
12287               as_Register($src2$$reg),
12288               Assembler::ASR,
12289               $src3$$constant & 0x3f);
12290   %}
12291 
12292   ins_pipe(ialu_reg_reg_shift);
12293 %}
12294 
12295 // This pattern is automatically generated from aarch64_ad.m4
12296 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12297 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12298                          iRegIorL2I src1, iRegIorL2I src2,
12299                          immI src3) %{
12300   match(Set dst (XorI src1 (LShiftI src2 src3)));
12301 
12302   ins_cost(1.9 * INSN_COST);
12303   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12304 
12305   ins_encode %{
12306     __ eorw(as_Register($dst$$reg),
12307               as_Register($src1$$reg),
12308               as_Register($src2$$reg),
12309               Assembler::LSL,
12310               $src3$$constant & 0x1f);
12311   %}
12312 
12313   ins_pipe(ialu_reg_reg_shift);
12314 %}
12315 
12316 // This pattern is automatically generated from aarch64_ad.m4
12317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12318 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12319                          iRegL src1, iRegL src2,
12320                          immI src3) %{
12321   match(Set dst (XorL src1 (LShiftL src2 src3)));
12322 
12323   ins_cost(1.9 * INSN_COST);
12324   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12325 
12326   ins_encode %{
12327     __ eor(as_Register($dst$$reg),
12328               as_Register($src1$$reg),
12329               as_Register($src2$$reg),
12330               Assembler::LSL,
12331               $src3$$constant & 0x3f);
12332   %}
12333 
12334   ins_pipe(ialu_reg_reg_shift);
12335 %}
12336 
12337 // This pattern is automatically generated from aarch64_ad.m4
12338 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12339 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12340                          iRegIorL2I src1, iRegIorL2I src2,
12341                          immI src3) %{
12342   match(Set dst (XorI src1 (RotateRight src2 src3)));
12343 
12344   ins_cost(1.9 * INSN_COST);
12345   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12346 
12347   ins_encode %{
12348     __ eorw(as_Register($dst$$reg),
12349               as_Register($src1$$reg),
12350               as_Register($src2$$reg),
12351               Assembler::ROR,
12352               $src3$$constant & 0x1f);
12353   %}
12354 
12355   ins_pipe(ialu_reg_reg_shift);
12356 %}
12357 
12358 // This pattern is automatically generated from aarch64_ad.m4
12359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12360 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12361                          iRegL src1, iRegL src2,
12362                          immI src3) %{
12363   match(Set dst (XorL src1 (RotateRight src2 src3)));
12364 
12365   ins_cost(1.9 * INSN_COST);
12366   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12367 
12368   ins_encode %{
12369     __ eor(as_Register($dst$$reg),
12370               as_Register($src1$$reg),
12371               as_Register($src2$$reg),
12372               Assembler::ROR,
12373               $src3$$constant & 0x3f);
12374   %}
12375 
12376   ins_pipe(ialu_reg_reg_shift);
12377 %}
12378 
12379 // This pattern is automatically generated from aarch64_ad.m4
12380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12381 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12382                          iRegIorL2I src1, iRegIorL2I src2,
12383                          immI src3) %{
12384   match(Set dst (OrI src1 (URShiftI src2 src3)));
12385 
12386   ins_cost(1.9 * INSN_COST);
12387   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12388 
12389   ins_encode %{
12390     __ orrw(as_Register($dst$$reg),
12391               as_Register($src1$$reg),
12392               as_Register($src2$$reg),
12393               Assembler::LSR,
12394               $src3$$constant & 0x1f);
12395   %}
12396 
12397   ins_pipe(ialu_reg_reg_shift);
12398 %}
12399 
12400 // This pattern is automatically generated from aarch64_ad.m4
12401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12402 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12403                          iRegL src1, iRegL src2,
12404                          immI src3) %{
12405   match(Set dst (OrL src1 (URShiftL src2 src3)));
12406 
12407   ins_cost(1.9 * INSN_COST);
12408   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12409 
12410   ins_encode %{
12411     __ orr(as_Register($dst$$reg),
12412               as_Register($src1$$reg),
12413               as_Register($src2$$reg),
12414               Assembler::LSR,
12415               $src3$$constant & 0x3f);
12416   %}
12417 
12418   ins_pipe(ialu_reg_reg_shift);
12419 %}
12420 
12421 // This pattern is automatically generated from aarch64_ad.m4
12422 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12423 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12424                          iRegIorL2I src1, iRegIorL2I src2,
12425                          immI src3) %{
12426   match(Set dst (OrI src1 (RShiftI src2 src3)));
12427 
12428   ins_cost(1.9 * INSN_COST);
12429   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12430 
12431   ins_encode %{
12432     __ orrw(as_Register($dst$$reg),
12433               as_Register($src1$$reg),
12434               as_Register($src2$$reg),
12435               Assembler::ASR,
12436               $src3$$constant & 0x1f);
12437   %}
12438 
12439   ins_pipe(ialu_reg_reg_shift);
12440 %}
12441 
12442 // This pattern is automatically generated from aarch64_ad.m4
12443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12444 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12445                          iRegL src1, iRegL src2,
12446                          immI src3) %{
12447   match(Set dst (OrL src1 (RShiftL src2 src3)));
12448 
12449   ins_cost(1.9 * INSN_COST);
12450   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12451 
12452   ins_encode %{
12453     __ orr(as_Register($dst$$reg),
12454               as_Register($src1$$reg),
12455               as_Register($src2$$reg),
12456               Assembler::ASR,
12457               $src3$$constant & 0x3f);
12458   %}
12459 
12460   ins_pipe(ialu_reg_reg_shift);
12461 %}
12462 
12463 // This pattern is automatically generated from aarch64_ad.m4
12464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12465 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12466                          iRegIorL2I src1, iRegIorL2I src2,
12467                          immI src3) %{
12468   match(Set dst (OrI src1 (LShiftI src2 src3)));
12469 
12470   ins_cost(1.9 * INSN_COST);
12471   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12472 
12473   ins_encode %{
12474     __ orrw(as_Register($dst$$reg),
12475               as_Register($src1$$reg),
12476               as_Register($src2$$reg),
12477               Assembler::LSL,
12478               $src3$$constant & 0x1f);
12479   %}
12480 
12481   ins_pipe(ialu_reg_reg_shift);
12482 %}
12483 
12484 // This pattern is automatically generated from aarch64_ad.m4
12485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12486 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12487                          iRegL src1, iRegL src2,
12488                          immI src3) %{
12489   match(Set dst (OrL src1 (LShiftL src2 src3)));
12490 
12491   ins_cost(1.9 * INSN_COST);
12492   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12493 
12494   ins_encode %{
12495     __ orr(as_Register($dst$$reg),
12496               as_Register($src1$$reg),
12497               as_Register($src2$$reg),
12498               Assembler::LSL,
12499               $src3$$constant & 0x3f);
12500   %}
12501 
12502   ins_pipe(ialu_reg_reg_shift);
12503 %}
12504 
12505 // This pattern is automatically generated from aarch64_ad.m4
12506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12507 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12508                          iRegIorL2I src1, iRegIorL2I src2,
12509                          immI src3) %{
12510   match(Set dst (OrI src1 (RotateRight src2 src3)));
12511 
12512   ins_cost(1.9 * INSN_COST);
12513   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12514 
12515   ins_encode %{
12516     __ orrw(as_Register($dst$$reg),
12517               as_Register($src1$$reg),
12518               as_Register($src2$$reg),
12519               Assembler::ROR,
12520               $src3$$constant & 0x1f);
12521   %}
12522 
12523   ins_pipe(ialu_reg_reg_shift);
12524 %}
12525 
12526 // This pattern is automatically generated from aarch64_ad.m4
12527 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12528 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12529                          iRegL src1, iRegL src2,
12530                          immI src3) %{
12531   match(Set dst (OrL src1 (RotateRight src2 src3)));
12532 
12533   ins_cost(1.9 * INSN_COST);
12534   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12535 
12536   ins_encode %{
12537     __ orr(as_Register($dst$$reg),
12538               as_Register($src1$$reg),
12539               as_Register($src2$$reg),
12540               Assembler::ROR,
12541               $src3$$constant & 0x3f);
12542   %}
12543 
12544   ins_pipe(ialu_reg_reg_shift);
12545 %}
12546 
12547 // This pattern is automatically generated from aarch64_ad.m4
12548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12549 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12550                          iRegIorL2I src1, iRegIorL2I src2,
12551                          immI src3) %{
12552   match(Set dst (AddI src1 (URShiftI src2 src3)));
12553 
12554   ins_cost(1.9 * INSN_COST);
12555   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12556 
12557   ins_encode %{
12558     __ addw(as_Register($dst$$reg),
12559               as_Register($src1$$reg),
12560               as_Register($src2$$reg),
12561               Assembler::LSR,
12562               $src3$$constant & 0x1f);
12563   %}
12564 
12565   ins_pipe(ialu_reg_reg_shift);
12566 %}
12567 
12568 // This pattern is automatically generated from aarch64_ad.m4
12569 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12570 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12571                          iRegL src1, iRegL src2,
12572                          immI src3) %{
12573   match(Set dst (AddL src1 (URShiftL src2 src3)));
12574 
12575   ins_cost(1.9 * INSN_COST);
12576   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12577 
12578   ins_encode %{
12579     __ add(as_Register($dst$$reg),
12580               as_Register($src1$$reg),
12581               as_Register($src2$$reg),
12582               Assembler::LSR,
12583               $src3$$constant & 0x3f);
12584   %}
12585 
12586   ins_pipe(ialu_reg_reg_shift);
12587 %}
12588 
12589 // This pattern is automatically generated from aarch64_ad.m4
12590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12591 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12592                          iRegIorL2I src1, iRegIorL2I src2,
12593                          immI src3) %{
12594   match(Set dst (AddI src1 (RShiftI src2 src3)));
12595 
12596   ins_cost(1.9 * INSN_COST);
12597   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12598 
12599   ins_encode %{
12600     __ addw(as_Register($dst$$reg),
12601               as_Register($src1$$reg),
12602               as_Register($src2$$reg),
12603               Assembler::ASR,
12604               $src3$$constant & 0x1f);
12605   %}
12606 
12607   ins_pipe(ialu_reg_reg_shift);
12608 %}
12609 
12610 // This pattern is automatically generated from aarch64_ad.m4
12611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12612 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12613                          iRegL src1, iRegL src2,
12614                          immI src3) %{
12615   match(Set dst (AddL src1 (RShiftL src2 src3)));
12616 
12617   ins_cost(1.9 * INSN_COST);
12618   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12619 
12620   ins_encode %{
12621     __ add(as_Register($dst$$reg),
12622               as_Register($src1$$reg),
12623               as_Register($src2$$reg),
12624               Assembler::ASR,
12625               $src3$$constant & 0x3f);
12626   %}
12627 
12628   ins_pipe(ialu_reg_reg_shift);
12629 %}
12630 
12631 // This pattern is automatically generated from aarch64_ad.m4
12632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12633 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12634                          iRegIorL2I src1, iRegIorL2I src2,
12635                          immI src3) %{
12636   match(Set dst (AddI src1 (LShiftI src2 src3)));
12637 
12638   ins_cost(1.9 * INSN_COST);
12639   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12640 
12641   ins_encode %{
12642     __ addw(as_Register($dst$$reg),
12643               as_Register($src1$$reg),
12644               as_Register($src2$$reg),
12645               Assembler::LSL,
12646               $src3$$constant & 0x1f);
12647   %}
12648 
12649   ins_pipe(ialu_reg_reg_shift);
12650 %}
12651 
12652 // This pattern is automatically generated from aarch64_ad.m4
12653 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12654 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12655                          iRegL src1, iRegL src2,
12656                          immI src3) %{
12657   match(Set dst (AddL src1 (LShiftL src2 src3)));
12658 
12659   ins_cost(1.9 * INSN_COST);
12660   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12661 
12662   ins_encode %{
12663     __ add(as_Register($dst$$reg),
12664               as_Register($src1$$reg),
12665               as_Register($src2$$reg),
12666               Assembler::LSL,
12667               $src3$$constant & 0x3f);
12668   %}
12669 
12670   ins_pipe(ialu_reg_reg_shift);
12671 %}
12672 
12673 // This pattern is automatically generated from aarch64_ad.m4
12674 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12675 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12676                          iRegIorL2I src1, iRegIorL2I src2,
12677                          immI src3) %{
12678   match(Set dst (SubI src1 (URShiftI src2 src3)));
12679 
12680   ins_cost(1.9 * INSN_COST);
12681   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12682 
12683   ins_encode %{
12684     __ subw(as_Register($dst$$reg),
12685               as_Register($src1$$reg),
12686               as_Register($src2$$reg),
12687               Assembler::LSR,
12688               $src3$$constant & 0x1f);
12689   %}
12690 
12691   ins_pipe(ialu_reg_reg_shift);
12692 %}
12693 
12694 // This pattern is automatically generated from aarch64_ad.m4
12695 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12696 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12697                          iRegL src1, iRegL src2,
12698                          immI src3) %{
12699   match(Set dst (SubL src1 (URShiftL src2 src3)));
12700 
12701   ins_cost(1.9 * INSN_COST);
12702   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12703 
12704   ins_encode %{
12705     __ sub(as_Register($dst$$reg),
12706               as_Register($src1$$reg),
12707               as_Register($src2$$reg),
12708               Assembler::LSR,
12709               $src3$$constant & 0x3f);
12710   %}
12711 
12712   ins_pipe(ialu_reg_reg_shift);
12713 %}
12714 
12715 // This pattern is automatically generated from aarch64_ad.m4
12716 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12717 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12718                          iRegIorL2I src1, iRegIorL2I src2,
12719                          immI src3) %{
12720   match(Set dst (SubI src1 (RShiftI src2 src3)));
12721 
12722   ins_cost(1.9 * INSN_COST);
12723   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12724 
12725   ins_encode %{
12726     __ subw(as_Register($dst$$reg),
12727               as_Register($src1$$reg),
12728               as_Register($src2$$reg),
12729               Assembler::ASR,
12730               $src3$$constant & 0x1f);
12731   %}
12732 
12733   ins_pipe(ialu_reg_reg_shift);
12734 %}
12735 
12736 // This pattern is automatically generated from aarch64_ad.m4
12737 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12738 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12739                          iRegL src1, iRegL src2,
12740                          immI src3) %{
12741   match(Set dst (SubL src1 (RShiftL src2 src3)));
12742 
12743   ins_cost(1.9 * INSN_COST);
12744   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12745 
12746   ins_encode %{
12747     __ sub(as_Register($dst$$reg),
12748               as_Register($src1$$reg),
12749               as_Register($src2$$reg),
12750               Assembler::ASR,
12751               $src3$$constant & 0x3f);
12752   %}
12753 
12754   ins_pipe(ialu_reg_reg_shift);
12755 %}
12756 
12757 // This pattern is automatically generated from aarch64_ad.m4
12758 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12759 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12760                          iRegIorL2I src1, iRegIorL2I src2,
12761                          immI src3) %{
12762   match(Set dst (SubI src1 (LShiftI src2 src3)));
12763 
12764   ins_cost(1.9 * INSN_COST);
12765   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12766 
12767   ins_encode %{
12768     __ subw(as_Register($dst$$reg),
12769               as_Register($src1$$reg),
12770               as_Register($src2$$reg),
12771               Assembler::LSL,
12772               $src3$$constant & 0x1f);
12773   %}
12774 
12775   ins_pipe(ialu_reg_reg_shift);
12776 %}
12777 
12778 // This pattern is automatically generated from aarch64_ad.m4
12779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12780 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12781                          iRegL src1, iRegL src2,
12782                          immI src3) %{
12783   match(Set dst (SubL src1 (LShiftL src2 src3)));
12784 
12785   ins_cost(1.9 * INSN_COST);
12786   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12787 
12788   ins_encode %{
12789     __ sub(as_Register($dst$$reg),
12790               as_Register($src1$$reg),
12791               as_Register($src2$$reg),
12792               Assembler::LSL,
12793               $src3$$constant & 0x3f);
12794   %}
12795 
12796   ins_pipe(ialu_reg_reg_shift);
12797 %}
12798 
12799 // This pattern is automatically generated from aarch64_ad.m4
12800 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12801 
12802 // Shift Left followed by Shift Right.
12803 // This idiom is used by the compiler for the i2b bytecode etc.
12804 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12805 %{
12806   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12807   ins_cost(INSN_COST * 2);
12808   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12809   ins_encode %{
12810     int lshift = $lshift_count$$constant & 63;
12811     int rshift = $rshift_count$$constant & 63;
12812     int s = 63 - lshift;
12813     int r = (rshift - lshift) & 63;
12814     __ sbfm(as_Register($dst$$reg),
12815             as_Register($src$$reg),
12816             r, s);
12817   %}
12818 
12819   ins_pipe(ialu_reg_shift);
12820 %}
12821 
12822 // This pattern is automatically generated from aarch64_ad.m4
12823 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12824 
12825 // Shift Left followed by Shift Right.
12826 // This idiom is used by the compiler for the i2b bytecode etc.
12827 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12828 %{
12829   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12830   ins_cost(INSN_COST * 2);
12831   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12832   ins_encode %{
12833     int lshift = $lshift_count$$constant & 31;
12834     int rshift = $rshift_count$$constant & 31;
12835     int s = 31 - lshift;
12836     int r = (rshift - lshift) & 31;
12837     __ sbfmw(as_Register($dst$$reg),
12838             as_Register($src$$reg),
12839             r, s);
12840   %}
12841 
12842   ins_pipe(ialu_reg_shift);
12843 %}
12844 
12845 // This pattern is automatically generated from aarch64_ad.m4
12846 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12847 
12848 // Shift Left followed by Shift Right.
12849 // This idiom is used by the compiler for the i2b bytecode etc.
12850 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12851 %{
12852   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12853   ins_cost(INSN_COST * 2);
12854   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12855   ins_encode %{
12856     int lshift = $lshift_count$$constant & 63;
12857     int rshift = $rshift_count$$constant & 63;
12858     int s = 63 - lshift;
12859     int r = (rshift - lshift) & 63;
12860     __ ubfm(as_Register($dst$$reg),
12861             as_Register($src$$reg),
12862             r, s);
12863   %}
12864 
12865   ins_pipe(ialu_reg_shift);
12866 %}
12867 
12868 // This pattern is automatically generated from aarch64_ad.m4
12869 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12870 
12871 // Shift Left followed by Shift Right.
12872 // This idiom is used by the compiler for the i2b bytecode etc.
12873 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12874 %{
12875   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12876   ins_cost(INSN_COST * 2);
12877   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12878   ins_encode %{
12879     int lshift = $lshift_count$$constant & 31;
12880     int rshift = $rshift_count$$constant & 31;
12881     int s = 31 - lshift;
12882     int r = (rshift - lshift) & 31;
12883     __ ubfmw(as_Register($dst$$reg),
12884             as_Register($src$$reg),
12885             r, s);
12886   %}
12887 
12888   ins_pipe(ialu_reg_shift);
12889 %}
12890 
12891 // Bitfield extract with shift & mask
12892 
12893 // This pattern is automatically generated from aarch64_ad.m4
12894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12895 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12896 %{
12897   match(Set dst (AndI (URShiftI src rshift) mask));
12898   // Make sure we are not going to exceed what ubfxw can do.
12899   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12900 
12901   ins_cost(INSN_COST);
12902   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12903   ins_encode %{
12904     int rshift = $rshift$$constant & 31;
12905     intptr_t mask = $mask$$constant;
12906     int width = exact_log2(mask+1);
12907     __ ubfxw(as_Register($dst$$reg),
12908             as_Register($src$$reg), rshift, width);
12909   %}
12910   ins_pipe(ialu_reg_shift);
12911 %}
12912 
12913 // This pattern is automatically generated from aarch64_ad.m4
12914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12915 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12916 %{
12917   match(Set dst (AndL (URShiftL src rshift) mask));
12918   // Make sure we are not going to exceed what ubfx can do.
12919   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12920 
12921   ins_cost(INSN_COST);
12922   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12923   ins_encode %{
12924     int rshift = $rshift$$constant & 63;
12925     intptr_t mask = $mask$$constant;
12926     int width = exact_log2_long(mask+1);
12927     __ ubfx(as_Register($dst$$reg),
12928             as_Register($src$$reg), rshift, width);
12929   %}
12930   ins_pipe(ialu_reg_shift);
12931 %}
12932 
12933 
12934 // This pattern is automatically generated from aarch64_ad.m4
12935 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12936 
12937 // We can use ubfx when extending an And with a mask when we know mask
12938 // is positive.  We know that because immI_bitmask guarantees it.
12939 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12940 %{
12941   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12942   // Make sure we are not going to exceed what ubfxw can do.
12943   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12944 
12945   ins_cost(INSN_COST * 2);
12946   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12947   ins_encode %{
12948     int rshift = $rshift$$constant & 31;
12949     intptr_t mask = $mask$$constant;
12950     int width = exact_log2(mask+1);
12951     __ ubfx(as_Register($dst$$reg),
12952             as_Register($src$$reg), rshift, width);
12953   %}
12954   ins_pipe(ialu_reg_shift);
12955 %}
12956 
12957 
12958 // This pattern is automatically generated from aarch64_ad.m4
12959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12960 
12961 // We can use ubfiz when masking by a positive number and then left shifting the result.
12962 // We know that the mask is positive because immI_bitmask guarantees it.
12963 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12964 %{
12965   match(Set dst (LShiftI (AndI src mask) lshift));
12966   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12967 
12968   ins_cost(INSN_COST);
12969   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12970   ins_encode %{
12971     int lshift = $lshift$$constant & 31;
12972     intptr_t mask = $mask$$constant;
12973     int width = exact_log2(mask+1);
12974     __ ubfizw(as_Register($dst$$reg),
12975           as_Register($src$$reg), lshift, width);
12976   %}
12977   ins_pipe(ialu_reg_shift);
12978 %}
12979 
12980 // This pattern is automatically generated from aarch64_ad.m4
12981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12982 
12983 // We can use ubfiz when masking by a positive number and then left shifting the result.
12984 // We know that the mask is positive because immL_bitmask guarantees it.
12985 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12986 %{
12987   match(Set dst (LShiftL (AndL src mask) lshift));
12988   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12989 
12990   ins_cost(INSN_COST);
12991   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12992   ins_encode %{
12993     int lshift = $lshift$$constant & 63;
12994     intptr_t mask = $mask$$constant;
12995     int width = exact_log2_long(mask+1);
12996     __ ubfiz(as_Register($dst$$reg),
12997           as_Register($src$$reg), lshift, width);
12998   %}
12999   ins_pipe(ialu_reg_shift);
13000 %}
13001 
13002 // This pattern is automatically generated from aarch64_ad.m4
13003 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13004 
13005 // We can use ubfiz when masking by a positive number and then left shifting the result.
13006 // We know that the mask is positive because immI_bitmask guarantees it.
13007 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13008 %{
13009   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
13010   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
13011 
13012   ins_cost(INSN_COST);
13013   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13014   ins_encode %{
13015     int lshift = $lshift$$constant & 31;
13016     intptr_t mask = $mask$$constant;
13017     int width = exact_log2(mask+1);
13018     __ ubfizw(as_Register($dst$$reg),
13019           as_Register($src$$reg), lshift, width);
13020   %}
13021   ins_pipe(ialu_reg_shift);
13022 %}
13023 
13024 // This pattern is automatically generated from aarch64_ad.m4
13025 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13026 
13027 // We can use ubfiz when masking by a positive number and then left shifting the result.
13028 // We know that the mask is positive because immL_bitmask guarantees it.
13029 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13030 %{
13031   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
13032   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
13033 
13034   ins_cost(INSN_COST);
13035   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13036   ins_encode %{
13037     int lshift = $lshift$$constant & 63;
13038     intptr_t mask = $mask$$constant;
13039     int width = exact_log2_long(mask+1);
13040     __ ubfiz(as_Register($dst$$reg),
13041           as_Register($src$$reg), lshift, width);
13042   %}
13043   ins_pipe(ialu_reg_shift);
13044 %}
13045 
13046 
13047 // This pattern is automatically generated from aarch64_ad.m4
13048 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13049 
13050 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13051 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13052 %{
13053   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13054   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13055 
13056   ins_cost(INSN_COST);
13057   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13058   ins_encode %{
13059     int lshift = $lshift$$constant & 63;
13060     intptr_t mask = $mask$$constant;
13061     int width = exact_log2(mask+1);
13062     __ ubfiz(as_Register($dst$$reg),
13063              as_Register($src$$reg), lshift, width);
13064   %}
13065   ins_pipe(ialu_reg_shift);
13066 %}
13067 
13068 // This pattern is automatically generated from aarch64_ad.m4
13069 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13070 
13071 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13072 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13073 %{
13074   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13075   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13076 
13077   ins_cost(INSN_COST);
13078   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13079   ins_encode %{
13080     int lshift = $lshift$$constant & 31;
13081     intptr_t mask = $mask$$constant;
13082     int width = exact_log2(mask+1);
13083     __ ubfiz(as_Register($dst$$reg),
13084              as_Register($src$$reg), lshift, width);
13085   %}
13086   ins_pipe(ialu_reg_shift);
13087 %}
13088 
13089 // This pattern is automatically generated from aarch64_ad.m4
13090 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13091 
13092 // Can skip int2long conversions after AND with small bitmask
13093 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13094 %{
13095   match(Set dst (ConvI2L (AndI src msk)));
13096   ins_cost(INSN_COST);
13097   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13098   ins_encode %{
13099     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13100   %}
13101   ins_pipe(ialu_reg_shift);
13102 %}
13103 
13104 
13105 // Rotations
13106 
13107 // This pattern is automatically generated from aarch64_ad.m4
13108 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13109 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13110 %{
13111   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13112   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13113 
13114   ins_cost(INSN_COST);
13115   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13116 
13117   ins_encode %{
13118     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13119             $rshift$$constant & 63);
13120   %}
13121   ins_pipe(ialu_reg_reg_extr);
13122 %}
13123 
13124 
13125 // This pattern is automatically generated from aarch64_ad.m4
13126 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13127 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13128 %{
13129   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13130   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13131 
13132   ins_cost(INSN_COST);
13133   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13134 
13135   ins_encode %{
13136     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13137             $rshift$$constant & 31);
13138   %}
13139   ins_pipe(ialu_reg_reg_extr);
13140 %}
13141 
13142 
13143 // This pattern is automatically generated from aarch64_ad.m4
13144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13145 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13146 %{
13147   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13148   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13149 
13150   ins_cost(INSN_COST);
13151   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13152 
13153   ins_encode %{
13154     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13155             $rshift$$constant & 63);
13156   %}
13157   ins_pipe(ialu_reg_reg_extr);
13158 %}
13159 
13160 
13161 // This pattern is automatically generated from aarch64_ad.m4
13162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13163 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13164 %{
13165   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13166   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13167 
13168   ins_cost(INSN_COST);
13169   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13170 
13171   ins_encode %{
13172     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13173             $rshift$$constant & 31);
13174   %}
13175   ins_pipe(ialu_reg_reg_extr);
13176 %}
13177 
13178 // This pattern is automatically generated from aarch64_ad.m4
13179 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13180 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13181 %{
13182   match(Set dst (RotateRight src shift));
13183 
13184   ins_cost(INSN_COST);
13185   format %{ "ror    $dst, $src, $shift" %}
13186 
13187   ins_encode %{
13188      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13189                $shift$$constant & 0x1f);
13190   %}
13191   ins_pipe(ialu_reg_reg_vshift);
13192 %}
13193 
13194 // This pattern is automatically generated from aarch64_ad.m4
13195 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13196 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13197 %{
13198   match(Set dst (RotateRight src shift));
13199 
13200   ins_cost(INSN_COST);
13201   format %{ "ror    $dst, $src, $shift" %}
13202 
13203   ins_encode %{
13204      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13205                $shift$$constant & 0x3f);
13206   %}
13207   ins_pipe(ialu_reg_reg_vshift);
13208 %}
13209 
13210 // This pattern is automatically generated from aarch64_ad.m4
13211 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13212 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13213 %{
13214   match(Set dst (RotateRight src shift));
13215 
13216   ins_cost(INSN_COST);
13217   format %{ "ror    $dst, $src, $shift" %}
13218 
13219   ins_encode %{
13220      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13221   %}
13222   ins_pipe(ialu_reg_reg_vshift);
13223 %}
13224 
13225 // This pattern is automatically generated from aarch64_ad.m4
13226 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13227 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13228 %{
13229   match(Set dst (RotateRight src shift));
13230 
13231   ins_cost(INSN_COST);
13232   format %{ "ror    $dst, $src, $shift" %}
13233 
13234   ins_encode %{
13235      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13236   %}
13237   ins_pipe(ialu_reg_reg_vshift);
13238 %}
13239 
13240 // This pattern is automatically generated from aarch64_ad.m4
13241 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13242 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13243 %{
13244   match(Set dst (RotateLeft src shift));
13245 
13246   ins_cost(INSN_COST);
13247   format %{ "rol    $dst, $src, $shift" %}
13248 
13249   ins_encode %{
13250      __ subw(rscratch1, zr, as_Register($shift$$reg));
13251      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13252   %}
13253   ins_pipe(ialu_reg_reg_vshift);
13254 %}
13255 
13256 // This pattern is automatically generated from aarch64_ad.m4
13257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13258 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13259 %{
13260   match(Set dst (RotateLeft src shift));
13261 
13262   ins_cost(INSN_COST);
13263   format %{ "rol    $dst, $src, $shift" %}
13264 
13265   ins_encode %{
13266      __ subw(rscratch1, zr, as_Register($shift$$reg));
13267      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13268   %}
13269   ins_pipe(ialu_reg_reg_vshift);
13270 %}
13271 
13272 
13273 // Add/subtract (extended)
13274 
13275 // This pattern is automatically generated from aarch64_ad.m4
13276 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13277 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13278 %{
13279   match(Set dst (AddL src1 (ConvI2L src2)));
13280   ins_cost(INSN_COST);
13281   format %{ "add  $dst, $src1, $src2, sxtw" %}
13282 
13283    ins_encode %{
13284      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13285             as_Register($src2$$reg), ext::sxtw);
13286    %}
13287   ins_pipe(ialu_reg_reg);
13288 %}
13289 
13290 // This pattern is automatically generated from aarch64_ad.m4
13291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13292 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13293 %{
13294   match(Set dst (SubL src1 (ConvI2L src2)));
13295   ins_cost(INSN_COST);
13296   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13297 
13298    ins_encode %{
13299      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13300             as_Register($src2$$reg), ext::sxtw);
13301    %}
13302   ins_pipe(ialu_reg_reg);
13303 %}
13304 
13305 // This pattern is automatically generated from aarch64_ad.m4
13306 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13307 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13308 %{
13309   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13310   ins_cost(INSN_COST);
13311   format %{ "add  $dst, $src1, $src2, sxth" %}
13312 
13313    ins_encode %{
13314      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13315             as_Register($src2$$reg), ext::sxth);
13316    %}
13317   ins_pipe(ialu_reg_reg);
13318 %}
13319 
13320 // This pattern is automatically generated from aarch64_ad.m4
13321 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13322 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13323 %{
13324   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13325   ins_cost(INSN_COST);
13326   format %{ "add  $dst, $src1, $src2, sxtb" %}
13327 
13328    ins_encode %{
13329      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13330             as_Register($src2$$reg), ext::sxtb);
13331    %}
13332   ins_pipe(ialu_reg_reg);
13333 %}
13334 
13335 // This pattern is automatically generated from aarch64_ad.m4
13336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13337 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13338 %{
13339   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13340   ins_cost(INSN_COST);
13341   format %{ "add  $dst, $src1, $src2, uxtb" %}
13342 
13343    ins_encode %{
13344      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13345             as_Register($src2$$reg), ext::uxtb);
13346    %}
13347   ins_pipe(ialu_reg_reg);
13348 %}
13349 
13350 // This pattern is automatically generated from aarch64_ad.m4
13351 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13352 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13353 %{
13354   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13355   ins_cost(INSN_COST);
13356   format %{ "add  $dst, $src1, $src2, sxth" %}
13357 
13358    ins_encode %{
13359      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13360             as_Register($src2$$reg), ext::sxth);
13361    %}
13362   ins_pipe(ialu_reg_reg);
13363 %}
13364 
13365 // This pattern is automatically generated from aarch64_ad.m4
13366 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13367 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13368 %{
13369   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13370   ins_cost(INSN_COST);
13371   format %{ "add  $dst, $src1, $src2, sxtw" %}
13372 
13373    ins_encode %{
13374      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13375             as_Register($src2$$reg), ext::sxtw);
13376    %}
13377   ins_pipe(ialu_reg_reg);
13378 %}
13379 
13380 // This pattern is automatically generated from aarch64_ad.m4
13381 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13382 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13383 %{
13384   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13385   ins_cost(INSN_COST);
13386   format %{ "add  $dst, $src1, $src2, sxtb" %}
13387 
13388    ins_encode %{
13389      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13390             as_Register($src2$$reg), ext::sxtb);
13391    %}
13392   ins_pipe(ialu_reg_reg);
13393 %}
13394 
13395 // This pattern is automatically generated from aarch64_ad.m4
13396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13397 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13398 %{
13399   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13400   ins_cost(INSN_COST);
13401   format %{ "add  $dst, $src1, $src2, uxtb" %}
13402 
13403    ins_encode %{
13404      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13405             as_Register($src2$$reg), ext::uxtb);
13406    %}
13407   ins_pipe(ialu_reg_reg);
13408 %}
13409 
13410 // This pattern is automatically generated from aarch64_ad.m4
13411 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13412 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13413 %{
13414   match(Set dst (AddI src1 (AndI src2 mask)));
13415   ins_cost(INSN_COST);
13416   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13417 
13418    ins_encode %{
13419      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13420             as_Register($src2$$reg), ext::uxtb);
13421    %}
13422   ins_pipe(ialu_reg_reg);
13423 %}
13424 
13425 // This pattern is automatically generated from aarch64_ad.m4
13426 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13427 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13428 %{
13429   match(Set dst (AddI src1 (AndI src2 mask)));
13430   ins_cost(INSN_COST);
13431   format %{ "addw  $dst, $src1, $src2, uxth" %}
13432 
13433    ins_encode %{
13434      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13435             as_Register($src2$$reg), ext::uxth);
13436    %}
13437   ins_pipe(ialu_reg_reg);
13438 %}
13439 
13440 // This pattern is automatically generated from aarch64_ad.m4
13441 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13442 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13443 %{
13444   match(Set dst (AddL src1 (AndL src2 mask)));
13445   ins_cost(INSN_COST);
13446   format %{ "add  $dst, $src1, $src2, uxtb" %}
13447 
13448    ins_encode %{
13449      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13450             as_Register($src2$$reg), ext::uxtb);
13451    %}
13452   ins_pipe(ialu_reg_reg);
13453 %}
13454 
13455 // This pattern is automatically generated from aarch64_ad.m4
13456 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13457 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13458 %{
13459   match(Set dst (AddL src1 (AndL src2 mask)));
13460   ins_cost(INSN_COST);
13461   format %{ "add  $dst, $src1, $src2, uxth" %}
13462 
13463    ins_encode %{
13464      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13465             as_Register($src2$$reg), ext::uxth);
13466    %}
13467   ins_pipe(ialu_reg_reg);
13468 %}
13469 
13470 // This pattern is automatically generated from aarch64_ad.m4
13471 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13472 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13473 %{
13474   match(Set dst (AddL src1 (AndL src2 mask)));
13475   ins_cost(INSN_COST);
13476   format %{ "add  $dst, $src1, $src2, uxtw" %}
13477 
13478    ins_encode %{
13479      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13480             as_Register($src2$$reg), ext::uxtw);
13481    %}
13482   ins_pipe(ialu_reg_reg);
13483 %}
13484 
13485 // This pattern is automatically generated from aarch64_ad.m4
13486 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13487 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13488 %{
13489   match(Set dst (SubI src1 (AndI src2 mask)));
13490   ins_cost(INSN_COST);
13491   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13492 
13493    ins_encode %{
13494      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13495             as_Register($src2$$reg), ext::uxtb);
13496    %}
13497   ins_pipe(ialu_reg_reg);
13498 %}
13499 
13500 // This pattern is automatically generated from aarch64_ad.m4
13501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13502 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13503 %{
13504   match(Set dst (SubI src1 (AndI src2 mask)));
13505   ins_cost(INSN_COST);
13506   format %{ "subw  $dst, $src1, $src2, uxth" %}
13507 
13508    ins_encode %{
13509      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13510             as_Register($src2$$reg), ext::uxth);
13511    %}
13512   ins_pipe(ialu_reg_reg);
13513 %}
13514 
13515 // This pattern is automatically generated from aarch64_ad.m4
13516 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13517 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13518 %{
13519   match(Set dst (SubL src1 (AndL src2 mask)));
13520   ins_cost(INSN_COST);
13521   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13522 
13523    ins_encode %{
13524      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13525             as_Register($src2$$reg), ext::uxtb);
13526    %}
13527   ins_pipe(ialu_reg_reg);
13528 %}
13529 
13530 // This pattern is automatically generated from aarch64_ad.m4
13531 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13532 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13533 %{
13534   match(Set dst (SubL src1 (AndL src2 mask)));
13535   ins_cost(INSN_COST);
13536   format %{ "sub  $dst, $src1, $src2, uxth" %}
13537 
13538    ins_encode %{
13539      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13540             as_Register($src2$$reg), ext::uxth);
13541    %}
13542   ins_pipe(ialu_reg_reg);
13543 %}
13544 
13545 // This pattern is automatically generated from aarch64_ad.m4
13546 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13547 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13548 %{
13549   match(Set dst (SubL src1 (AndL src2 mask)));
13550   ins_cost(INSN_COST);
13551   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13552 
13553    ins_encode %{
13554      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13555             as_Register($src2$$reg), ext::uxtw);
13556    %}
13557   ins_pipe(ialu_reg_reg);
13558 %}
13559 
13560 
13561 // This pattern is automatically generated from aarch64_ad.m4
13562 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13563 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13564 %{
13565   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13566   ins_cost(1.9 * INSN_COST);
13567   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13568 
13569    ins_encode %{
13570      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13571             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13572    %}
13573   ins_pipe(ialu_reg_reg_shift);
13574 %}
13575 
13576 // This pattern is automatically generated from aarch64_ad.m4
13577 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13578 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13579 %{
13580   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13581   ins_cost(1.9 * INSN_COST);
13582   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13583 
13584    ins_encode %{
13585      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13586             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13587    %}
13588   ins_pipe(ialu_reg_reg_shift);
13589 %}
13590 
13591 // This pattern is automatically generated from aarch64_ad.m4
13592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13593 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13594 %{
13595   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13596   ins_cost(1.9 * INSN_COST);
13597   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13598 
13599    ins_encode %{
13600      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13601             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13602    %}
13603   ins_pipe(ialu_reg_reg_shift);
13604 %}
13605 
13606 // This pattern is automatically generated from aarch64_ad.m4
13607 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13608 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13609 %{
13610   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13611   ins_cost(1.9 * INSN_COST);
13612   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13613 
13614    ins_encode %{
13615      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13616             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13617    %}
13618   ins_pipe(ialu_reg_reg_shift);
13619 %}
13620 
13621 // This pattern is automatically generated from aarch64_ad.m4
13622 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13623 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13624 %{
13625   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13626   ins_cost(1.9 * INSN_COST);
13627   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13628 
13629    ins_encode %{
13630      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13631             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13632    %}
13633   ins_pipe(ialu_reg_reg_shift);
13634 %}
13635 
13636 // This pattern is automatically generated from aarch64_ad.m4
13637 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13638 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13639 %{
13640   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13641   ins_cost(1.9 * INSN_COST);
13642   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13643 
13644    ins_encode %{
13645      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13646             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13647    %}
13648   ins_pipe(ialu_reg_reg_shift);
13649 %}
13650 
13651 // This pattern is automatically generated from aarch64_ad.m4
13652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13653 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13654 %{
13655   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13656   ins_cost(1.9 * INSN_COST);
13657   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13658 
13659    ins_encode %{
13660      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13661             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13662    %}
13663   ins_pipe(ialu_reg_reg_shift);
13664 %}
13665 
13666 // This pattern is automatically generated from aarch64_ad.m4
13667 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13668 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13669 %{
13670   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13671   ins_cost(1.9 * INSN_COST);
13672   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13673 
13674    ins_encode %{
13675      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13676             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13677    %}
13678   ins_pipe(ialu_reg_reg_shift);
13679 %}
13680 
13681 // This pattern is automatically generated from aarch64_ad.m4
13682 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13683 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13684 %{
13685   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13686   ins_cost(1.9 * INSN_COST);
13687   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13688 
13689    ins_encode %{
13690      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13691             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13692    %}
13693   ins_pipe(ialu_reg_reg_shift);
13694 %}
13695 
13696 // This pattern is automatically generated from aarch64_ad.m4
13697 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13698 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13699 %{
13700   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13701   ins_cost(1.9 * INSN_COST);
13702   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13703 
13704    ins_encode %{
13705      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13706             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13707    %}
13708   ins_pipe(ialu_reg_reg_shift);
13709 %}
13710 
13711 // This pattern is automatically generated from aarch64_ad.m4
13712 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13713 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13714 %{
13715   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13716   ins_cost(1.9 * INSN_COST);
13717   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13718 
13719    ins_encode %{
13720      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13721             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13722    %}
13723   ins_pipe(ialu_reg_reg_shift);
13724 %}
13725 
13726 // This pattern is automatically generated from aarch64_ad.m4
13727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13728 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13729 %{
13730   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13731   ins_cost(1.9 * INSN_COST);
13732   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13733 
13734    ins_encode %{
13735      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13736             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13737    %}
13738   ins_pipe(ialu_reg_reg_shift);
13739 %}
13740 
13741 // This pattern is automatically generated from aarch64_ad.m4
13742 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13743 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13744 %{
13745   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13746   ins_cost(1.9 * INSN_COST);
13747   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13748 
13749    ins_encode %{
13750      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13751             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13752    %}
13753   ins_pipe(ialu_reg_reg_shift);
13754 %}
13755 
13756 // This pattern is automatically generated from aarch64_ad.m4
13757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13758 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13759 %{
13760   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13761   ins_cost(1.9 * INSN_COST);
13762   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13763 
13764    ins_encode %{
13765      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13766             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13767    %}
13768   ins_pipe(ialu_reg_reg_shift);
13769 %}
13770 
13771 // This pattern is automatically generated from aarch64_ad.m4
13772 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13773 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13774 %{
13775   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13776   ins_cost(1.9 * INSN_COST);
13777   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13778 
13779    ins_encode %{
13780      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13781             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13782    %}
13783   ins_pipe(ialu_reg_reg_shift);
13784 %}
13785 
13786 // This pattern is automatically generated from aarch64_ad.m4
13787 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13788 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13789 %{
13790   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13791   ins_cost(1.9 * INSN_COST);
13792   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13793 
13794    ins_encode %{
13795      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13796             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13797    %}
13798   ins_pipe(ialu_reg_reg_shift);
13799 %}
13800 
13801 // This pattern is automatically generated from aarch64_ad.m4
13802 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13803 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13804 %{
13805   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13806   ins_cost(1.9 * INSN_COST);
13807   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13808 
13809    ins_encode %{
13810      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13811             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13812    %}
13813   ins_pipe(ialu_reg_reg_shift);
13814 %}
13815 
13816 // This pattern is automatically generated from aarch64_ad.m4
13817 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13818 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13819 %{
13820   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13821   ins_cost(1.9 * INSN_COST);
13822   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13823 
13824    ins_encode %{
13825      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13826             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13827    %}
13828   ins_pipe(ialu_reg_reg_shift);
13829 %}
13830 
13831 // This pattern is automatically generated from aarch64_ad.m4
13832 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13833 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13834 %{
13835   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13836   ins_cost(1.9 * INSN_COST);
13837   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13838 
13839    ins_encode %{
13840      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13841             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13842    %}
13843   ins_pipe(ialu_reg_reg_shift);
13844 %}
13845 
13846 // This pattern is automatically generated from aarch64_ad.m4
13847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13848 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13849 %{
13850   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13851   ins_cost(1.9 * INSN_COST);
13852   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13853 
13854    ins_encode %{
13855      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13856             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13857    %}
13858   ins_pipe(ialu_reg_reg_shift);
13859 %}
13860 
13861 // This pattern is automatically generated from aarch64_ad.m4
13862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13863 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13864 %{
13865   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13866   ins_cost(1.9 * INSN_COST);
13867   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13868 
13869    ins_encode %{
13870      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13871             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13872    %}
13873   ins_pipe(ialu_reg_reg_shift);
13874 %}
13875 
13876 // This pattern is automatically generated from aarch64_ad.m4
13877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13878 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13879 %{
13880   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13881   ins_cost(1.9 * INSN_COST);
13882   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13883 
13884    ins_encode %{
13885      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13886             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13887    %}
13888   ins_pipe(ialu_reg_reg_shift);
13889 %}
13890 
13891 // This pattern is automatically generated from aarch64_ad.m4
13892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13893 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13894 %{
13895   effect(DEF dst, USE src1, USE src2, USE cr);
13896   ins_cost(INSN_COST * 2);
13897   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
13898 
13899   ins_encode %{
13900     __ cselw($dst$$Register,
13901              $src1$$Register,
13902              $src2$$Register,
13903              Assembler::LT);
13904   %}
13905   ins_pipe(icond_reg_reg);
13906 %}
13907 
13908 // This pattern is automatically generated from aarch64_ad.m4
13909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13910 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13911 %{
13912   effect(DEF dst, USE src1, USE src2, USE cr);
13913   ins_cost(INSN_COST * 2);
13914   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
13915 
13916   ins_encode %{
13917     __ cselw($dst$$Register,
13918              $src1$$Register,
13919              $src2$$Register,
13920              Assembler::GT);
13921   %}
13922   ins_pipe(icond_reg_reg);
13923 %}
13924 
13925 // This pattern is automatically generated from aarch64_ad.m4
13926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13927 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13928 %{
13929   effect(DEF dst, USE src1, USE cr);
13930   ins_cost(INSN_COST * 2);
13931   format %{ "cselw $dst, $src1, zr lt\t"  %}
13932 
13933   ins_encode %{
13934     __ cselw($dst$$Register,
13935              $src1$$Register,
13936              zr,
13937              Assembler::LT);
13938   %}
13939   ins_pipe(icond_reg);
13940 %}
13941 
13942 // This pattern is automatically generated from aarch64_ad.m4
13943 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13944 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13945 %{
13946   effect(DEF dst, USE src1, USE cr);
13947   ins_cost(INSN_COST * 2);
13948   format %{ "cselw $dst, $src1, zr gt\t"  %}
13949 
13950   ins_encode %{
13951     __ cselw($dst$$Register,
13952              $src1$$Register,
13953              zr,
13954              Assembler::GT);
13955   %}
13956   ins_pipe(icond_reg);
13957 %}
13958 
13959 // This pattern is automatically generated from aarch64_ad.m4
13960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13961 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13962 %{
13963   effect(DEF dst, USE src1, USE cr);
13964   ins_cost(INSN_COST * 2);
13965   format %{ "csincw $dst, $src1, zr le\t"  %}
13966 
13967   ins_encode %{
13968     __ csincw($dst$$Register,
13969              $src1$$Register,
13970              zr,
13971              Assembler::LE);
13972   %}
13973   ins_pipe(icond_reg);
13974 %}
13975 
13976 // This pattern is automatically generated from aarch64_ad.m4
13977 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13978 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13979 %{
13980   effect(DEF dst, USE src1, USE cr);
13981   ins_cost(INSN_COST * 2);
13982   format %{ "csincw $dst, $src1, zr gt\t"  %}
13983 
13984   ins_encode %{
13985     __ csincw($dst$$Register,
13986              $src1$$Register,
13987              zr,
13988              Assembler::GT);
13989   %}
13990   ins_pipe(icond_reg);
13991 %}
13992 
13993 // This pattern is automatically generated from aarch64_ad.m4
13994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13995 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13996 %{
13997   effect(DEF dst, USE src1, USE cr);
13998   ins_cost(INSN_COST * 2);
13999   format %{ "csinvw $dst, $src1, zr lt\t"  %}
14000 
14001   ins_encode %{
14002     __ csinvw($dst$$Register,
14003              $src1$$Register,
14004              zr,
14005              Assembler::LT);
14006   %}
14007   ins_pipe(icond_reg);
14008 %}
14009 
14010 // This pattern is automatically generated from aarch64_ad.m4
14011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14012 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14013 %{
14014   effect(DEF dst, USE src1, USE cr);
14015   ins_cost(INSN_COST * 2);
14016   format %{ "csinvw $dst, $src1, zr ge\t"  %}
14017 
14018   ins_encode %{
14019     __ csinvw($dst$$Register,
14020              $src1$$Register,
14021              zr,
14022              Assembler::GE);
14023   %}
14024   ins_pipe(icond_reg);
14025 %}
14026 
14027 // This pattern is automatically generated from aarch64_ad.m4
14028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14029 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
14030 %{
14031   match(Set dst (MinI src imm));
14032   ins_cost(INSN_COST * 3);
14033   expand %{
14034     rFlagsReg cr;
14035     compI_reg_imm0(cr, src);
14036     cmovI_reg_imm0_lt(dst, src, cr);
14037   %}
14038 %}
14039 
14040 // This pattern is automatically generated from aarch64_ad.m4
14041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14042 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
14043 %{
14044   match(Set dst (MinI imm src));
14045   ins_cost(INSN_COST * 3);
14046   expand %{
14047     rFlagsReg cr;
14048     compI_reg_imm0(cr, src);
14049     cmovI_reg_imm0_lt(dst, src, cr);
14050   %}
14051 %}
14052 
14053 // This pattern is automatically generated from aarch64_ad.m4
14054 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14055 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
14056 %{
14057   match(Set dst (MinI src imm));
14058   ins_cost(INSN_COST * 3);
14059   expand %{
14060     rFlagsReg cr;
14061     compI_reg_imm0(cr, src);
14062     cmovI_reg_imm1_le(dst, src, cr);
14063   %}
14064 %}
14065 
14066 // This pattern is automatically generated from aarch64_ad.m4
14067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14068 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14069 %{
14070   match(Set dst (MinI imm src));
14071   ins_cost(INSN_COST * 3);
14072   expand %{
14073     rFlagsReg cr;
14074     compI_reg_imm0(cr, src);
14075     cmovI_reg_imm1_le(dst, src, cr);
14076   %}
14077 %}
14078 
14079 // This pattern is automatically generated from aarch64_ad.m4
14080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14081 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14082 %{
14083   match(Set dst (MinI src imm));
14084   ins_cost(INSN_COST * 3);
14085   expand %{
14086     rFlagsReg cr;
14087     compI_reg_imm0(cr, src);
14088     cmovI_reg_immM1_lt(dst, src, cr);
14089   %}
14090 %}
14091 
14092 // This pattern is automatically generated from aarch64_ad.m4
14093 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14094 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14095 %{
14096   match(Set dst (MinI imm src));
14097   ins_cost(INSN_COST * 3);
14098   expand %{
14099     rFlagsReg cr;
14100     compI_reg_imm0(cr, src);
14101     cmovI_reg_immM1_lt(dst, src, cr);
14102   %}
14103 %}
14104 
14105 // This pattern is automatically generated from aarch64_ad.m4
14106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14107 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
14108 %{
14109   match(Set dst (MaxI src imm));
14110   ins_cost(INSN_COST * 3);
14111   expand %{
14112     rFlagsReg cr;
14113     compI_reg_imm0(cr, src);
14114     cmovI_reg_imm0_gt(dst, src, cr);
14115   %}
14116 %}
14117 
14118 // This pattern is automatically generated from aarch64_ad.m4
14119 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14120 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
14121 %{
14122   match(Set dst (MaxI imm src));
14123   ins_cost(INSN_COST * 3);
14124   expand %{
14125     rFlagsReg cr;
14126     compI_reg_imm0(cr, src);
14127     cmovI_reg_imm0_gt(dst, src, cr);
14128   %}
14129 %}
14130 
14131 // This pattern is automatically generated from aarch64_ad.m4
14132 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14133 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
14134 %{
14135   match(Set dst (MaxI src imm));
14136   ins_cost(INSN_COST * 3);
14137   expand %{
14138     rFlagsReg cr;
14139     compI_reg_imm0(cr, src);
14140     cmovI_reg_imm1_gt(dst, src, cr);
14141   %}
14142 %}
14143 
14144 // This pattern is automatically generated from aarch64_ad.m4
14145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14146 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14147 %{
14148   match(Set dst (MaxI imm src));
14149   ins_cost(INSN_COST * 3);
14150   expand %{
14151     rFlagsReg cr;
14152     compI_reg_imm0(cr, src);
14153     cmovI_reg_imm1_gt(dst, src, cr);
14154   %}
14155 %}
14156 
14157 // This pattern is automatically generated from aarch64_ad.m4
14158 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14159 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14160 %{
14161   match(Set dst (MaxI src imm));
14162   ins_cost(INSN_COST * 3);
14163   expand %{
14164     rFlagsReg cr;
14165     compI_reg_imm0(cr, src);
14166     cmovI_reg_immM1_ge(dst, src, cr);
14167   %}
14168 %}
14169 
14170 // This pattern is automatically generated from aarch64_ad.m4
14171 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14172 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14173 %{
14174   match(Set dst (MaxI imm src));
14175   ins_cost(INSN_COST * 3);
14176   expand %{
14177     rFlagsReg cr;
14178     compI_reg_imm0(cr, src);
14179     cmovI_reg_immM1_ge(dst, src, cr);
14180   %}
14181 %}
14182 
14183 // This pattern is automatically generated from aarch64_ad.m4
14184 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14185 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
14186 %{
14187   match(Set dst (ReverseI src));
14188   ins_cost(INSN_COST);
14189   format %{ "rbitw  $dst, $src" %}
14190   ins_encode %{
14191     __ rbitw($dst$$Register, $src$$Register);
14192   %}
14193   ins_pipe(ialu_reg);
14194 %}
14195 
14196 // This pattern is automatically generated from aarch64_ad.m4
14197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14198 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
14199 %{
14200   match(Set dst (ReverseL src));
14201   ins_cost(INSN_COST);
14202   format %{ "rbit  $dst, $src" %}
14203   ins_encode %{
14204     __ rbit($dst$$Register, $src$$Register);
14205   %}
14206   ins_pipe(ialu_reg);
14207 %}
14208 
14209 
14210 // END This section of the file is automatically generated. Do not edit --------------
14211 
14212 
14213 // ============================================================================
14214 // Floating Point Arithmetic Instructions
14215 
14216 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14217   match(Set dst (AddF src1 src2));
14218 
14219   ins_cost(INSN_COST * 5);
14220   format %{ "fadds   $dst, $src1, $src2" %}
14221 
14222   ins_encode %{
14223     __ fadds(as_FloatRegister($dst$$reg),
14224              as_FloatRegister($src1$$reg),
14225              as_FloatRegister($src2$$reg));
14226   %}
14227 
14228   ins_pipe(fp_dop_reg_reg_s);
14229 %}
14230 
14231 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14232   match(Set dst (AddD src1 src2));
14233 
14234   ins_cost(INSN_COST * 5);
14235   format %{ "faddd   $dst, $src1, $src2" %}
14236 
14237   ins_encode %{
14238     __ faddd(as_FloatRegister($dst$$reg),
14239              as_FloatRegister($src1$$reg),
14240              as_FloatRegister($src2$$reg));
14241   %}
14242 
14243   ins_pipe(fp_dop_reg_reg_d);
14244 %}
14245 
14246 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14247   match(Set dst (SubF src1 src2));
14248 
14249   ins_cost(INSN_COST * 5);
14250   format %{ "fsubs   $dst, $src1, $src2" %}
14251 
14252   ins_encode %{
14253     __ fsubs(as_FloatRegister($dst$$reg),
14254              as_FloatRegister($src1$$reg),
14255              as_FloatRegister($src2$$reg));
14256   %}
14257 
14258   ins_pipe(fp_dop_reg_reg_s);
14259 %}
14260 
14261 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14262   match(Set dst (SubD src1 src2));
14263 
14264   ins_cost(INSN_COST * 5);
14265   format %{ "fsubd   $dst, $src1, $src2" %}
14266 
14267   ins_encode %{
14268     __ fsubd(as_FloatRegister($dst$$reg),
14269              as_FloatRegister($src1$$reg),
14270              as_FloatRegister($src2$$reg));
14271   %}
14272 
14273   ins_pipe(fp_dop_reg_reg_d);
14274 %}
14275 
14276 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14277   match(Set dst (MulF src1 src2));
14278 
14279   ins_cost(INSN_COST * 6);
14280   format %{ "fmuls   $dst, $src1, $src2" %}
14281 
14282   ins_encode %{
14283     __ fmuls(as_FloatRegister($dst$$reg),
14284              as_FloatRegister($src1$$reg),
14285              as_FloatRegister($src2$$reg));
14286   %}
14287 
14288   ins_pipe(fp_dop_reg_reg_s);
14289 %}
14290 
14291 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14292   match(Set dst (MulD src1 src2));
14293 
14294   ins_cost(INSN_COST * 6);
14295   format %{ "fmuld   $dst, $src1, $src2" %}
14296 
14297   ins_encode %{
14298     __ fmuld(as_FloatRegister($dst$$reg),
14299              as_FloatRegister($src1$$reg),
14300              as_FloatRegister($src2$$reg));
14301   %}
14302 
14303   ins_pipe(fp_dop_reg_reg_d);
14304 %}
14305 
14306 // src1 * src2 + src3
14307 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14308   predicate(UseFMA);
14309   match(Set dst (FmaF src3 (Binary src1 src2)));
14310 
14311   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
14312 
14313   ins_encode %{
14314     __ fmadds(as_FloatRegister($dst$$reg),
14315              as_FloatRegister($src1$$reg),
14316              as_FloatRegister($src2$$reg),
14317              as_FloatRegister($src3$$reg));
14318   %}
14319 
14320   ins_pipe(pipe_class_default);
14321 %}
14322 
14323 // src1 * src2 + src3
14324 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14325   predicate(UseFMA);
14326   match(Set dst (FmaD src3 (Binary src1 src2)));
14327 
14328   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
14329 
14330   ins_encode %{
14331     __ fmaddd(as_FloatRegister($dst$$reg),
14332              as_FloatRegister($src1$$reg),
14333              as_FloatRegister($src2$$reg),
14334              as_FloatRegister($src3$$reg));
14335   %}
14336 
14337   ins_pipe(pipe_class_default);
14338 %}
14339 
14340 // -src1 * src2 + src3
14341 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14342   predicate(UseFMA);
14343   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14344   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14345 
14346   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
14347 
14348   ins_encode %{
14349     __ fmsubs(as_FloatRegister($dst$$reg),
14350               as_FloatRegister($src1$$reg),
14351               as_FloatRegister($src2$$reg),
14352               as_FloatRegister($src3$$reg));
14353   %}
14354 
14355   ins_pipe(pipe_class_default);
14356 %}
14357 
14358 // -src1 * src2 + src3
14359 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14360   predicate(UseFMA);
14361   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14362   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14363 
14364   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
14365 
14366   ins_encode %{
14367     __ fmsubd(as_FloatRegister($dst$$reg),
14368               as_FloatRegister($src1$$reg),
14369               as_FloatRegister($src2$$reg),
14370               as_FloatRegister($src3$$reg));
14371   %}
14372 
14373   ins_pipe(pipe_class_default);
14374 %}
14375 
14376 // -src1 * src2 - src3
14377 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14378   predicate(UseFMA);
14379   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14380   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14381 
14382   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14383 
14384   ins_encode %{
14385     __ fnmadds(as_FloatRegister($dst$$reg),
14386                as_FloatRegister($src1$$reg),
14387                as_FloatRegister($src2$$reg),
14388                as_FloatRegister($src3$$reg));
14389   %}
14390 
14391   ins_pipe(pipe_class_default);
14392 %}
14393 
14394 // -src1 * src2 - src3
14395 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14396   predicate(UseFMA);
14397   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14398   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14399 
14400   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14401 
14402   ins_encode %{
14403     __ fnmaddd(as_FloatRegister($dst$$reg),
14404                as_FloatRegister($src1$$reg),
14405                as_FloatRegister($src2$$reg),
14406                as_FloatRegister($src3$$reg));
14407   %}
14408 
14409   ins_pipe(pipe_class_default);
14410 %}
14411 
14412 // src1 * src2 - src3
14413 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14414   predicate(UseFMA);
14415   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14416 
14417   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14418 
14419   ins_encode %{
14420     __ fnmsubs(as_FloatRegister($dst$$reg),
14421                as_FloatRegister($src1$$reg),
14422                as_FloatRegister($src2$$reg),
14423                as_FloatRegister($src3$$reg));
14424   %}
14425 
14426   ins_pipe(pipe_class_default);
14427 %}
14428 
14429 // src1 * src2 - src3
14430 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14431   predicate(UseFMA);
14432   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14433 
14434   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14435 
14436   ins_encode %{
14437   // n.b. insn name should be fnmsubd
14438     __ fnmsub(as_FloatRegister($dst$$reg),
14439               as_FloatRegister($src1$$reg),
14440               as_FloatRegister($src2$$reg),
14441               as_FloatRegister($src3$$reg));
14442   %}
14443 
14444   ins_pipe(pipe_class_default);
14445 %}
14446 
14447 
14448 // Math.max(FF)F
14449 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14450   match(Set dst (MaxF src1 src2));
14451 
14452   format %{ "fmaxs   $dst, $src1, $src2" %}
14453   ins_encode %{
14454     __ fmaxs(as_FloatRegister($dst$$reg),
14455              as_FloatRegister($src1$$reg),
14456              as_FloatRegister($src2$$reg));
14457   %}
14458 
14459   ins_pipe(fp_dop_reg_reg_s);
14460 %}
14461 
14462 // Math.min(FF)F
14463 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14464   match(Set dst (MinF src1 src2));
14465 
14466   format %{ "fmins   $dst, $src1, $src2" %}
14467   ins_encode %{
14468     __ fmins(as_FloatRegister($dst$$reg),
14469              as_FloatRegister($src1$$reg),
14470              as_FloatRegister($src2$$reg));
14471   %}
14472 
14473   ins_pipe(fp_dop_reg_reg_s);
14474 %}
14475 
14476 // Math.max(DD)D
14477 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14478   match(Set dst (MaxD src1 src2));
14479 
14480   format %{ "fmaxd   $dst, $src1, $src2" %}
14481   ins_encode %{
14482     __ fmaxd(as_FloatRegister($dst$$reg),
14483              as_FloatRegister($src1$$reg),
14484              as_FloatRegister($src2$$reg));
14485   %}
14486 
14487   ins_pipe(fp_dop_reg_reg_d);
14488 %}
14489 
14490 // Math.min(DD)D
14491 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14492   match(Set dst (MinD src1 src2));
14493 
14494   format %{ "fmind   $dst, $src1, $src2" %}
14495   ins_encode %{
14496     __ fmind(as_FloatRegister($dst$$reg),
14497              as_FloatRegister($src1$$reg),
14498              as_FloatRegister($src2$$reg));
14499   %}
14500 
14501   ins_pipe(fp_dop_reg_reg_d);
14502 %}
14503 
14504 
14505 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14506   match(Set dst (DivF src1  src2));
14507 
14508   ins_cost(INSN_COST * 18);
14509   format %{ "fdivs   $dst, $src1, $src2" %}
14510 
14511   ins_encode %{
14512     __ fdivs(as_FloatRegister($dst$$reg),
14513              as_FloatRegister($src1$$reg),
14514              as_FloatRegister($src2$$reg));
14515   %}
14516 
14517   ins_pipe(fp_div_s);
14518 %}
14519 
14520 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14521   match(Set dst (DivD src1  src2));
14522 
14523   ins_cost(INSN_COST * 32);
14524   format %{ "fdivd   $dst, $src1, $src2" %}
14525 
14526   ins_encode %{
14527     __ fdivd(as_FloatRegister($dst$$reg),
14528              as_FloatRegister($src1$$reg),
14529              as_FloatRegister($src2$$reg));
14530   %}
14531 
14532   ins_pipe(fp_div_d);
14533 %}
14534 
14535 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14536   match(Set dst (NegF src));
14537 
14538   ins_cost(INSN_COST * 3);
14539   format %{ "fneg   $dst, $src" %}
14540 
14541   ins_encode %{
14542     __ fnegs(as_FloatRegister($dst$$reg),
14543              as_FloatRegister($src$$reg));
14544   %}
14545 
14546   ins_pipe(fp_uop_s);
14547 %}
14548 
14549 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14550   match(Set dst (NegD src));
14551 
14552   ins_cost(INSN_COST * 3);
14553   format %{ "fnegd   $dst, $src" %}
14554 
14555   ins_encode %{
14556     __ fnegd(as_FloatRegister($dst$$reg),
14557              as_FloatRegister($src$$reg));
14558   %}
14559 
14560   ins_pipe(fp_uop_d);
14561 %}
14562 
14563 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14564 %{
14565   match(Set dst (AbsI src));
14566 
14567   effect(KILL cr);
14568   ins_cost(INSN_COST * 2);
14569   format %{ "cmpw  $src, zr\n\t"
14570             "cnegw $dst, $src, Assembler::LT\t# int abs"
14571   %}
14572 
14573   ins_encode %{
14574     __ cmpw(as_Register($src$$reg), zr);
14575     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14576   %}
14577   ins_pipe(pipe_class_default);
14578 %}
14579 
14580 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14581 %{
14582   match(Set dst (AbsL src));
14583 
14584   effect(KILL cr);
14585   ins_cost(INSN_COST * 2);
14586   format %{ "cmp  $src, zr\n\t"
14587             "cneg $dst, $src, Assembler::LT\t# long abs"
14588   %}
14589 
14590   ins_encode %{
14591     __ cmp(as_Register($src$$reg), zr);
14592     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14593   %}
14594   ins_pipe(pipe_class_default);
14595 %}
14596 
14597 instruct absF_reg(vRegF dst, vRegF src) %{
14598   match(Set dst (AbsF src));
14599 
14600   ins_cost(INSN_COST * 3);
14601   format %{ "fabss   $dst, $src" %}
14602   ins_encode %{
14603     __ fabss(as_FloatRegister($dst$$reg),
14604              as_FloatRegister($src$$reg));
14605   %}
14606 
14607   ins_pipe(fp_uop_s);
14608 %}
14609 
14610 instruct absD_reg(vRegD dst, vRegD src) %{
14611   match(Set dst (AbsD src));
14612 
14613   ins_cost(INSN_COST * 3);
14614   format %{ "fabsd   $dst, $src" %}
14615   ins_encode %{
14616     __ fabsd(as_FloatRegister($dst$$reg),
14617              as_FloatRegister($src$$reg));
14618   %}
14619 
14620   ins_pipe(fp_uop_d);
14621 %}
14622 
14623 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14624   match(Set dst (AbsF (SubF src1 src2)));
14625 
14626   ins_cost(INSN_COST * 3);
14627   format %{ "fabds   $dst, $src1, $src2" %}
14628   ins_encode %{
14629     __ fabds(as_FloatRegister($dst$$reg),
14630              as_FloatRegister($src1$$reg),
14631              as_FloatRegister($src2$$reg));
14632   %}
14633 
14634   ins_pipe(fp_uop_s);
14635 %}
14636 
14637 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14638   match(Set dst (AbsD (SubD src1 src2)));
14639 
14640   ins_cost(INSN_COST * 3);
14641   format %{ "fabdd   $dst, $src1, $src2" %}
14642   ins_encode %{
14643     __ fabdd(as_FloatRegister($dst$$reg),
14644              as_FloatRegister($src1$$reg),
14645              as_FloatRegister($src2$$reg));
14646   %}
14647 
14648   ins_pipe(fp_uop_d);
14649 %}
14650 
14651 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14652   match(Set dst (SqrtD src));
14653 
14654   ins_cost(INSN_COST * 50);
14655   format %{ "fsqrtd  $dst, $src" %}
14656   ins_encode %{
14657     __ fsqrtd(as_FloatRegister($dst$$reg),
14658              as_FloatRegister($src$$reg));
14659   %}
14660 
14661   ins_pipe(fp_div_s);
14662 %}
14663 
14664 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14665   match(Set dst (SqrtF src));
14666 
14667   ins_cost(INSN_COST * 50);
14668   format %{ "fsqrts  $dst, $src" %}
14669   ins_encode %{
14670     __ fsqrts(as_FloatRegister($dst$$reg),
14671              as_FloatRegister($src$$reg));
14672   %}
14673 
14674   ins_pipe(fp_div_d);
14675 %}
14676 
14677 // Math.rint, floor, ceil
14678 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14679   match(Set dst (RoundDoubleMode src rmode));
14680   format %{ "frint  $dst, $src, $rmode" %}
14681   ins_encode %{
14682     switch ($rmode$$constant) {
14683       case RoundDoubleModeNode::rmode_rint:
14684         __ frintnd(as_FloatRegister($dst$$reg),
14685                    as_FloatRegister($src$$reg));
14686         break;
14687       case RoundDoubleModeNode::rmode_floor:
14688         __ frintmd(as_FloatRegister($dst$$reg),
14689                    as_FloatRegister($src$$reg));
14690         break;
14691       case RoundDoubleModeNode::rmode_ceil:
14692         __ frintpd(as_FloatRegister($dst$$reg),
14693                    as_FloatRegister($src$$reg));
14694         break;
14695     }
14696   %}
14697   ins_pipe(fp_uop_d);
14698 %}
14699 
14700 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14701   match(Set dst (CopySignD src1 (Binary src2 zero)));
14702   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14703   format %{ "CopySignD  $dst $src1 $src2" %}
14704   ins_encode %{
14705     FloatRegister dst = as_FloatRegister($dst$$reg),
14706                   src1 = as_FloatRegister($src1$$reg),
14707                   src2 = as_FloatRegister($src2$$reg),
14708                   zero = as_FloatRegister($zero$$reg);
14709     __ fnegd(dst, zero);
14710     __ bsl(dst, __ T8B, src2, src1);
14711   %}
14712   ins_pipe(fp_uop_d);
14713 %}
14714 
14715 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14716   match(Set dst (CopySignF src1 src2));
14717   effect(TEMP_DEF dst, USE src1, USE src2);
14718   format %{ "CopySignF  $dst $src1 $src2" %}
14719   ins_encode %{
14720     FloatRegister dst = as_FloatRegister($dst$$reg),
14721                   src1 = as_FloatRegister($src1$$reg),
14722                   src2 = as_FloatRegister($src2$$reg);
14723     __ movi(dst, __ T2S, 0x80, 24);
14724     __ bsl(dst, __ T8B, src2, src1);
14725   %}
14726   ins_pipe(fp_uop_d);
14727 %}
14728 
14729 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14730   match(Set dst (SignumD src (Binary zero one)));
14731   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14732   format %{ "signumD  $dst, $src" %}
14733   ins_encode %{
14734     FloatRegister src = as_FloatRegister($src$$reg),
14735                   dst = as_FloatRegister($dst$$reg),
14736                   zero = as_FloatRegister($zero$$reg),
14737                   one = as_FloatRegister($one$$reg);
14738     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14739     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14740     // Bit selection instruction gets bit from "one" for each enabled bit in
14741     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14742     // NaN the whole "src" will be copied because "dst" is zero. For all other
14743     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14744     // from "src", and all other bits are copied from 1.0.
14745     __ bsl(dst, __ T8B, one, src);
14746   %}
14747   ins_pipe(fp_uop_d);
14748 %}
14749 
14750 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14751   match(Set dst (SignumF src (Binary zero one)));
14752   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14753   format %{ "signumF  $dst, $src" %}
14754   ins_encode %{
14755     FloatRegister src = as_FloatRegister($src$$reg),
14756                   dst = as_FloatRegister($dst$$reg),
14757                   zero = as_FloatRegister($zero$$reg),
14758                   one = as_FloatRegister($one$$reg);
14759     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14760     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14761     // Bit selection instruction gets bit from "one" for each enabled bit in
14762     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14763     // NaN the whole "src" will be copied because "dst" is zero. For all other
14764     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14765     // from "src", and all other bits are copied from 1.0.
14766     __ bsl(dst, __ T8B, one, src);
14767   %}
14768   ins_pipe(fp_uop_d);
14769 %}
14770 
14771 instruct onspinwait() %{
14772   match(OnSpinWait);
14773   ins_cost(INSN_COST);
14774 
14775   format %{ "onspinwait" %}
14776 
14777   ins_encode %{
14778     __ spin_wait();
14779   %}
14780   ins_pipe(pipe_class_empty);
14781 %}
14782 
14783 // ============================================================================
14784 // Logical Instructions
14785 
14786 // Integer Logical Instructions
14787 
14788 // And Instructions
14789 
14790 
14791 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14792   match(Set dst (AndI src1 src2));
14793 
14794   format %{ "andw  $dst, $src1, $src2\t# int" %}
14795 
14796   ins_cost(INSN_COST);
14797   ins_encode %{
14798     __ andw(as_Register($dst$$reg),
14799             as_Register($src1$$reg),
14800             as_Register($src2$$reg));
14801   %}
14802 
14803   ins_pipe(ialu_reg_reg);
14804 %}
14805 
14806 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14807   match(Set dst (AndI src1 src2));
14808 
14809   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14810 
14811   ins_cost(INSN_COST);
14812   ins_encode %{
14813     __ andw(as_Register($dst$$reg),
14814             as_Register($src1$$reg),
14815             (uint64_t)($src2$$constant));
14816   %}
14817 
14818   ins_pipe(ialu_reg_imm);
14819 %}
14820 
14821 // Or Instructions
14822 
14823 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14824   match(Set dst (OrI src1 src2));
14825 
14826   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14827 
14828   ins_cost(INSN_COST);
14829   ins_encode %{
14830     __ orrw(as_Register($dst$$reg),
14831             as_Register($src1$$reg),
14832             as_Register($src2$$reg));
14833   %}
14834 
14835   ins_pipe(ialu_reg_reg);
14836 %}
14837 
14838 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14839   match(Set dst (OrI src1 src2));
14840 
14841   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14842 
14843   ins_cost(INSN_COST);
14844   ins_encode %{
14845     __ orrw(as_Register($dst$$reg),
14846             as_Register($src1$$reg),
14847             (uint64_t)($src2$$constant));
14848   %}
14849 
14850   ins_pipe(ialu_reg_imm);
14851 %}
14852 
14853 // Xor Instructions
14854 
14855 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14856   match(Set dst (XorI src1 src2));
14857 
14858   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14859 
14860   ins_cost(INSN_COST);
14861   ins_encode %{
14862     __ eorw(as_Register($dst$$reg),
14863             as_Register($src1$$reg),
14864             as_Register($src2$$reg));
14865   %}
14866 
14867   ins_pipe(ialu_reg_reg);
14868 %}
14869 
14870 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14871   match(Set dst (XorI src1 src2));
14872 
14873   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14874 
14875   ins_cost(INSN_COST);
14876   ins_encode %{
14877     __ eorw(as_Register($dst$$reg),
14878             as_Register($src1$$reg),
14879             (uint64_t)($src2$$constant));
14880   %}
14881 
14882   ins_pipe(ialu_reg_imm);
14883 %}
14884 
14885 // Long Logical Instructions
14886 // TODO
14887 
14888 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14889   match(Set dst (AndL src1 src2));
14890 
14891   format %{ "and  $dst, $src1, $src2\t# int" %}
14892 
14893   ins_cost(INSN_COST);
14894   ins_encode %{
14895     __ andr(as_Register($dst$$reg),
14896             as_Register($src1$$reg),
14897             as_Register($src2$$reg));
14898   %}
14899 
14900   ins_pipe(ialu_reg_reg);
14901 %}
14902 
14903 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14904   match(Set dst (AndL src1 src2));
14905 
14906   format %{ "and  $dst, $src1, $src2\t# int" %}
14907 
14908   ins_cost(INSN_COST);
14909   ins_encode %{
14910     __ andr(as_Register($dst$$reg),
14911             as_Register($src1$$reg),
14912             (uint64_t)($src2$$constant));
14913   %}
14914 
14915   ins_pipe(ialu_reg_imm);
14916 %}
14917 
14918 // Or Instructions
14919 
14920 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14921   match(Set dst (OrL src1 src2));
14922 
14923   format %{ "orr  $dst, $src1, $src2\t# int" %}
14924 
14925   ins_cost(INSN_COST);
14926   ins_encode %{
14927     __ orr(as_Register($dst$$reg),
14928            as_Register($src1$$reg),
14929            as_Register($src2$$reg));
14930   %}
14931 
14932   ins_pipe(ialu_reg_reg);
14933 %}
14934 
14935 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14936   match(Set dst (OrL src1 src2));
14937 
14938   format %{ "orr  $dst, $src1, $src2\t# int" %}
14939 
14940   ins_cost(INSN_COST);
14941   ins_encode %{
14942     __ orr(as_Register($dst$$reg),
14943            as_Register($src1$$reg),
14944            (uint64_t)($src2$$constant));
14945   %}
14946 
14947   ins_pipe(ialu_reg_imm);
14948 %}
14949 
14950 // Xor Instructions
14951 
14952 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14953   match(Set dst (XorL src1 src2));
14954 
14955   format %{ "eor  $dst, $src1, $src2\t# int" %}
14956 
14957   ins_cost(INSN_COST);
14958   ins_encode %{
14959     __ eor(as_Register($dst$$reg),
14960            as_Register($src1$$reg),
14961            as_Register($src2$$reg));
14962   %}
14963 
14964   ins_pipe(ialu_reg_reg);
14965 %}
14966 
14967 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14968   match(Set dst (XorL src1 src2));
14969 
14970   ins_cost(INSN_COST);
14971   format %{ "eor  $dst, $src1, $src2\t# int" %}
14972 
14973   ins_encode %{
14974     __ eor(as_Register($dst$$reg),
14975            as_Register($src1$$reg),
14976            (uint64_t)($src2$$constant));
14977   %}
14978 
14979   ins_pipe(ialu_reg_imm);
14980 %}
14981 
14982 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14983 %{
14984   match(Set dst (ConvI2L src));
14985 
14986   ins_cost(INSN_COST);
14987   format %{ "sxtw  $dst, $src\t# i2l" %}
14988   ins_encode %{
14989     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14990   %}
14991   ins_pipe(ialu_reg_shift);
14992 %}
14993 
14994 // this pattern occurs in bigmath arithmetic
14995 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14996 %{
14997   match(Set dst (AndL (ConvI2L src) mask));
14998 
14999   ins_cost(INSN_COST);
15000   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
15001   ins_encode %{
15002     __ ubfm($dst$$Register, $src$$Register, 0, 31);
15003   %}
15004 
15005   ins_pipe(ialu_reg_shift);
15006 %}
15007 
15008 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
15009   match(Set dst (ConvL2I src));
15010 
15011   ins_cost(INSN_COST);
15012   format %{ "movw  $dst, $src \t// l2i" %}
15013 
15014   ins_encode %{
15015     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
15016   %}
15017 
15018   ins_pipe(ialu_reg);
15019 %}
15020 
15021 instruct convD2F_reg(vRegF dst, vRegD src) %{
15022   match(Set dst (ConvD2F src));
15023 
15024   ins_cost(INSN_COST * 5);
15025   format %{ "fcvtd  $dst, $src \t// d2f" %}
15026 
15027   ins_encode %{
15028     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15029   %}
15030 
15031   ins_pipe(fp_d2f);
15032 %}
15033 
15034 instruct convF2D_reg(vRegD dst, vRegF src) %{
15035   match(Set dst (ConvF2D src));
15036 
15037   ins_cost(INSN_COST * 5);
15038   format %{ "fcvts  $dst, $src \t// f2d" %}
15039 
15040   ins_encode %{
15041     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15042   %}
15043 
15044   ins_pipe(fp_f2d);
15045 %}
15046 
15047 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15048   match(Set dst (ConvF2I src));
15049 
15050   ins_cost(INSN_COST * 5);
15051   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
15052 
15053   ins_encode %{
15054     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15055   %}
15056 
15057   ins_pipe(fp_f2i);
15058 %}
15059 
15060 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
15061   match(Set dst (ConvF2L src));
15062 
15063   ins_cost(INSN_COST * 5);
15064   format %{ "fcvtzs  $dst, $src \t// f2l" %}
15065 
15066   ins_encode %{
15067     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15068   %}
15069 
15070   ins_pipe(fp_f2l);
15071 %}
15072 
15073 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
15074   match(Set dst (ConvF2HF src));
15075   format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
15076             "smov $dst, $tmp\t# move result from $tmp to $dst"
15077   %}
15078   effect(TEMP tmp);
15079   ins_encode %{
15080       __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
15081   %}
15082   ins_pipe(pipe_slow);
15083 %}
15084 
15085 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
15086   match(Set dst (ConvHF2F src));
15087   format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
15088             "fcvt $dst, $tmp\t# convert half to single precision"
15089   %}
15090   effect(TEMP tmp);
15091   ins_encode %{
15092       __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
15093   %}
15094   ins_pipe(pipe_slow);
15095 %}
15096 
15097 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
15098   match(Set dst (ConvI2F src));
15099 
15100   ins_cost(INSN_COST * 5);
15101   format %{ "scvtfws  $dst, $src \t// i2f" %}
15102 
15103   ins_encode %{
15104     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15105   %}
15106 
15107   ins_pipe(fp_i2f);
15108 %}
15109 
15110 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
15111   match(Set dst (ConvL2F src));
15112 
15113   ins_cost(INSN_COST * 5);
15114   format %{ "scvtfs  $dst, $src \t// l2f" %}
15115 
15116   ins_encode %{
15117     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15118   %}
15119 
15120   ins_pipe(fp_l2f);
15121 %}
15122 
15123 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
15124   match(Set dst (ConvD2I src));
15125 
15126   ins_cost(INSN_COST * 5);
15127   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
15128 
15129   ins_encode %{
15130     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15131   %}
15132 
15133   ins_pipe(fp_d2i);
15134 %}
15135 
15136 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15137   match(Set dst (ConvD2L src));
15138 
15139   ins_cost(INSN_COST * 5);
15140   format %{ "fcvtzd  $dst, $src \t// d2l" %}
15141 
15142   ins_encode %{
15143     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15144   %}
15145 
15146   ins_pipe(fp_d2l);
15147 %}
15148 
15149 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
15150   match(Set dst (ConvI2D src));
15151 
15152   ins_cost(INSN_COST * 5);
15153   format %{ "scvtfwd  $dst, $src \t// i2d" %}
15154 
15155   ins_encode %{
15156     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15157   %}
15158 
15159   ins_pipe(fp_i2d);
15160 %}
15161 
15162 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
15163   match(Set dst (ConvL2D src));
15164 
15165   ins_cost(INSN_COST * 5);
15166   format %{ "scvtfd  $dst, $src \t// l2d" %}
15167 
15168   ins_encode %{
15169     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15170   %}
15171 
15172   ins_pipe(fp_l2d);
15173 %}
15174 
15175 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
15176 %{
15177   match(Set dst (RoundD src));
15178   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15179   format %{ "java_round_double $dst,$src"%}
15180   ins_encode %{
15181     __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
15182                          as_FloatRegister($ftmp$$reg));
15183   %}
15184   ins_pipe(pipe_slow);
15185 %}
15186 
15187 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
15188 %{
15189   match(Set dst (RoundF src));
15190   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15191   format %{ "java_round_float $dst,$src"%}
15192   ins_encode %{
15193     __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
15194                         as_FloatRegister($ftmp$$reg));
15195   %}
15196   ins_pipe(pipe_slow);
15197 %}
15198 
15199 // stack <-> reg and reg <-> reg shuffles with no conversion
15200 
15201 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15202 
15203   match(Set dst (MoveF2I src));
15204 
15205   effect(DEF dst, USE src);
15206 
15207   ins_cost(4 * INSN_COST);
15208 
15209   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15210 
15211   ins_encode %{
15212     __ ldrw($dst$$Register, Address(sp, $src$$disp));
15213   %}
15214 
15215   ins_pipe(iload_reg_reg);
15216 
15217 %}
15218 
15219 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15220 
15221   match(Set dst (MoveI2F src));
15222 
15223   effect(DEF dst, USE src);
15224 
15225   ins_cost(4 * INSN_COST);
15226 
15227   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15228 
15229   ins_encode %{
15230     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15231   %}
15232 
15233   ins_pipe(pipe_class_memory);
15234 
15235 %}
15236 
15237 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15238 
15239   match(Set dst (MoveD2L src));
15240 
15241   effect(DEF dst, USE src);
15242 
15243   ins_cost(4 * INSN_COST);
15244 
15245   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15246 
15247   ins_encode %{
15248     __ ldr($dst$$Register, Address(sp, $src$$disp));
15249   %}
15250 
15251   ins_pipe(iload_reg_reg);
15252 
15253 %}
15254 
15255 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15256 
15257   match(Set dst (MoveL2D src));
15258 
15259   effect(DEF dst, USE src);
15260 
15261   ins_cost(4 * INSN_COST);
15262 
15263   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15264 
15265   ins_encode %{
15266     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15267   %}
15268 
15269   ins_pipe(pipe_class_memory);
15270 
15271 %}
15272 
15273 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15274 
15275   match(Set dst (MoveF2I src));
15276 
15277   effect(DEF dst, USE src);
15278 
15279   ins_cost(INSN_COST);
15280 
15281   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15282 
15283   ins_encode %{
15284     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15285   %}
15286 
15287   ins_pipe(pipe_class_memory);
15288 
15289 %}
15290 
15291 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15292 
15293   match(Set dst (MoveI2F src));
15294 
15295   effect(DEF dst, USE src);
15296 
15297   ins_cost(INSN_COST);
15298 
15299   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15300 
15301   ins_encode %{
15302     __ strw($src$$Register, Address(sp, $dst$$disp));
15303   %}
15304 
15305   ins_pipe(istore_reg_reg);
15306 
15307 %}
15308 
15309 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15310 
15311   match(Set dst (MoveD2L src));
15312 
15313   effect(DEF dst, USE src);
15314 
15315   ins_cost(INSN_COST);
15316 
15317   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15318 
15319   ins_encode %{
15320     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15321   %}
15322 
15323   ins_pipe(pipe_class_memory);
15324 
15325 %}
15326 
15327 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15328 
15329   match(Set dst (MoveL2D src));
15330 
15331   effect(DEF dst, USE src);
15332 
15333   ins_cost(INSN_COST);
15334 
15335   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15336 
15337   ins_encode %{
15338     __ str($src$$Register, Address(sp, $dst$$disp));
15339   %}
15340 
15341   ins_pipe(istore_reg_reg);
15342 
15343 %}
15344 
15345 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15346 
15347   match(Set dst (MoveF2I src));
15348 
15349   effect(DEF dst, USE src);
15350 
15351   ins_cost(INSN_COST);
15352 
15353   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15354 
15355   ins_encode %{
15356     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15357   %}
15358 
15359   ins_pipe(fp_f2i);
15360 
15361 %}
15362 
15363 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15364 
15365   match(Set dst (MoveI2F src));
15366 
15367   effect(DEF dst, USE src);
15368 
15369   ins_cost(INSN_COST);
15370 
15371   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15372 
15373   ins_encode %{
15374     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15375   %}
15376 
15377   ins_pipe(fp_i2f);
15378 
15379 %}
15380 
15381 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15382 
15383   match(Set dst (MoveD2L src));
15384 
15385   effect(DEF dst, USE src);
15386 
15387   ins_cost(INSN_COST);
15388 
15389   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15390 
15391   ins_encode %{
15392     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15393   %}
15394 
15395   ins_pipe(fp_d2l);
15396 
15397 %}
15398 
15399 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15400 
15401   match(Set dst (MoveL2D src));
15402 
15403   effect(DEF dst, USE src);
15404 
15405   ins_cost(INSN_COST);
15406 
15407   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15408 
15409   ins_encode %{
15410     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15411   %}
15412 
15413   ins_pipe(fp_l2d);
15414 
15415 %}
15416 
15417 // ============================================================================
15418 // clearing of an array
15419 
15420 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15421 %{
15422   match(Set dummy (ClearArray cnt base));
15423   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15424 
15425   ins_cost(4 * INSN_COST);
15426   format %{ "ClearArray $cnt, $base" %}
15427 
15428   ins_encode %{
15429     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15430     if (tpc == NULL) {
15431       ciEnv::current()->record_failure("CodeCache is full");
15432       return;
15433     }
15434   %}
15435 
15436   ins_pipe(pipe_class_memory);
15437 %}
15438 
15439 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15440 %{
15441   predicate((uint64_t)n->in(2)->get_long()
15442             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15443   match(Set dummy (ClearArray cnt base));
15444   effect(TEMP temp, USE_KILL base, KILL cr);
15445 
15446   ins_cost(4 * INSN_COST);
15447   format %{ "ClearArray $cnt, $base" %}
15448 
15449   ins_encode %{
15450     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15451     if (tpc == NULL) {
15452       ciEnv::current()->record_failure("CodeCache is full");
15453       return;
15454     }
15455   %}
15456 
15457   ins_pipe(pipe_class_memory);
15458 %}
15459 
15460 // ============================================================================
15461 // Overflow Math Instructions
15462 
15463 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15464 %{
15465   match(Set cr (OverflowAddI op1 op2));
15466 
15467   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15468   ins_cost(INSN_COST);
15469   ins_encode %{
15470     __ cmnw($op1$$Register, $op2$$Register);
15471   %}
15472 
15473   ins_pipe(icmp_reg_reg);
15474 %}
15475 
15476 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15477 %{
15478   match(Set cr (OverflowAddI op1 op2));
15479 
15480   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15481   ins_cost(INSN_COST);
15482   ins_encode %{
15483     __ cmnw($op1$$Register, $op2$$constant);
15484   %}
15485 
15486   ins_pipe(icmp_reg_imm);
15487 %}
15488 
15489 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15490 %{
15491   match(Set cr (OverflowAddL op1 op2));
15492 
15493   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15494   ins_cost(INSN_COST);
15495   ins_encode %{
15496     __ cmn($op1$$Register, $op2$$Register);
15497   %}
15498 
15499   ins_pipe(icmp_reg_reg);
15500 %}
15501 
15502 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15503 %{
15504   match(Set cr (OverflowAddL op1 op2));
15505 
15506   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
15507   ins_cost(INSN_COST);
15508   ins_encode %{
15509     __ adds(zr, $op1$$Register, $op2$$constant);
15510   %}
15511 
15512   ins_pipe(icmp_reg_imm);
15513 %}
15514 
15515 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15516 %{
15517   match(Set cr (OverflowSubI op1 op2));
15518 
15519   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15520   ins_cost(INSN_COST);
15521   ins_encode %{
15522     __ cmpw($op1$$Register, $op2$$Register);
15523   %}
15524 
15525   ins_pipe(icmp_reg_reg);
15526 %}
15527 
15528 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15529 %{
15530   match(Set cr (OverflowSubI op1 op2));
15531 
15532   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15533   ins_cost(INSN_COST);
15534   ins_encode %{
15535     __ cmpw($op1$$Register, $op2$$constant);
15536   %}
15537 
15538   ins_pipe(icmp_reg_imm);
15539 %}
15540 
15541 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15542 %{
15543   match(Set cr (OverflowSubL op1 op2));
15544 
15545   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15546   ins_cost(INSN_COST);
15547   ins_encode %{
15548     __ cmp($op1$$Register, $op2$$Register);
15549   %}
15550 
15551   ins_pipe(icmp_reg_reg);
15552 %}
15553 
15554 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15555 %{
15556   match(Set cr (OverflowSubL op1 op2));
15557 
15558   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15559   ins_cost(INSN_COST);
15560   ins_encode %{
15561     __ subs(zr, $op1$$Register, $op2$$constant);
15562   %}
15563 
15564   ins_pipe(icmp_reg_imm);
15565 %}
15566 
15567 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15568 %{
15569   match(Set cr (OverflowSubI zero op1));
15570 
15571   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15572   ins_cost(INSN_COST);
15573   ins_encode %{
15574     __ cmpw(zr, $op1$$Register);
15575   %}
15576 
15577   ins_pipe(icmp_reg_imm);
15578 %}
15579 
15580 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15581 %{
15582   match(Set cr (OverflowSubL zero op1));
15583 
15584   format %{ "cmp   zr, $op1\t# overflow check long" %}
15585   ins_cost(INSN_COST);
15586   ins_encode %{
15587     __ cmp(zr, $op1$$Register);
15588   %}
15589 
15590   ins_pipe(icmp_reg_imm);
15591 %}
15592 
15593 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15594 %{
15595   match(Set cr (OverflowMulI op1 op2));
15596 
15597   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15598             "cmp   rscratch1, rscratch1, sxtw\n\t"
15599             "movw  rscratch1, #0x80000000\n\t"
15600             "cselw rscratch1, rscratch1, zr, NE\n\t"
15601             "cmpw  rscratch1, #1" %}
15602   ins_cost(5 * INSN_COST);
15603   ins_encode %{
15604     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15605     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15606     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15607     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15608     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15609   %}
15610 
15611   ins_pipe(pipe_slow);
15612 %}
15613 
15614 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15615 %{
15616   match(If cmp (OverflowMulI op1 op2));
15617   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15618             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15619   effect(USE labl, KILL cr);
15620 
15621   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15622             "cmp   rscratch1, rscratch1, sxtw\n\t"
15623             "b$cmp   $labl" %}
15624   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15625   ins_encode %{
15626     Label* L = $labl$$label;
15627     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15628     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15629     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15630     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15631   %}
15632 
15633   ins_pipe(pipe_serial);
15634 %}
15635 
15636 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15637 %{
15638   match(Set cr (OverflowMulL op1 op2));
15639 
15640   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15641             "smulh rscratch2, $op1, $op2\n\t"
15642             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15643             "movw  rscratch1, #0x80000000\n\t"
15644             "cselw rscratch1, rscratch1, zr, NE\n\t"
15645             "cmpw  rscratch1, #1" %}
15646   ins_cost(6 * INSN_COST);
15647   ins_encode %{
15648     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15649     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15650     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15651     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15652     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15653     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15654   %}
15655 
15656   ins_pipe(pipe_slow);
15657 %}
15658 
15659 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15660 %{
15661   match(If cmp (OverflowMulL op1 op2));
15662   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15663             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15664   effect(USE labl, KILL cr);
15665 
15666   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15667             "smulh rscratch2, $op1, $op2\n\t"
15668             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15669             "b$cmp $labl" %}
15670   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15671   ins_encode %{
15672     Label* L = $labl$$label;
15673     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15674     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15675     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15676     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15677     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15678   %}
15679 
15680   ins_pipe(pipe_serial);
15681 %}
15682 
15683 // ============================================================================
15684 // Compare Instructions
15685 
15686 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15687 %{
15688   match(Set cr (CmpI op1 op2));
15689 
15690   effect(DEF cr, USE op1, USE op2);
15691 
15692   ins_cost(INSN_COST);
15693   format %{ "cmpw  $op1, $op2" %}
15694 
15695   ins_encode(aarch64_enc_cmpw(op1, op2));
15696 
15697   ins_pipe(icmp_reg_reg);
15698 %}
15699 
15700 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15701 %{
15702   match(Set cr (CmpI op1 zero));
15703 
15704   effect(DEF cr, USE op1);
15705 
15706   ins_cost(INSN_COST);
15707   format %{ "cmpw $op1, 0" %}
15708 
15709   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15710 
15711   ins_pipe(icmp_reg_imm);
15712 %}
15713 
15714 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15715 %{
15716   match(Set cr (CmpI op1 op2));
15717 
15718   effect(DEF cr, USE op1);
15719 
15720   ins_cost(INSN_COST);
15721   format %{ "cmpw  $op1, $op2" %}
15722 
15723   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15724 
15725   ins_pipe(icmp_reg_imm);
15726 %}
15727 
15728 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15729 %{
15730   match(Set cr (CmpI op1 op2));
15731 
15732   effect(DEF cr, USE op1);
15733 
15734   ins_cost(INSN_COST * 2);
15735   format %{ "cmpw  $op1, $op2" %}
15736 
15737   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15738 
15739   ins_pipe(icmp_reg_imm);
15740 %}
15741 
15742 // Unsigned compare Instructions; really, same as signed compare
15743 // except it should only be used to feed an If or a CMovI which takes a
15744 // cmpOpU.
15745 
15746 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15747 %{
15748   match(Set cr (CmpU op1 op2));
15749 
15750   effect(DEF cr, USE op1, USE op2);
15751 
15752   ins_cost(INSN_COST);
15753   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15754 
15755   ins_encode(aarch64_enc_cmpw(op1, op2));
15756 
15757   ins_pipe(icmp_reg_reg);
15758 %}
15759 
15760 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15761 %{
15762   match(Set cr (CmpU op1 zero));
15763 
15764   effect(DEF cr, USE op1);
15765 
15766   ins_cost(INSN_COST);
15767   format %{ "cmpw $op1, #0\t# unsigned" %}
15768 
15769   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15770 
15771   ins_pipe(icmp_reg_imm);
15772 %}
15773 
15774 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15775 %{
15776   match(Set cr (CmpU op1 op2));
15777 
15778   effect(DEF cr, USE op1);
15779 
15780   ins_cost(INSN_COST);
15781   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15782 
15783   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15784 
15785   ins_pipe(icmp_reg_imm);
15786 %}
15787 
15788 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15789 %{
15790   match(Set cr (CmpU op1 op2));
15791 
15792   effect(DEF cr, USE op1);
15793 
15794   ins_cost(INSN_COST * 2);
15795   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15796 
15797   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15798 
15799   ins_pipe(icmp_reg_imm);
15800 %}
15801 
15802 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15803 %{
15804   match(Set cr (CmpL op1 op2));
15805 
15806   effect(DEF cr, USE op1, USE op2);
15807 
15808   ins_cost(INSN_COST);
15809   format %{ "cmp  $op1, $op2" %}
15810 
15811   ins_encode(aarch64_enc_cmp(op1, op2));
15812 
15813   ins_pipe(icmp_reg_reg);
15814 %}
15815 
15816 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15817 %{
15818   match(Set cr (CmpL op1 zero));
15819 
15820   effect(DEF cr, USE op1);
15821 
15822   ins_cost(INSN_COST);
15823   format %{ "tst  $op1" %}
15824 
15825   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15826 
15827   ins_pipe(icmp_reg_imm);
15828 %}
15829 
15830 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15831 %{
15832   match(Set cr (CmpL op1 op2));
15833 
15834   effect(DEF cr, USE op1);
15835 
15836   ins_cost(INSN_COST);
15837   format %{ "cmp  $op1, $op2" %}
15838 
15839   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15840 
15841   ins_pipe(icmp_reg_imm);
15842 %}
15843 
15844 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15845 %{
15846   match(Set cr (CmpL op1 op2));
15847 
15848   effect(DEF cr, USE op1);
15849 
15850   ins_cost(INSN_COST * 2);
15851   format %{ "cmp  $op1, $op2" %}
15852 
15853   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15854 
15855   ins_pipe(icmp_reg_imm);
15856 %}
15857 
15858 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15859 %{
15860   match(Set cr (CmpUL op1 op2));
15861 
15862   effect(DEF cr, USE op1, USE op2);
15863 
15864   ins_cost(INSN_COST);
15865   format %{ "cmp  $op1, $op2" %}
15866 
15867   ins_encode(aarch64_enc_cmp(op1, op2));
15868 
15869   ins_pipe(icmp_reg_reg);
15870 %}
15871 
15872 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15873 %{
15874   match(Set cr (CmpUL op1 zero));
15875 
15876   effect(DEF cr, USE op1);
15877 
15878   ins_cost(INSN_COST);
15879   format %{ "tst  $op1" %}
15880 
15881   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15882 
15883   ins_pipe(icmp_reg_imm);
15884 %}
15885 
15886 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15887 %{
15888   match(Set cr (CmpUL op1 op2));
15889 
15890   effect(DEF cr, USE op1);
15891 
15892   ins_cost(INSN_COST);
15893   format %{ "cmp  $op1, $op2" %}
15894 
15895   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15896 
15897   ins_pipe(icmp_reg_imm);
15898 %}
15899 
15900 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15901 %{
15902   match(Set cr (CmpUL op1 op2));
15903 
15904   effect(DEF cr, USE op1);
15905 
15906   ins_cost(INSN_COST * 2);
15907   format %{ "cmp  $op1, $op2" %}
15908 
15909   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15910 
15911   ins_pipe(icmp_reg_imm);
15912 %}
15913 
15914 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15915 %{
15916   match(Set cr (CmpP op1 op2));
15917 
15918   effect(DEF cr, USE op1, USE op2);
15919 
15920   ins_cost(INSN_COST);
15921   format %{ "cmp  $op1, $op2\t // ptr" %}
15922 
15923   ins_encode(aarch64_enc_cmpp(op1, op2));
15924 
15925   ins_pipe(icmp_reg_reg);
15926 %}
15927 
15928 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15929 %{
15930   match(Set cr (CmpN op1 op2));
15931 
15932   effect(DEF cr, USE op1, USE op2);
15933 
15934   ins_cost(INSN_COST);
15935   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15936 
15937   ins_encode(aarch64_enc_cmpn(op1, op2));
15938 
15939   ins_pipe(icmp_reg_reg);
15940 %}
15941 
15942 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15943 %{
15944   match(Set cr (CmpP op1 zero));
15945 
15946   effect(DEF cr, USE op1, USE zero);
15947 
15948   ins_cost(INSN_COST);
15949   format %{ "cmp  $op1, 0\t // ptr" %}
15950 
15951   ins_encode(aarch64_enc_testp(op1));
15952 
15953   ins_pipe(icmp_reg_imm);
15954 %}
15955 
15956 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15957 %{
15958   match(Set cr (CmpN op1 zero));
15959 
15960   effect(DEF cr, USE op1, USE zero);
15961 
15962   ins_cost(INSN_COST);
15963   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15964 
15965   ins_encode(aarch64_enc_testn(op1));
15966 
15967   ins_pipe(icmp_reg_imm);
15968 %}
15969 
15970 // FP comparisons
15971 //
15972 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15973 // using normal cmpOp. See declaration of rFlagsReg for details.
15974 
15975 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15976 %{
15977   match(Set cr (CmpF src1 src2));
15978 
15979   ins_cost(3 * INSN_COST);
15980   format %{ "fcmps $src1, $src2" %}
15981 
15982   ins_encode %{
15983     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15984   %}
15985 
15986   ins_pipe(pipe_class_compare);
15987 %}
15988 
15989 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15990 %{
15991   match(Set cr (CmpF src1 src2));
15992 
15993   ins_cost(3 * INSN_COST);
15994   format %{ "fcmps $src1, 0.0" %}
15995 
15996   ins_encode %{
15997     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15998   %}
15999 
16000   ins_pipe(pipe_class_compare);
16001 %}
16002 // FROM HERE
16003 
16004 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
16005 %{
16006   match(Set cr (CmpD src1 src2));
16007 
16008   ins_cost(3 * INSN_COST);
16009   format %{ "fcmpd $src1, $src2" %}
16010 
16011   ins_encode %{
16012     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16013   %}
16014 
16015   ins_pipe(pipe_class_compare);
16016 %}
16017 
16018 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
16019 %{
16020   match(Set cr (CmpD src1 src2));
16021 
16022   ins_cost(3 * INSN_COST);
16023   format %{ "fcmpd $src1, 0.0" %}
16024 
16025   ins_encode %{
16026     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
16027   %}
16028 
16029   ins_pipe(pipe_class_compare);
16030 %}
16031 
16032 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
16033 %{
16034   match(Set dst (CmpF3 src1 src2));
16035   effect(KILL cr);
16036 
16037   ins_cost(5 * INSN_COST);
16038   format %{ "fcmps $src1, $src2\n\t"
16039             "csinvw($dst, zr, zr, eq\n\t"
16040             "csnegw($dst, $dst, $dst, lt)"
16041   %}
16042 
16043   ins_encode %{
16044     Label done;
16045     FloatRegister s1 = as_FloatRegister($src1$$reg);
16046     FloatRegister s2 = as_FloatRegister($src2$$reg);
16047     Register d = as_Register($dst$$reg);
16048     __ fcmps(s1, s2);
16049     // installs 0 if EQ else -1
16050     __ csinvw(d, zr, zr, Assembler::EQ);
16051     // keeps -1 if less or unordered else installs 1
16052     __ csnegw(d, d, d, Assembler::LT);
16053     __ bind(done);
16054   %}
16055 
16056   ins_pipe(pipe_class_default);
16057 
16058 %}
16059 
16060 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
16061 %{
16062   match(Set dst (CmpD3 src1 src2));
16063   effect(KILL cr);
16064 
16065   ins_cost(5 * INSN_COST);
16066   format %{ "fcmpd $src1, $src2\n\t"
16067             "csinvw($dst, zr, zr, eq\n\t"
16068             "csnegw($dst, $dst, $dst, lt)"
16069   %}
16070 
16071   ins_encode %{
16072     Label done;
16073     FloatRegister s1 = as_FloatRegister($src1$$reg);
16074     FloatRegister s2 = as_FloatRegister($src2$$reg);
16075     Register d = as_Register($dst$$reg);
16076     __ fcmpd(s1, s2);
16077     // installs 0 if EQ else -1
16078     __ csinvw(d, zr, zr, Assembler::EQ);
16079     // keeps -1 if less or unordered else installs 1
16080     __ csnegw(d, d, d, Assembler::LT);
16081     __ bind(done);
16082   %}
16083   ins_pipe(pipe_class_default);
16084 
16085 %}
16086 
16087 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
16088 %{
16089   match(Set dst (CmpF3 src1 zero));
16090   effect(KILL cr);
16091 
16092   ins_cost(5 * INSN_COST);
16093   format %{ "fcmps $src1, 0.0\n\t"
16094             "csinvw($dst, zr, zr, eq\n\t"
16095             "csnegw($dst, $dst, $dst, lt)"
16096   %}
16097 
16098   ins_encode %{
16099     Label done;
16100     FloatRegister s1 = as_FloatRegister($src1$$reg);
16101     Register d = as_Register($dst$$reg);
16102     __ fcmps(s1, 0.0);
16103     // installs 0 if EQ else -1
16104     __ csinvw(d, zr, zr, Assembler::EQ);
16105     // keeps -1 if less or unordered else installs 1
16106     __ csnegw(d, d, d, Assembler::LT);
16107     __ bind(done);
16108   %}
16109 
16110   ins_pipe(pipe_class_default);
16111 
16112 %}
16113 
16114 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
16115 %{
16116   match(Set dst (CmpD3 src1 zero));
16117   effect(KILL cr);
16118 
16119   ins_cost(5 * INSN_COST);
16120   format %{ "fcmpd $src1, 0.0\n\t"
16121             "csinvw($dst, zr, zr, eq\n\t"
16122             "csnegw($dst, $dst, $dst, lt)"
16123   %}
16124 
16125   ins_encode %{
16126     Label done;
16127     FloatRegister s1 = as_FloatRegister($src1$$reg);
16128     Register d = as_Register($dst$$reg);
16129     __ fcmpd(s1, 0.0);
16130     // installs 0 if EQ else -1
16131     __ csinvw(d, zr, zr, Assembler::EQ);
16132     // keeps -1 if less or unordered else installs 1
16133     __ csnegw(d, d, d, Assembler::LT);
16134     __ bind(done);
16135   %}
16136   ins_pipe(pipe_class_default);
16137 
16138 %}
16139 
16140 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
16141 %{
16142   match(Set dst (CmpLTMask p q));
16143   effect(KILL cr);
16144 
16145   ins_cost(3 * INSN_COST);
16146 
16147   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
16148             "csetw $dst, lt\n\t"
16149             "subw $dst, zr, $dst"
16150   %}
16151 
16152   ins_encode %{
16153     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
16154     __ csetw(as_Register($dst$$reg), Assembler::LT);
16155     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
16156   %}
16157 
16158   ins_pipe(ialu_reg_reg);
16159 %}
16160 
16161 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
16162 %{
16163   match(Set dst (CmpLTMask src zero));
16164   effect(KILL cr);
16165 
16166   ins_cost(INSN_COST);
16167 
16168   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
16169 
16170   ins_encode %{
16171     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
16172   %}
16173 
16174   ins_pipe(ialu_reg_shift);
16175 %}
16176 
16177 // ============================================================================
16178 // Max and Min
16179 
16180 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
16181 
16182 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
16183 %{
16184   effect(DEF cr, USE src);
16185   ins_cost(INSN_COST);
16186   format %{ "cmpw $src, 0" %}
16187 
16188   ins_encode %{
16189     __ cmpw($src$$Register, 0);
16190   %}
16191   ins_pipe(icmp_reg_imm);
16192 %}
16193 
16194 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16195 %{
16196   match(Set dst (MinI src1 src2));
16197   ins_cost(INSN_COST * 3);
16198 
16199   expand %{
16200     rFlagsReg cr;
16201     compI_reg_reg(cr, src1, src2);
16202     cmovI_reg_reg_lt(dst, src1, src2, cr);
16203   %}
16204 %}
16205 
16206 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16207 %{
16208   match(Set dst (MaxI src1 src2));
16209   ins_cost(INSN_COST * 3);
16210 
16211   expand %{
16212     rFlagsReg cr;
16213     compI_reg_reg(cr, src1, src2);
16214     cmovI_reg_reg_gt(dst, src1, src2, cr);
16215   %}
16216 %}
16217 
16218 
16219 // ============================================================================
16220 // Branch Instructions
16221 
16222 // Direct Branch.
16223 instruct branch(label lbl)
16224 %{
16225   match(Goto);
16226 
16227   effect(USE lbl);
16228 
16229   ins_cost(BRANCH_COST);
16230   format %{ "b  $lbl" %}
16231 
16232   ins_encode(aarch64_enc_b(lbl));
16233 
16234   ins_pipe(pipe_branch);
16235 %}
16236 
16237 // Conditional Near Branch
16238 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16239 %{
16240   // Same match rule as `branchConFar'.
16241   match(If cmp cr);
16242 
16243   effect(USE lbl);
16244 
16245   ins_cost(BRANCH_COST);
16246   // If set to 1 this indicates that the current instruction is a
16247   // short variant of a long branch. This avoids using this
16248   // instruction in first-pass matching. It will then only be used in
16249   // the `Shorten_branches' pass.
16250   // ins_short_branch(1);
16251   format %{ "b$cmp  $lbl" %}
16252 
16253   ins_encode(aarch64_enc_br_con(cmp, lbl));
16254 
16255   ins_pipe(pipe_branch_cond);
16256 %}
16257 
16258 // Conditional Near Branch Unsigned
16259 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16260 %{
16261   // Same match rule as `branchConFar'.
16262   match(If cmp cr);
16263 
16264   effect(USE lbl);
16265 
16266   ins_cost(BRANCH_COST);
16267   // If set to 1 this indicates that the current instruction is a
16268   // short variant of a long branch. This avoids using this
16269   // instruction in first-pass matching. It will then only be used in
16270   // the `Shorten_branches' pass.
16271   // ins_short_branch(1);
16272   format %{ "b$cmp  $lbl\t# unsigned" %}
16273 
16274   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16275 
16276   ins_pipe(pipe_branch_cond);
16277 %}
16278 
16279 // Make use of CBZ and CBNZ.  These instructions, as well as being
16280 // shorter than (cmp; branch), have the additional benefit of not
16281 // killing the flags.
16282 
16283 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16284   match(If cmp (CmpI op1 op2));
16285   effect(USE labl);
16286 
16287   ins_cost(BRANCH_COST);
16288   format %{ "cbw$cmp   $op1, $labl" %}
16289   ins_encode %{
16290     Label* L = $labl$$label;
16291     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16292     if (cond == Assembler::EQ)
16293       __ cbzw($op1$$Register, *L);
16294     else
16295       __ cbnzw($op1$$Register, *L);
16296   %}
16297   ins_pipe(pipe_cmp_branch);
16298 %}
16299 
16300 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16301   match(If cmp (CmpL op1 op2));
16302   effect(USE labl);
16303 
16304   ins_cost(BRANCH_COST);
16305   format %{ "cb$cmp   $op1, $labl" %}
16306   ins_encode %{
16307     Label* L = $labl$$label;
16308     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16309     if (cond == Assembler::EQ)
16310       __ cbz($op1$$Register, *L);
16311     else
16312       __ cbnz($op1$$Register, *L);
16313   %}
16314   ins_pipe(pipe_cmp_branch);
16315 %}
16316 
16317 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16318   match(If cmp (CmpP op1 op2));
16319   effect(USE labl);
16320 
16321   ins_cost(BRANCH_COST);
16322   format %{ "cb$cmp   $op1, $labl" %}
16323   ins_encode %{
16324     Label* L = $labl$$label;
16325     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16326     if (cond == Assembler::EQ)
16327       __ cbz($op1$$Register, *L);
16328     else
16329       __ cbnz($op1$$Register, *L);
16330   %}
16331   ins_pipe(pipe_cmp_branch);
16332 %}
16333 
16334 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16335   match(If cmp (CmpN op1 op2));
16336   effect(USE labl);
16337 
16338   ins_cost(BRANCH_COST);
16339   format %{ "cbw$cmp   $op1, $labl" %}
16340   ins_encode %{
16341     Label* L = $labl$$label;
16342     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16343     if (cond == Assembler::EQ)
16344       __ cbzw($op1$$Register, *L);
16345     else
16346       __ cbnzw($op1$$Register, *L);
16347   %}
16348   ins_pipe(pipe_cmp_branch);
16349 %}
16350 
16351 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16352   match(If cmp (CmpP (DecodeN oop) zero));
16353   effect(USE labl);
16354 
16355   ins_cost(BRANCH_COST);
16356   format %{ "cb$cmp   $oop, $labl" %}
16357   ins_encode %{
16358     Label* L = $labl$$label;
16359     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16360     if (cond == Assembler::EQ)
16361       __ cbzw($oop$$Register, *L);
16362     else
16363       __ cbnzw($oop$$Register, *L);
16364   %}
16365   ins_pipe(pipe_cmp_branch);
16366 %}
16367 
16368 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16369   match(If cmp (CmpU op1 op2));
16370   effect(USE labl);
16371 
16372   ins_cost(BRANCH_COST);
16373   format %{ "cbw$cmp   $op1, $labl" %}
16374   ins_encode %{
16375     Label* L = $labl$$label;
16376     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16377     if (cond == Assembler::EQ || cond == Assembler::LS)
16378       __ cbzw($op1$$Register, *L);
16379     else
16380       __ cbnzw($op1$$Register, *L);
16381   %}
16382   ins_pipe(pipe_cmp_branch);
16383 %}
16384 
16385 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16386   match(If cmp (CmpUL op1 op2));
16387   effect(USE labl);
16388 
16389   ins_cost(BRANCH_COST);
16390   format %{ "cb$cmp   $op1, $labl" %}
16391   ins_encode %{
16392     Label* L = $labl$$label;
16393     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16394     if (cond == Assembler::EQ || cond == Assembler::LS)
16395       __ cbz($op1$$Register, *L);
16396     else
16397       __ cbnz($op1$$Register, *L);
16398   %}
16399   ins_pipe(pipe_cmp_branch);
16400 %}
16401 
16402 // Test bit and Branch
16403 
16404 // Patterns for short (< 32KiB) variants
16405 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16406   match(If cmp (CmpL op1 op2));
16407   effect(USE labl);
16408 
16409   ins_cost(BRANCH_COST);
16410   format %{ "cb$cmp   $op1, $labl # long" %}
16411   ins_encode %{
16412     Label* L = $labl$$label;
16413     Assembler::Condition cond =
16414       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16415     __ tbr(cond, $op1$$Register, 63, *L);
16416   %}
16417   ins_pipe(pipe_cmp_branch);
16418   ins_short_branch(1);
16419 %}
16420 
16421 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16422   match(If cmp (CmpI op1 op2));
16423   effect(USE labl);
16424 
16425   ins_cost(BRANCH_COST);
16426   format %{ "cb$cmp   $op1, $labl # int" %}
16427   ins_encode %{
16428     Label* L = $labl$$label;
16429     Assembler::Condition cond =
16430       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16431     __ tbr(cond, $op1$$Register, 31, *L);
16432   %}
16433   ins_pipe(pipe_cmp_branch);
16434   ins_short_branch(1);
16435 %}
16436 
16437 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16438   match(If cmp (CmpL (AndL op1 op2) op3));
16439   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16440   effect(USE labl);
16441 
16442   ins_cost(BRANCH_COST);
16443   format %{ "tb$cmp   $op1, $op2, $labl" %}
16444   ins_encode %{
16445     Label* L = $labl$$label;
16446     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16447     int bit = exact_log2_long($op2$$constant);
16448     __ tbr(cond, $op1$$Register, bit, *L);
16449   %}
16450   ins_pipe(pipe_cmp_branch);
16451   ins_short_branch(1);
16452 %}
16453 
16454 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16455   match(If cmp (CmpI (AndI op1 op2) op3));
16456   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16457   effect(USE labl);
16458 
16459   ins_cost(BRANCH_COST);
16460   format %{ "tb$cmp   $op1, $op2, $labl" %}
16461   ins_encode %{
16462     Label* L = $labl$$label;
16463     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16464     int bit = exact_log2((juint)$op2$$constant);
16465     __ tbr(cond, $op1$$Register, bit, *L);
16466   %}
16467   ins_pipe(pipe_cmp_branch);
16468   ins_short_branch(1);
16469 %}
16470 
16471 // And far variants
16472 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16473   match(If cmp (CmpL op1 op2));
16474   effect(USE labl);
16475 
16476   ins_cost(BRANCH_COST);
16477   format %{ "cb$cmp   $op1, $labl # long" %}
16478   ins_encode %{
16479     Label* L = $labl$$label;
16480     Assembler::Condition cond =
16481       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16482     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16483   %}
16484   ins_pipe(pipe_cmp_branch);
16485 %}
16486 
16487 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16488   match(If cmp (CmpI op1 op2));
16489   effect(USE labl);
16490 
16491   ins_cost(BRANCH_COST);
16492   format %{ "cb$cmp   $op1, $labl # int" %}
16493   ins_encode %{
16494     Label* L = $labl$$label;
16495     Assembler::Condition cond =
16496       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16497     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16498   %}
16499   ins_pipe(pipe_cmp_branch);
16500 %}
16501 
16502 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16503   match(If cmp (CmpL (AndL op1 op2) op3));
16504   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16505   effect(USE labl);
16506 
16507   ins_cost(BRANCH_COST);
16508   format %{ "tb$cmp   $op1, $op2, $labl" %}
16509   ins_encode %{
16510     Label* L = $labl$$label;
16511     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16512     int bit = exact_log2_long($op2$$constant);
16513     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16514   %}
16515   ins_pipe(pipe_cmp_branch);
16516 %}
16517 
16518 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16519   match(If cmp (CmpI (AndI op1 op2) op3));
16520   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16521   effect(USE labl);
16522 
16523   ins_cost(BRANCH_COST);
16524   format %{ "tb$cmp   $op1, $op2, $labl" %}
16525   ins_encode %{
16526     Label* L = $labl$$label;
16527     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16528     int bit = exact_log2((juint)$op2$$constant);
16529     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16530   %}
16531   ins_pipe(pipe_cmp_branch);
16532 %}
16533 
16534 // Test bits
16535 
16536 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16537   match(Set cr (CmpL (AndL op1 op2) op3));
16538   predicate(Assembler::operand_valid_for_logical_immediate
16539             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16540 
16541   ins_cost(INSN_COST);
16542   format %{ "tst $op1, $op2 # long" %}
16543   ins_encode %{
16544     __ tst($op1$$Register, $op2$$constant);
16545   %}
16546   ins_pipe(ialu_reg_reg);
16547 %}
16548 
16549 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16550   match(Set cr (CmpI (AndI op1 op2) op3));
16551   predicate(Assembler::operand_valid_for_logical_immediate
16552             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16553 
16554   ins_cost(INSN_COST);
16555   format %{ "tst $op1, $op2 # int" %}
16556   ins_encode %{
16557     __ tstw($op1$$Register, $op2$$constant);
16558   %}
16559   ins_pipe(ialu_reg_reg);
16560 %}
16561 
16562 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16563   match(Set cr (CmpL (AndL op1 op2) op3));
16564 
16565   ins_cost(INSN_COST);
16566   format %{ "tst $op1, $op2 # long" %}
16567   ins_encode %{
16568     __ tst($op1$$Register, $op2$$Register);
16569   %}
16570   ins_pipe(ialu_reg_reg);
16571 %}
16572 
16573 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16574   match(Set cr (CmpI (AndI op1 op2) op3));
16575 
16576   ins_cost(INSN_COST);
16577   format %{ "tstw $op1, $op2 # int" %}
16578   ins_encode %{
16579     __ tstw($op1$$Register, $op2$$Register);
16580   %}
16581   ins_pipe(ialu_reg_reg);
16582 %}
16583 
16584 
16585 // Conditional Far Branch
16586 // Conditional Far Branch Unsigned
16587 // TODO: fixme
16588 
16589 // counted loop end branch near
16590 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16591 %{
16592   match(CountedLoopEnd cmp cr);
16593 
16594   effect(USE lbl);
16595 
16596   ins_cost(BRANCH_COST);
16597   // short variant.
16598   // ins_short_branch(1);
16599   format %{ "b$cmp $lbl \t// counted loop end" %}
16600 
16601   ins_encode(aarch64_enc_br_con(cmp, lbl));
16602 
16603   ins_pipe(pipe_branch);
16604 %}
16605 
16606 // counted loop end branch far
16607 // TODO: fixme
16608 
16609 // ============================================================================
16610 // inlined locking and unlocking
16611 
16612 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16613 %{
16614   match(Set cr (FastLock object box));
16615   effect(TEMP tmp, TEMP tmp2);
16616 
16617   // TODO
16618   // identify correct cost
16619   ins_cost(5 * INSN_COST);
16620   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16621 
16622   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16623 
16624   ins_pipe(pipe_serial);
16625 %}
16626 
16627 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16628 %{
16629   match(Set cr (FastUnlock object box));
16630   effect(TEMP tmp, TEMP tmp2);
16631 
16632   ins_cost(5 * INSN_COST);
16633   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16634 
16635   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16636 
16637   ins_pipe(pipe_serial);
16638 %}
16639 
16640 
16641 // ============================================================================
16642 // Safepoint Instructions
16643 
16644 // TODO
16645 // provide a near and far version of this code
16646 
16647 instruct safePoint(rFlagsReg cr, iRegP poll)
16648 %{
16649   match(SafePoint poll);
16650   effect(KILL cr);
16651 
16652   format %{
16653     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16654   %}
16655   ins_encode %{
16656     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16657   %}
16658   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16659 %}
16660 
16661 
16662 // ============================================================================
16663 // Procedure Call/Return Instructions
16664 
16665 // Call Java Static Instruction
16666 
16667 instruct CallStaticJavaDirect(method meth)
16668 %{
16669   match(CallStaticJava);
16670 
16671   effect(USE meth);
16672 
16673   ins_cost(CALL_COST);
16674 
16675   format %{ "call,static $meth \t// ==> " %}
16676 
16677   ins_encode(aarch64_enc_java_static_call(meth),
16678              aarch64_enc_call_epilog);
16679 
16680   ins_pipe(pipe_class_call);
16681 %}
16682 
16683 // TO HERE
16684 
16685 // Call Java Dynamic Instruction
16686 instruct CallDynamicJavaDirect(method meth)
16687 %{
16688   match(CallDynamicJava);
16689 
16690   effect(USE meth);
16691 
16692   ins_cost(CALL_COST);
16693 
16694   format %{ "CALL,dynamic $meth \t// ==> " %}
16695 
16696   ins_encode(aarch64_enc_java_dynamic_call(meth),
16697              aarch64_enc_call_epilog);
16698 
16699   ins_pipe(pipe_class_call);
16700 %}
16701 
16702 // Call Runtime Instruction
16703 
16704 instruct CallRuntimeDirect(method meth)
16705 %{
16706   match(CallRuntime);
16707 
16708   effect(USE meth);
16709 
16710   ins_cost(CALL_COST);
16711 
16712   format %{ "CALL, runtime $meth" %}
16713 
16714   ins_encode( aarch64_enc_java_to_runtime(meth) );
16715 
16716   ins_pipe(pipe_class_call);
16717 %}
16718 
16719 // Call Runtime Instruction
16720 
16721 instruct CallLeafDirect(method meth)
16722 %{
16723   match(CallLeaf);
16724 
16725   effect(USE meth);
16726 
16727   ins_cost(CALL_COST);
16728 
16729   format %{ "CALL, runtime leaf $meth" %}
16730 
16731   ins_encode( aarch64_enc_java_to_runtime(meth) );
16732 
16733   ins_pipe(pipe_class_call);
16734 %}
16735 
16736 // Call Runtime Instruction
16737 
16738 instruct CallLeafNoFPDirect(method meth)
16739 %{
16740   match(CallLeafNoFP);
16741 
16742   effect(USE meth);
16743 
16744   ins_cost(CALL_COST);
16745 
16746   format %{ "CALL, runtime leaf nofp $meth" %}
16747 
16748   ins_encode( aarch64_enc_java_to_runtime(meth) );
16749 
16750   ins_pipe(pipe_class_call);
16751 %}
16752 
16753 // Tail Call; Jump from runtime stub to Java code.
16754 // Also known as an 'interprocedural jump'.
16755 // Target of jump will eventually return to caller.
16756 // TailJump below removes the return address.
16757 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16758 %{
16759   match(TailCall jump_target method_ptr);
16760 
16761   ins_cost(CALL_COST);
16762 
16763   format %{ "br $jump_target\t# $method_ptr holds method" %}
16764 
16765   ins_encode(aarch64_enc_tail_call(jump_target));
16766 
16767   ins_pipe(pipe_class_call);
16768 %}
16769 
16770 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16771 %{
16772   match(TailJump jump_target ex_oop);
16773 
16774   ins_cost(CALL_COST);
16775 
16776   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16777 
16778   ins_encode(aarch64_enc_tail_jmp(jump_target));
16779 
16780   ins_pipe(pipe_class_call);
16781 %}
16782 
16783 // Create exception oop: created by stack-crawling runtime code.
16784 // Created exception is now available to this handler, and is setup
16785 // just prior to jumping to this handler. No code emitted.
16786 // TODO check
16787 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16788 instruct CreateException(iRegP_R0 ex_oop)
16789 %{
16790   match(Set ex_oop (CreateEx));
16791 
16792   format %{ " -- \t// exception oop; no code emitted" %}
16793 
16794   size(0);
16795 
16796   ins_encode( /*empty*/ );
16797 
16798   ins_pipe(pipe_class_empty);
16799 %}
16800 
16801 // Rethrow exception: The exception oop will come in the first
16802 // argument position. Then JUMP (not call) to the rethrow stub code.
16803 instruct RethrowException() %{
16804   match(Rethrow);
16805   ins_cost(CALL_COST);
16806 
16807   format %{ "b rethrow_stub" %}
16808 
16809   ins_encode( aarch64_enc_rethrow() );
16810 
16811   ins_pipe(pipe_class_call);
16812 %}
16813 
16814 
16815 // Return Instruction
16816 // epilog node loads ret address into lr as part of frame pop
16817 instruct Ret()
16818 %{
16819   match(Return);
16820 
16821   format %{ "ret\t// return register" %}
16822 
16823   ins_encode( aarch64_enc_ret() );
16824 
16825   ins_pipe(pipe_branch);
16826 %}
16827 
16828 // Die now.
16829 instruct ShouldNotReachHere() %{
16830   match(Halt);
16831 
16832   ins_cost(CALL_COST);
16833   format %{ "ShouldNotReachHere" %}
16834 
16835   ins_encode %{
16836     if (is_reachable()) {
16837       __ stop(_halt_reason);
16838     }
16839   %}
16840 
16841   ins_pipe(pipe_class_default);
16842 %}
16843 
16844 // ============================================================================
16845 // Partial Subtype Check
16846 //
16847 // superklass array for an instance of the superklass.  Set a hidden
16848 // internal cache on a hit (cache is checked with exposed code in
16849 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16850 // encoding ALSO sets flags.
16851 
16852 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16853 %{
16854   match(Set result (PartialSubtypeCheck sub super));
16855   effect(KILL cr, KILL temp);
16856 
16857   ins_cost(1100);  // slightly larger than the next version
16858   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16859 
16860   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16861 
16862   opcode(0x1); // Force zero of result reg on hit
16863 
16864   ins_pipe(pipe_class_memory);
16865 %}
16866 
16867 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16868 %{
16869   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16870   effect(KILL temp, KILL result);
16871 
16872   ins_cost(1100);  // slightly larger than the next version
16873   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16874 
16875   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16876 
16877   opcode(0x0); // Don't zero result reg on hit
16878 
16879   ins_pipe(pipe_class_memory);
16880 %}
16881 
16882 // Intrisics for String.compareTo()
16883 
16884 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16885                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16886 %{
16887   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16888   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16889   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16890 
16891   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16892   ins_encode %{
16893     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16894     __ string_compare($str1$$Register, $str2$$Register,
16895                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16896                       $tmp1$$Register, $tmp2$$Register,
16897                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16898   %}
16899   ins_pipe(pipe_class_memory);
16900 %}
16901 
16902 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16903                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16904 %{
16905   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16906   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16907   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16908 
16909   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16910   ins_encode %{
16911     __ string_compare($str1$$Register, $str2$$Register,
16912                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16913                       $tmp1$$Register, $tmp2$$Register,
16914                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16915   %}
16916   ins_pipe(pipe_class_memory);
16917 %}
16918 
16919 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16920                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16921                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16922 %{
16923   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16924   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16925   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16926          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16927 
16928   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16929   ins_encode %{
16930     __ string_compare($str1$$Register, $str2$$Register,
16931                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16932                       $tmp1$$Register, $tmp2$$Register,
16933                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16934                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16935   %}
16936   ins_pipe(pipe_class_memory);
16937 %}
16938 
16939 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16940                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16941                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16942 %{
16943   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16944   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16945   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16946          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16947 
16948   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16949   ins_encode %{
16950     __ string_compare($str1$$Register, $str2$$Register,
16951                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16952                       $tmp1$$Register, $tmp2$$Register,
16953                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16954                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16955   %}
16956   ins_pipe(pipe_class_memory);
16957 %}
16958 
16959 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16960 // these string_compare variants as NEON register type for convenience so that the prototype of
16961 // string_compare can be shared with all variants.
16962 
16963 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16964                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16965                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16966                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16967 %{
16968   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16969   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16970   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16971          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16972 
16973   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16974   ins_encode %{
16975     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16976     __ string_compare($str1$$Register, $str2$$Register,
16977                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16978                       $tmp1$$Register, $tmp2$$Register,
16979                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16980                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16981                       StrIntrinsicNode::LL);
16982   %}
16983   ins_pipe(pipe_class_memory);
16984 %}
16985 
16986 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16987                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16988                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16989                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16990 %{
16991   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16992   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16993   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16994          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16995 
16996   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16997   ins_encode %{
16998     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16999     __ string_compare($str1$$Register, $str2$$Register,
17000                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
17001                       $tmp1$$Register, $tmp2$$Register,
17002                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17003                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17004                       StrIntrinsicNode::LU);
17005   %}
17006   ins_pipe(pipe_class_memory);
17007 %}
17008 
17009 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17010                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17011                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
17012                               pRegGov_P1 pgtmp2, rFlagsReg cr)
17013 %{
17014   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
17015   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17016   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17017          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17018 
17019   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
17020   ins_encode %{
17021     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17022     __ string_compare($str1$$Register, $str2$$Register,
17023                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
17024                       $tmp1$$Register, $tmp2$$Register,
17025                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17026                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17027                       StrIntrinsicNode::UL);
17028   %}
17029   ins_pipe(pipe_class_memory);
17030 %}
17031 
17032 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17033                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17034                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
17035                               pRegGov_P1 pgtmp2, rFlagsReg cr)
17036 %{
17037   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
17038   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17039   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17040          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17041 
17042   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
17043   ins_encode %{
17044     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17045     __ string_compare($str1$$Register, $str2$$Register,
17046                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
17047                       $tmp1$$Register, $tmp2$$Register,
17048                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17049                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17050                       StrIntrinsicNode::UU);
17051   %}
17052   ins_pipe(pipe_class_memory);
17053 %}
17054 
17055 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17056                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17057                           iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17058                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17059 %{
17060   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17061   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17062   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17063          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
17064          TEMP vtmp0, TEMP vtmp1, KILL cr);
17065   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
17066             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17067 
17068   ins_encode %{
17069     __ string_indexof($str1$$Register, $str2$$Register,
17070                       $cnt1$$Register, $cnt2$$Register,
17071                       $tmp1$$Register, $tmp2$$Register,
17072                       $tmp3$$Register, $tmp4$$Register,
17073                       $tmp5$$Register, $tmp6$$Register,
17074                       -1, $result$$Register, StrIntrinsicNode::UU);
17075   %}
17076   ins_pipe(pipe_class_memory);
17077 %}
17078 
17079 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17080                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17081                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17082                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17083 %{
17084   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17085   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17086   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17087          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
17088          TEMP vtmp0, TEMP vtmp1, KILL cr);
17089   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
17090             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17091 
17092   ins_encode %{
17093     __ string_indexof($str1$$Register, $str2$$Register,
17094                       $cnt1$$Register, $cnt2$$Register,
17095                       $tmp1$$Register, $tmp2$$Register,
17096                       $tmp3$$Register, $tmp4$$Register,
17097                       $tmp5$$Register, $tmp6$$Register,
17098                       -1, $result$$Register, StrIntrinsicNode::LL);
17099   %}
17100   ins_pipe(pipe_class_memory);
17101 %}
17102 
17103 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17104                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
17105                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17106                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17107 %{
17108   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17109   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17110   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17111          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
17112          TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
17113   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
17114             "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17115 
17116   ins_encode %{
17117     __ string_indexof($str1$$Register, $str2$$Register,
17118                       $cnt1$$Register, $cnt2$$Register,
17119                       $tmp1$$Register, $tmp2$$Register,
17120                       $tmp3$$Register, $tmp4$$Register,
17121                       $tmp5$$Register, $tmp6$$Register,
17122                       -1, $result$$Register, StrIntrinsicNode::UL);
17123   %}
17124   ins_pipe(pipe_class_memory);
17125 %}
17126 
17127 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17128                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17129                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17130 %{
17131   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17132   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17133   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17134          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17135   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
17136             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17137 
17138   ins_encode %{
17139     int icnt2 = (int)$int_cnt2$$constant;
17140     __ string_indexof($str1$$Register, $str2$$Register,
17141                       $cnt1$$Register, zr,
17142                       $tmp1$$Register, $tmp2$$Register,
17143                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17144                       icnt2, $result$$Register, StrIntrinsicNode::UU);
17145   %}
17146   ins_pipe(pipe_class_memory);
17147 %}
17148 
17149 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17150                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17151                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17152 %{
17153   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17154   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17155   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17156          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17157   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
17158             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17159 
17160   ins_encode %{
17161     int icnt2 = (int)$int_cnt2$$constant;
17162     __ string_indexof($str1$$Register, $str2$$Register,
17163                       $cnt1$$Register, zr,
17164                       $tmp1$$Register, $tmp2$$Register,
17165                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17166                       icnt2, $result$$Register, StrIntrinsicNode::LL);
17167   %}
17168   ins_pipe(pipe_class_memory);
17169 %}
17170 
17171 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17172                               immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17173                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17174 %{
17175   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17176   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17177   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17178          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17179   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
17180             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17181 
17182   ins_encode %{
17183     int icnt2 = (int)$int_cnt2$$constant;
17184     __ string_indexof($str1$$Register, $str2$$Register,
17185                       $cnt1$$Register, zr,
17186                       $tmp1$$Register, $tmp2$$Register,
17187                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17188                       icnt2, $result$$Register, StrIntrinsicNode::UL);
17189   %}
17190   ins_pipe(pipe_class_memory);
17191 %}
17192 
17193 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17194                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17195                              iRegINoSp tmp3, rFlagsReg cr)
17196 %{
17197   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17198   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17199   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17200          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17201 
17202   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17203 
17204   ins_encode %{
17205     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17206                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
17207                            $tmp3$$Register);
17208   %}
17209   ins_pipe(pipe_class_memory);
17210 %}
17211 
17212 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17213                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17214                               iRegINoSp tmp3, rFlagsReg cr)
17215 %{
17216   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17217   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17218   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17219          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17220 
17221   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17222 
17223   ins_encode %{
17224     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17225                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
17226                             $tmp3$$Register);
17227   %}
17228   ins_pipe(pipe_class_memory);
17229 %}
17230 
17231 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17232                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17233                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17234   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17235   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17236   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17237   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17238   ins_encode %{
17239     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17240                                $result$$Register, $ztmp1$$FloatRegister,
17241                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17242                                $ptmp$$PRegister, true /* isL */);
17243   %}
17244   ins_pipe(pipe_class_memory);
17245 %}
17246 
17247 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17248                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17249                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17250   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17251   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17252   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17253   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17254   ins_encode %{
17255     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17256                                $result$$Register, $ztmp1$$FloatRegister,
17257                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17258                                $ptmp$$PRegister, false /* isL */);
17259   %}
17260   ins_pipe(pipe_class_memory);
17261 %}
17262 
17263 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17264                         iRegI_R0 result, rFlagsReg cr)
17265 %{
17266   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17267   match(Set result (StrEquals (Binary str1 str2) cnt));
17268   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17269 
17270   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17271   ins_encode %{
17272     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17273     __ string_equals($str1$$Register, $str2$$Register,
17274                      $result$$Register, $cnt$$Register, 1);
17275   %}
17276   ins_pipe(pipe_class_memory);
17277 %}
17278 
17279 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17280                         iRegI_R0 result, rFlagsReg cr)
17281 %{
17282   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17283   match(Set result (StrEquals (Binary str1 str2) cnt));
17284   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17285 
17286   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17287   ins_encode %{
17288     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17289     __ string_equals($str1$$Register, $str2$$Register,
17290                      $result$$Register, $cnt$$Register, 2);
17291   %}
17292   ins_pipe(pipe_class_memory);
17293 %}
17294 
17295 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17296                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17297                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17298                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17299                        iRegP_R10 tmp, rFlagsReg cr)
17300 %{
17301   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17302   match(Set result (AryEq ary1 ary2));
17303   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17304          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17305          TEMP vtmp6, TEMP vtmp7, KILL cr);
17306 
17307   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17308   ins_encode %{
17309     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17310                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17311                                    $result$$Register, $tmp$$Register, 1);
17312     if (tpc == NULL) {
17313       ciEnv::current()->record_failure("CodeCache is full");
17314       return;
17315     }
17316   %}
17317   ins_pipe(pipe_class_memory);
17318 %}
17319 
17320 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17321                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17322                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17323                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17324                        iRegP_R10 tmp, rFlagsReg cr)
17325 %{
17326   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17327   match(Set result (AryEq ary1 ary2));
17328   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17329          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17330          TEMP vtmp6, TEMP vtmp7, KILL cr);
17331 
17332   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17333   ins_encode %{
17334     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17335                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17336                                    $result$$Register, $tmp$$Register, 2);
17337     if (tpc == NULL) {
17338       ciEnv::current()->record_failure("CodeCache is full");
17339       return;
17340     }
17341   %}
17342   ins_pipe(pipe_class_memory);
17343 %}
17344 
17345 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17346 %{
17347   match(Set result (CountPositives ary1 len));
17348   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17349   format %{ "count positives byte[] $ary1,$len -> $result" %}
17350   ins_encode %{
17351     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17352     if (tpc == NULL) {
17353       ciEnv::current()->record_failure("CodeCache is full");
17354       return;
17355     }
17356   %}
17357   ins_pipe( pipe_slow );
17358 %}
17359 
17360 // fast char[] to byte[] compression
17361 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17362                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17363                          vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17364                          iRegI_R0 result, rFlagsReg cr)
17365 %{
17366   match(Set result (StrCompressedCopy src (Binary dst len)));
17367   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17368          USE_KILL src, USE_KILL dst, USE len, KILL cr);
17369 
17370   format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17371   ins_encode %{
17372     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17373                            $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17374                            $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17375                            $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17376   %}
17377   ins_pipe(pipe_slow);
17378 %}
17379 
17380 // fast byte[] to char[] inflation
17381 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17382                         vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17383                         vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17384 %{
17385   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17386   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17387          TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17388          USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17389 
17390   format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17391   ins_encode %{
17392     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17393                                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17394                                         $vtmp2$$FloatRegister, $tmp$$Register);
17395     if (tpc == NULL) {
17396       ciEnv::current()->record_failure("CodeCache is full");
17397       return;
17398     }
17399   %}
17400   ins_pipe(pipe_class_memory);
17401 %}
17402 
17403 // encode char[] to byte[] in ISO_8859_1
17404 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17405                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17406                           vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17407                           iRegI_R0 result, rFlagsReg cr)
17408 %{
17409   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17410   match(Set result (EncodeISOArray src (Binary dst len)));
17411   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17412          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17413 
17414   format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17415   ins_encode %{
17416     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17417                         $result$$Register, false,
17418                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17419                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17420                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17421   %}
17422   ins_pipe(pipe_class_memory);
17423 %}
17424 
17425 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17426                             vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17427                             vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17428                             iRegI_R0 result, rFlagsReg cr)
17429 %{
17430   predicate(((EncodeISOArrayNode*)n)->is_ascii());
17431   match(Set result (EncodeISOArray src (Binary dst len)));
17432   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17433          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17434 
17435   format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17436   ins_encode %{
17437     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17438                         $result$$Register, true,
17439                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17440                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17441                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17442   %}
17443   ins_pipe(pipe_class_memory);
17444 %}
17445 
17446 //----------------------------- CompressBits/ExpandBits ------------------------
17447 
17448 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17449                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17450   match(Set dst (CompressBits src mask));
17451   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17452   format %{ "mov    $tsrc, $src\n\t"
17453             "mov    $tmask, $mask\n\t"
17454             "bext   $tdst, $tsrc, $tmask\n\t"
17455             "mov    $dst, $tdst"
17456           %}
17457   ins_encode %{
17458     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17459     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17460     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17461     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17462   %}
17463   ins_pipe(pipe_slow);
17464 %}
17465 
17466 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17467                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17468   match(Set dst (CompressBits (LoadI mem) mask));
17469   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17470   format %{ "ldrs   $tsrc, $mem\n\t"
17471             "ldrs   $tmask, $mask\n\t"
17472             "bext   $tdst, $tsrc, $tmask\n\t"
17473             "mov    $dst, $tdst"
17474           %}
17475   ins_encode %{
17476     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17477               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17478     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17479     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17480     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17481   %}
17482   ins_pipe(pipe_slow);
17483 %}
17484 
17485 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17486                            vRegD tdst, vRegD tsrc, vRegD tmask) %{
17487   match(Set dst (CompressBits src mask));
17488   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17489   format %{ "mov    $tsrc, $src\n\t"
17490             "mov    $tmask, $mask\n\t"
17491             "bext   $tdst, $tsrc, $tmask\n\t"
17492             "mov    $dst, $tdst"
17493           %}
17494   ins_encode %{
17495     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17496     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17497     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17498     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17499   %}
17500   ins_pipe(pipe_slow);
17501 %}
17502 
17503 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17504                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17505   match(Set dst (CompressBits (LoadL mem) mask));
17506   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17507   format %{ "ldrd   $tsrc, $mem\n\t"
17508             "ldrd   $tmask, $mask\n\t"
17509             "bext   $tdst, $tsrc, $tmask\n\t"
17510             "mov    $dst, $tdst"
17511           %}
17512   ins_encode %{
17513     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17514               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17515     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17516     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17517     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17518   %}
17519   ins_pipe(pipe_slow);
17520 %}
17521 
17522 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17523                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17524   match(Set dst (ExpandBits src mask));
17525   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17526   format %{ "mov    $tsrc, $src\n\t"
17527             "mov    $tmask, $mask\n\t"
17528             "bdep   $tdst, $tsrc, $tmask\n\t"
17529             "mov    $dst, $tdst"
17530           %}
17531   ins_encode %{
17532     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17533     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17534     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17535     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17536   %}
17537   ins_pipe(pipe_slow);
17538 %}
17539 
17540 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17541                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17542   match(Set dst (ExpandBits (LoadI mem) mask));
17543   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17544   format %{ "ldrs   $tsrc, $mem\n\t"
17545             "ldrs   $tmask, $mask\n\t"
17546             "bdep   $tdst, $tsrc, $tmask\n\t"
17547             "mov    $dst, $tdst"
17548           %}
17549   ins_encode %{
17550     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17551               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17552     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17553     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17554     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17555   %}
17556   ins_pipe(pipe_slow);
17557 %}
17558 
17559 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17560                          vRegD tdst, vRegD tsrc, vRegD tmask) %{
17561   match(Set dst (ExpandBits src mask));
17562   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17563   format %{ "mov    $tsrc, $src\n\t"
17564             "mov    $tmask, $mask\n\t"
17565             "bdep   $tdst, $tsrc, $tmask\n\t"
17566             "mov    $dst, $tdst"
17567           %}
17568   ins_encode %{
17569     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17570     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17571     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17572     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17573   %}
17574   ins_pipe(pipe_slow);
17575 %}
17576 
17577 
17578 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17579                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17580   match(Set dst (ExpandBits (LoadL mem) mask));
17581   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17582   format %{ "ldrd   $tsrc, $mem\n\t"
17583             "ldrd   $tmask, $mask\n\t"
17584             "bdep   $tdst, $tsrc, $tmask\n\t"
17585             "mov    $dst, $tdst"
17586           %}
17587   ins_encode %{
17588     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17589               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17590     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17591     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17592     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17593   %}
17594   ins_pipe(pipe_slow);
17595 %}
17596 
17597 // ============================================================================
17598 // This name is KNOWN by the ADLC and cannot be changed.
17599 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17600 // for this guy.
17601 instruct tlsLoadP(thread_RegP dst)
17602 %{
17603   match(Set dst (ThreadLocal));
17604 
17605   ins_cost(0);
17606 
17607   format %{ " -- \t// $dst=Thread::current(), empty" %}
17608 
17609   size(0);
17610 
17611   ins_encode( /*empty*/ );
17612 
17613   ins_pipe(pipe_class_empty);
17614 %}
17615 
17616 //----------PEEPHOLE RULES-----------------------------------------------------
17617 // These must follow all instruction definitions as they use the names
17618 // defined in the instructions definitions.
17619 //
17620 // peepmatch ( root_instr_name [preceding_instruction]* );
17621 //
17622 // peepconstraint %{
17623 // (instruction_number.operand_name relational_op instruction_number.operand_name
17624 //  [, ...] );
17625 // // instruction numbers are zero-based using left to right order in peepmatch
17626 //
17627 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17628 // // provide an instruction_number.operand_name for each operand that appears
17629 // // in the replacement instruction's match rule
17630 //
17631 // ---------VM FLAGS---------------------------------------------------------
17632 //
17633 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17634 //
17635 // Each peephole rule is given an identifying number starting with zero and
17636 // increasing by one in the order seen by the parser.  An individual peephole
17637 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17638 // on the command-line.
17639 //
17640 // ---------CURRENT LIMITATIONS----------------------------------------------
17641 //
17642 // Only match adjacent instructions in same basic block
17643 // Only equality constraints
17644 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17645 // Only one replacement instruction
17646 //
17647 // ---------EXAMPLE----------------------------------------------------------
17648 //
17649 // // pertinent parts of existing instructions in architecture description
17650 // instruct movI(iRegINoSp dst, iRegI src)
17651 // %{
17652 //   match(Set dst (CopyI src));
17653 // %}
17654 //
17655 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17656 // %{
17657 //   match(Set dst (AddI dst src));
17658 //   effect(KILL cr);
17659 // %}
17660 //
17661 // // Change (inc mov) to lea
17662 // peephole %{
17663 //   // increment preceded by register-register move
17664 //   peepmatch ( incI_iReg movI );
17665 //   // require that the destination register of the increment
17666 //   // match the destination register of the move
17667 //   peepconstraint ( 0.dst == 1.dst );
17668 //   // construct a replacement instruction that sets
17669 //   // the destination to ( move's source register + one )
17670 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17671 // %}
17672 //
17673 
17674 // Implementation no longer uses movX instructions since
17675 // machine-independent system no longer uses CopyX nodes.
17676 //
17677 // peephole
17678 // %{
17679 //   peepmatch (incI_iReg movI);
17680 //   peepconstraint (0.dst == 1.dst);
17681 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17682 // %}
17683 
17684 // peephole
17685 // %{
17686 //   peepmatch (decI_iReg movI);
17687 //   peepconstraint (0.dst == 1.dst);
17688 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17689 // %}
17690 
17691 // peephole
17692 // %{
17693 //   peepmatch (addI_iReg_imm movI);
17694 //   peepconstraint (0.dst == 1.dst);
17695 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17696 // %}
17697 
17698 // peephole
17699 // %{
17700 //   peepmatch (incL_iReg movL);
17701 //   peepconstraint (0.dst == 1.dst);
17702 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17703 // %}
17704 
17705 // peephole
17706 // %{
17707 //   peepmatch (decL_iReg movL);
17708 //   peepconstraint (0.dst == 1.dst);
17709 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17710 // %}
17711 
17712 // peephole
17713 // %{
17714 //   peepmatch (addL_iReg_imm movL);
17715 //   peepconstraint (0.dst == 1.dst);
17716 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17717 // %}
17718 
17719 // peephole
17720 // %{
17721 //   peepmatch (addP_iReg_imm movP);
17722 //   peepconstraint (0.dst == 1.dst);
17723 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17724 // %}
17725 
17726 // // Change load of spilled value to only a spill
17727 // instruct storeI(memory mem, iRegI src)
17728 // %{
17729 //   match(Set mem (StoreI mem src));
17730 // %}
17731 //
17732 // instruct loadI(iRegINoSp dst, memory mem)
17733 // %{
17734 //   match(Set dst (LoadI mem));
17735 // %}
17736 //
17737 
17738 //----------SMARTSPILL RULES---------------------------------------------------
17739 // These must follow all instruction definitions as they use the names
17740 // defined in the instructions definitions.
17741 
17742 // Local Variables:
17743 // mode: c++
17744 // End: