1 //
    2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // architecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182 
  183   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  184   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  185   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  186   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  187 
  188   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  189   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  190   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  191   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  192 
  193   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  194   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  195   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  196   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  197 
  198   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  199   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  200   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  201   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  202 
  203   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  204   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  205   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  206   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  207 
  208   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  209   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  210   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  211   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  212 
  213   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  214   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  215   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  216   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  217 
  218   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  219   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  220   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  221   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  222 
  223   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  224   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  225   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  226   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  227 
  228   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  229   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  230   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  231   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  232 
  233   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  234   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  235   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  236   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  237 
  238   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  239   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  240   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  241   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  242 
  243   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  244   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  245   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  246   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  247 
  248   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  249   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  250   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  251   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  252 
  253   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  254   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  255   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  256   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  257 
  258   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  259   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  260   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  261   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  262 
  263   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  264   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  265   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  266   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  267 
  268   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  269   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  270   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  271   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  272 
  273   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  274   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  275   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  276   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  277 
  278   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  279   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  280   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  281   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  282 
  283   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  284   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  285   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  286   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  287 
  288   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  289   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  290   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  291   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  292 
  293   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  294   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  295   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  296   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  297 
  298   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  299   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  300   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  301   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  302 
  303   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  304   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  305   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  306   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  307 
  308   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  309   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  310   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  311   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  312 
  313   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  314   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  315   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  316   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  317 
  318   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  319   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  320   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  321   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  322 
  323   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  324   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  325   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  326   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  327 
  328   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  329   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  330   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  331   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  332 
  333   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  334   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  335   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  336   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  337 
  338 // ----------------------------
  339 // SVE Predicate Registers
  340 // ----------------------------
  341   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  342   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  343   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  344   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  345   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  346   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  347   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  348   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  349   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  350   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  351   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  352   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  353   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  354   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  355   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  356   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  357 
  358 // ----------------------------
  359 // Special Registers
  360 // ----------------------------
  361 
  362 // the AArch64 CSPR status flag register is not directly accessible as
  363 // instruction operand. the FPSR status flag register is a system
  364 // register which can be written/read using MSR/MRS but again does not
  365 // appear as an operand (a code identifying the FSPR occurs as an
  366 // immediate value in the instruction).
  367 
  368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  369 
  370 // Specify priority of register selection within phases of register
  371 // allocation.  Highest priority is first.  A useful heuristic is to
  372 // give registers a low priority when they are required by machine
  373 // instructions, like EAX and EDX on I486, and choose no-save registers
  374 // before save-on-call, & save-on-call before save-on-entry.  Registers
  375 // which participate in fixed calling sequences should come last.
  376 // Registers which are used as pairs must fall on an even boundary.
  377 
  378 alloc_class chunk0(
  379     // volatiles
  380     R10, R10_H,
  381     R11, R11_H,
  382     R12, R12_H,
  383     R13, R13_H,
  384     R14, R14_H,
  385     R15, R15_H,
  386     R16, R16_H,
  387     R17, R17_H,
  388     R18, R18_H,
  389 
  390     // arg registers
  391     R0, R0_H,
  392     R1, R1_H,
  393     R2, R2_H,
  394     R3, R3_H,
  395     R4, R4_H,
  396     R5, R5_H,
  397     R6, R6_H,
  398     R7, R7_H,
  399 
  400     // non-volatiles
  401     R19, R19_H,
  402     R20, R20_H,
  403     R21, R21_H,
  404     R22, R22_H,
  405     R23, R23_H,
  406     R24, R24_H,
  407     R25, R25_H,
  408     R26, R26_H,
  409 
  410     // non-allocatable registers
  411 
  412     R27, R27_H, // heapbase
  413     R28, R28_H, // thread
  414     R29, R29_H, // fp
  415     R30, R30_H, // lr
  416     R31, R31_H, // sp
  417     R8, R8_H,   // rscratch1
  418     R9, R9_H,   // rscratch2
  419 );
  420 
  421 alloc_class chunk1(
  422 
  423     // no save
  424     V16, V16_H, V16_J, V16_K,
  425     V17, V17_H, V17_J, V17_K,
  426     V18, V18_H, V18_J, V18_K,
  427     V19, V19_H, V19_J, V19_K,
  428     V20, V20_H, V20_J, V20_K,
  429     V21, V21_H, V21_J, V21_K,
  430     V22, V22_H, V22_J, V22_K,
  431     V23, V23_H, V23_J, V23_K,
  432     V24, V24_H, V24_J, V24_K,
  433     V25, V25_H, V25_J, V25_K,
  434     V26, V26_H, V26_J, V26_K,
  435     V27, V27_H, V27_J, V27_K,
  436     V28, V28_H, V28_J, V28_K,
  437     V29, V29_H, V29_J, V29_K,
  438     V30, V30_H, V30_J, V30_K,
  439     V31, V31_H, V31_J, V31_K,
  440 
  441     // arg registers
  442     V0, V0_H, V0_J, V0_K,
  443     V1, V1_H, V1_J, V1_K,
  444     V2, V2_H, V2_J, V2_K,
  445     V3, V3_H, V3_J, V3_K,
  446     V4, V4_H, V4_J, V4_K,
  447     V5, V5_H, V5_J, V5_K,
  448     V6, V6_H, V6_J, V6_K,
  449     V7, V7_H, V7_J, V7_K,
  450 
  451     // non-volatiles
  452     V8, V8_H, V8_J, V8_K,
  453     V9, V9_H, V9_J, V9_K,
  454     V10, V10_H, V10_J, V10_K,
  455     V11, V11_H, V11_J, V11_K,
  456     V12, V12_H, V12_J, V12_K,
  457     V13, V13_H, V13_J, V13_K,
  458     V14, V14_H, V14_J, V14_K,
  459     V15, V15_H, V15_J, V15_K,
  460 );
  461 
  462 alloc_class chunk2 (
  463     // Governing predicates for load/store and arithmetic
  464     P0,
  465     P1,
  466     P2,
  467     P3,
  468     P4,
  469     P5,
  470     P6,
  471 
  472     // Extra predicates
  473     P8,
  474     P9,
  475     P10,
  476     P11,
  477     P12,
  478     P13,
  479     P14,
  480     P15,
  481 
  482     // Preserved for all-true predicate
  483     P7,
  484 );
  485 
  486 alloc_class chunk3(RFLAGS);
  487 
  488 //----------Architecture Description Register Classes--------------------------
  489 // Several register classes are automatically defined based upon information in
  490 // this architecture description.
  491 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  493 //
  494 
  495 // Class for all 32 bit general purpose registers
  496 reg_class all_reg32(
  497     R0,
  498     R1,
  499     R2,
  500     R3,
  501     R4,
  502     R5,
  503     R6,
  504     R7,
  505     R10,
  506     R11,
  507     R12,
  508     R13,
  509     R14,
  510     R15,
  511     R16,
  512     R17,
  513     R18,
  514     R19,
  515     R20,
  516     R21,
  517     R22,
  518     R23,
  519     R24,
  520     R25,
  521     R26,
  522     R27,
  523     R28,
  524     R29,
  525     R30,
  526     R31
  527 );
  528 
  529 
  530 // Class for all 32 bit integer registers (excluding SP which
  531 // will never be used as an integer register)
  532 reg_class any_reg32 %{
  533   return _ANY_REG32_mask;
  534 %}
  535 
  536 // Singleton class for R0 int register
  537 reg_class int_r0_reg(R0);
  538 
  539 // Singleton class for R2 int register
  540 reg_class int_r2_reg(R2);
  541 
  542 // Singleton class for R3 int register
  543 reg_class int_r3_reg(R3);
  544 
  545 // Singleton class for R4 int register
  546 reg_class int_r4_reg(R4);
  547 
  548 // Singleton class for R31 int register
  549 reg_class int_r31_reg(R31);
  550 
  551 // Class for all 64 bit general purpose registers
  552 reg_class all_reg(
  553     R0, R0_H,
  554     R1, R1_H,
  555     R2, R2_H,
  556     R3, R3_H,
  557     R4, R4_H,
  558     R5, R5_H,
  559     R6, R6_H,
  560     R7, R7_H,
  561     R10, R10_H,
  562     R11, R11_H,
  563     R12, R12_H,
  564     R13, R13_H,
  565     R14, R14_H,
  566     R15, R15_H,
  567     R16, R16_H,
  568     R17, R17_H,
  569     R18, R18_H,
  570     R19, R19_H,
  571     R20, R20_H,
  572     R21, R21_H,
  573     R22, R22_H,
  574     R23, R23_H,
  575     R24, R24_H,
  576     R25, R25_H,
  577     R26, R26_H,
  578     R27, R27_H,
  579     R28, R28_H,
  580     R29, R29_H,
  581     R30, R30_H,
  582     R31, R31_H
  583 );
  584 
  585 // Class for all long integer registers (including SP)
  586 reg_class any_reg %{
  587   return _ANY_REG_mask;
  588 %}
  589 
  590 // Class for non-allocatable 32 bit registers
  591 reg_class non_allocatable_reg32(
  592 #ifdef R18_RESERVED
  593     // See comment in register_aarch64.hpp
  594     R18,                        // tls on Windows
  595 #endif
  596     R28,                        // thread
  597     R30,                        // lr
  598     R31                         // sp
  599 );
  600 
  601 // Class for non-allocatable 64 bit registers
  602 reg_class non_allocatable_reg(
  603 #ifdef R18_RESERVED
  604     // See comment in register_aarch64.hpp
  605     R18, R18_H,                 // tls on Windows, platform register on macOS
  606 #endif
  607     R28, R28_H,                 // thread
  608     R30, R30_H,                 // lr
  609     R31, R31_H                  // sp
  610 );
  611 
  612 // Class for all non-special integer registers
  613 reg_class no_special_reg32 %{
  614   return _NO_SPECIAL_REG32_mask;
  615 %}
  616 
  617 // Class for all non-special long integer registers
  618 reg_class no_special_reg %{
  619   return _NO_SPECIAL_REG_mask;
  620 %}
  621 
  622 // Class for 64 bit register r0
  623 reg_class r0_reg(
  624     R0, R0_H
  625 );
  626 
  627 // Class for 64 bit register r1
  628 reg_class r1_reg(
  629     R1, R1_H
  630 );
  631 
  632 // Class for 64 bit register r2
  633 reg_class r2_reg(
  634     R2, R2_H
  635 );
  636 
  637 // Class for 64 bit register r3
  638 reg_class r3_reg(
  639     R3, R3_H
  640 );
  641 
  642 // Class for 64 bit register r4
  643 reg_class r4_reg(
  644     R4, R4_H
  645 );
  646 
  647 // Class for 64 bit register r5
  648 reg_class r5_reg(
  649     R5, R5_H
  650 );
  651 
  652 // Class for 64 bit register r10
  653 reg_class r10_reg(
  654     R10, R10_H
  655 );
  656 
  657 // Class for 64 bit register r11
  658 reg_class r11_reg(
  659     R11, R11_H
  660 );
  661 
  662 // Class for method register
  663 reg_class method_reg(
  664     R12, R12_H
  665 );
  666 
  667 // Class for thread register
  668 reg_class thread_reg(
  669     R28, R28_H
  670 );
  671 
  672 // Class for frame pointer register
  673 reg_class fp_reg(
  674     R29, R29_H
  675 );
  676 
  677 // Class for link register
  678 reg_class lr_reg(
  679     R30, R30_H
  680 );
  681 
  682 // Class for long sp register
  683 reg_class sp_reg(
  684   R31, R31_H
  685 );
  686 
  687 // Class for all pointer registers
  688 reg_class ptr_reg %{
  689   return _PTR_REG_mask;
  690 %}
  691 
  692 // Class for all non_special pointer registers
  693 reg_class no_special_ptr_reg %{
  694   return _NO_SPECIAL_PTR_REG_mask;
  695 %}
  696 
  697 // Class for all non_special pointer registers (excluding rfp)
  698 reg_class no_special_no_rfp_ptr_reg %{
  699   return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
  700 %}
  701 
  702 // Class for all float registers
  703 reg_class float_reg(
  704     V0,
  705     V1,
  706     V2,
  707     V3,
  708     V4,
  709     V5,
  710     V6,
  711     V7,
  712     V8,
  713     V9,
  714     V10,
  715     V11,
  716     V12,
  717     V13,
  718     V14,
  719     V15,
  720     V16,
  721     V17,
  722     V18,
  723     V19,
  724     V20,
  725     V21,
  726     V22,
  727     V23,
  728     V24,
  729     V25,
  730     V26,
  731     V27,
  732     V28,
  733     V29,
  734     V30,
  735     V31
  736 );
  737 
  738 // Double precision float registers have virtual `high halves' that
  739 // are needed by the allocator.
  740 // Class for all double registers
  741 reg_class double_reg(
  742     V0, V0_H,
  743     V1, V1_H,
  744     V2, V2_H,
  745     V3, V3_H,
  746     V4, V4_H,
  747     V5, V5_H,
  748     V6, V6_H,
  749     V7, V7_H,
  750     V8, V8_H,
  751     V9, V9_H,
  752     V10, V10_H,
  753     V11, V11_H,
  754     V12, V12_H,
  755     V13, V13_H,
  756     V14, V14_H,
  757     V15, V15_H,
  758     V16, V16_H,
  759     V17, V17_H,
  760     V18, V18_H,
  761     V19, V19_H,
  762     V20, V20_H,
  763     V21, V21_H,
  764     V22, V22_H,
  765     V23, V23_H,
  766     V24, V24_H,
  767     V25, V25_H,
  768     V26, V26_H,
  769     V27, V27_H,
  770     V28, V28_H,
  771     V29, V29_H,
  772     V30, V30_H,
  773     V31, V31_H
  774 );
  775 
  776 // Class for all SVE vector registers.
  777 reg_class vectora_reg (
  778     V0, V0_H, V0_J, V0_K,
  779     V1, V1_H, V1_J, V1_K,
  780     V2, V2_H, V2_J, V2_K,
  781     V3, V3_H, V3_J, V3_K,
  782     V4, V4_H, V4_J, V4_K,
  783     V5, V5_H, V5_J, V5_K,
  784     V6, V6_H, V6_J, V6_K,
  785     V7, V7_H, V7_J, V7_K,
  786     V8, V8_H, V8_J, V8_K,
  787     V9, V9_H, V9_J, V9_K,
  788     V10, V10_H, V10_J, V10_K,
  789     V11, V11_H, V11_J, V11_K,
  790     V12, V12_H, V12_J, V12_K,
  791     V13, V13_H, V13_J, V13_K,
  792     V14, V14_H, V14_J, V14_K,
  793     V15, V15_H, V15_J, V15_K,
  794     V16, V16_H, V16_J, V16_K,
  795     V17, V17_H, V17_J, V17_K,
  796     V18, V18_H, V18_J, V18_K,
  797     V19, V19_H, V19_J, V19_K,
  798     V20, V20_H, V20_J, V20_K,
  799     V21, V21_H, V21_J, V21_K,
  800     V22, V22_H, V22_J, V22_K,
  801     V23, V23_H, V23_J, V23_K,
  802     V24, V24_H, V24_J, V24_K,
  803     V25, V25_H, V25_J, V25_K,
  804     V26, V26_H, V26_J, V26_K,
  805     V27, V27_H, V27_J, V27_K,
  806     V28, V28_H, V28_J, V28_K,
  807     V29, V29_H, V29_J, V29_K,
  808     V30, V30_H, V30_J, V30_K,
  809     V31, V31_H, V31_J, V31_K,
  810 );
  811 
  812 // Class for all 64bit vector registers
  813 reg_class vectord_reg(
  814     V0, V0_H,
  815     V1, V1_H,
  816     V2, V2_H,
  817     V3, V3_H,
  818     V4, V4_H,
  819     V5, V5_H,
  820     V6, V6_H,
  821     V7, V7_H,
  822     V8, V8_H,
  823     V9, V9_H,
  824     V10, V10_H,
  825     V11, V11_H,
  826     V12, V12_H,
  827     V13, V13_H,
  828     V14, V14_H,
  829     V15, V15_H,
  830     V16, V16_H,
  831     V17, V17_H,
  832     V18, V18_H,
  833     V19, V19_H,
  834     V20, V20_H,
  835     V21, V21_H,
  836     V22, V22_H,
  837     V23, V23_H,
  838     V24, V24_H,
  839     V25, V25_H,
  840     V26, V26_H,
  841     V27, V27_H,
  842     V28, V28_H,
  843     V29, V29_H,
  844     V30, V30_H,
  845     V31, V31_H
  846 );
  847 
  848 // Class for all 128bit vector registers
  849 reg_class vectorx_reg(
  850     V0, V0_H, V0_J, V0_K,
  851     V1, V1_H, V1_J, V1_K,
  852     V2, V2_H, V2_J, V2_K,
  853     V3, V3_H, V3_J, V3_K,
  854     V4, V4_H, V4_J, V4_K,
  855     V5, V5_H, V5_J, V5_K,
  856     V6, V6_H, V6_J, V6_K,
  857     V7, V7_H, V7_J, V7_K,
  858     V8, V8_H, V8_J, V8_K,
  859     V9, V9_H, V9_J, V9_K,
  860     V10, V10_H, V10_J, V10_K,
  861     V11, V11_H, V11_J, V11_K,
  862     V12, V12_H, V12_J, V12_K,
  863     V13, V13_H, V13_J, V13_K,
  864     V14, V14_H, V14_J, V14_K,
  865     V15, V15_H, V15_J, V15_K,
  866     V16, V16_H, V16_J, V16_K,
  867     V17, V17_H, V17_J, V17_K,
  868     V18, V18_H, V18_J, V18_K,
  869     V19, V19_H, V19_J, V19_K,
  870     V20, V20_H, V20_J, V20_K,
  871     V21, V21_H, V21_J, V21_K,
  872     V22, V22_H, V22_J, V22_K,
  873     V23, V23_H, V23_J, V23_K,
  874     V24, V24_H, V24_J, V24_K,
  875     V25, V25_H, V25_J, V25_K,
  876     V26, V26_H, V26_J, V26_K,
  877     V27, V27_H, V27_J, V27_K,
  878     V28, V28_H, V28_J, V28_K,
  879     V29, V29_H, V29_J, V29_K,
  880     V30, V30_H, V30_J, V30_K,
  881     V31, V31_H, V31_J, V31_K
  882 );
  883 
  884 // Class for vector register V10
  885 reg_class v10_veca_reg(
  886     V10, V10_H, V10_J, V10_K
  887 );
  888 
  889 // Class for vector register V11
  890 reg_class v11_veca_reg(
  891     V11, V11_H, V11_J, V11_K
  892 );
  893 
  894 // Class for vector register V12
  895 reg_class v12_veca_reg(
  896     V12, V12_H, V12_J, V12_K
  897 );
  898 
  899 // Class for vector register V13
  900 reg_class v13_veca_reg(
  901     V13, V13_H, V13_J, V13_K
  902 );
  903 
  904 // Class for vector register V17
  905 reg_class v17_veca_reg(
  906     V17, V17_H, V17_J, V17_K
  907 );
  908 
  909 // Class for vector register V18
  910 reg_class v18_veca_reg(
  911     V18, V18_H, V18_J, V18_K
  912 );
  913 
  914 // Class for vector register V23
  915 reg_class v23_veca_reg(
  916     V23, V23_H, V23_J, V23_K
  917 );
  918 
  919 // Class for vector register V24
  920 reg_class v24_veca_reg(
  921     V24, V24_H, V24_J, V24_K
  922 );
  923 
  924 // Class for 128 bit register v0
  925 reg_class v0_reg(
  926     V0, V0_H
  927 );
  928 
  929 // Class for 128 bit register v1
  930 reg_class v1_reg(
  931     V1, V1_H
  932 );
  933 
  934 // Class for 128 bit register v2
  935 reg_class v2_reg(
  936     V2, V2_H
  937 );
  938 
  939 // Class for 128 bit register v3
  940 reg_class v3_reg(
  941     V3, V3_H
  942 );
  943 
  944 // Class for 128 bit register v4
  945 reg_class v4_reg(
  946     V4, V4_H
  947 );
  948 
  949 // Class for 128 bit register v5
  950 reg_class v5_reg(
  951     V5, V5_H
  952 );
  953 
  954 // Class for 128 bit register v6
  955 reg_class v6_reg(
  956     V6, V6_H
  957 );
  958 
  959 // Class for 128 bit register v7
  960 reg_class v7_reg(
  961     V7, V7_H
  962 );
  963 
  964 // Class for 128 bit register v8
  965 reg_class v8_reg(
  966     V8, V8_H
  967 );
  968 
  969 // Class for 128 bit register v9
  970 reg_class v9_reg(
  971     V9, V9_H
  972 );
  973 
  974 // Class for 128 bit register v10
  975 reg_class v10_reg(
  976     V10, V10_H
  977 );
  978 
  979 // Class for 128 bit register v11
  980 reg_class v11_reg(
  981     V11, V11_H
  982 );
  983 
  984 // Class for 128 bit register v12
  985 reg_class v12_reg(
  986     V12, V12_H
  987 );
  988 
  989 // Class for 128 bit register v13
  990 reg_class v13_reg(
  991     V13, V13_H
  992 );
  993 
  994 // Class for 128 bit register v14
  995 reg_class v14_reg(
  996     V14, V14_H
  997 );
  998 
  999 // Class for 128 bit register v15
 1000 reg_class v15_reg(
 1001     V15, V15_H
 1002 );
 1003 
 1004 // Class for 128 bit register v16
 1005 reg_class v16_reg(
 1006     V16, V16_H
 1007 );
 1008 
 1009 // Class for 128 bit register v17
 1010 reg_class v17_reg(
 1011     V17, V17_H
 1012 );
 1013 
 1014 // Class for 128 bit register v18
 1015 reg_class v18_reg(
 1016     V18, V18_H
 1017 );
 1018 
 1019 // Class for 128 bit register v19
 1020 reg_class v19_reg(
 1021     V19, V19_H
 1022 );
 1023 
 1024 // Class for 128 bit register v20
 1025 reg_class v20_reg(
 1026     V20, V20_H
 1027 );
 1028 
 1029 // Class for 128 bit register v21
 1030 reg_class v21_reg(
 1031     V21, V21_H
 1032 );
 1033 
 1034 // Class for 128 bit register v22
 1035 reg_class v22_reg(
 1036     V22, V22_H
 1037 );
 1038 
 1039 // Class for 128 bit register v23
 1040 reg_class v23_reg(
 1041     V23, V23_H
 1042 );
 1043 
 1044 // Class for 128 bit register v24
 1045 reg_class v24_reg(
 1046     V24, V24_H
 1047 );
 1048 
 1049 // Class for 128 bit register v25
 1050 reg_class v25_reg(
 1051     V25, V25_H
 1052 );
 1053 
 1054 // Class for 128 bit register v26
 1055 reg_class v26_reg(
 1056     V26, V26_H
 1057 );
 1058 
 1059 // Class for 128 bit register v27
 1060 reg_class v27_reg(
 1061     V27, V27_H
 1062 );
 1063 
 1064 // Class for 128 bit register v28
 1065 reg_class v28_reg(
 1066     V28, V28_H
 1067 );
 1068 
 1069 // Class for 128 bit register v29
 1070 reg_class v29_reg(
 1071     V29, V29_H
 1072 );
 1073 
 1074 // Class for 128 bit register v30
 1075 reg_class v30_reg(
 1076     V30, V30_H
 1077 );
 1078 
 1079 // Class for 128 bit register v31
 1080 reg_class v31_reg(
 1081     V31, V31_H
 1082 );
 1083 
 1084 // Class for all SVE predicate registers.
 1085 reg_class pr_reg (
 1086     P0,
 1087     P1,
 1088     P2,
 1089     P3,
 1090     P4,
 1091     P5,
 1092     P6,
 1093     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1094     P8,
 1095     P9,
 1096     P10,
 1097     P11,
 1098     P12,
 1099     P13,
 1100     P14,
 1101     P15
 1102 );
 1103 
 1104 // Class for SVE governing predicate registers, which are used
 1105 // to determine the active elements of a predicated instruction.
 1106 reg_class gov_pr (
 1107     P0,
 1108     P1,
 1109     P2,
 1110     P3,
 1111     P4,
 1112     P5,
 1113     P6,
 1114     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1115 );
 1116 
 1117 reg_class p0_reg(P0);
 1118 reg_class p1_reg(P1);
 1119 
 1120 // Singleton class for condition codes
 1121 reg_class int_flags(RFLAGS);
 1122 
 1123 %}
 1124 
 1125 //----------DEFINITION BLOCK---------------------------------------------------
 1126 // Define name --> value mappings to inform the ADLC of an integer valued name
 1127 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1128 // Format:
 1129 //        int_def  <name>         ( <int_value>, <expression>);
 1130 // Generated Code in ad_<arch>.hpp
 1131 //        #define  <name>   (<expression>)
 1132 //        // value == <int_value>
 1133 // Generated code in ad_<arch>.cpp adlc_verification()
 1134 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1135 //
 1136 
 1137 // we follow the ppc-aix port in using a simple cost model which ranks
 1138 // register operations as cheap, memory ops as more expensive and
 1139 // branches as most expensive. the first two have a low as well as a
 1140 // normal cost. huge cost appears to be a way of saying don't do
 1141 // something
 1142 
 1143 definitions %{
 1144   // The default cost (of a register move instruction).
 1145   int_def INSN_COST            (    100,     100);
 1146   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1147   int_def CALL_COST            (    200,     2 * INSN_COST);
 1148   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1149 %}
 1150 
 1151 
 1152 //----------SOURCE BLOCK-------------------------------------------------------
 1153 // This is a block of C++ code which provides values, functions, and
 1154 // definitions necessary in the rest of the architecture description
 1155 
 1156 source_hpp %{
 1157 
 1158 #include "asm/macroAssembler.hpp"
 1159 #include "gc/shared/barrierSetAssembler.hpp"
 1160 #include "gc/shared/cardTable.hpp"
 1161 #include "gc/shared/cardTableBarrierSet.hpp"
 1162 #include "gc/shared/collectedHeap.hpp"
 1163 #include "opto/addnode.hpp"
 1164 #include "opto/convertnode.hpp"
 1165 #include "runtime/objectMonitor.hpp"
 1166 
 1167 extern RegMask _ANY_REG32_mask;
 1168 extern RegMask _ANY_REG_mask;
 1169 extern RegMask _PTR_REG_mask;
 1170 extern RegMask _NO_SPECIAL_REG32_mask;
 1171 extern RegMask _NO_SPECIAL_REG_mask;
 1172 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1173 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
 1174 
 1175 class CallStubImpl {
 1176 
 1177   //--------------------------------------------------------------
 1178   //---<  Used for optimization in Compile::shorten_branches  >---
 1179   //--------------------------------------------------------------
 1180 
 1181  public:
 1182   // Size of call trampoline stub.
 1183   static uint size_call_trampoline() {
 1184     return 0; // no call trampolines on this platform
 1185   }
 1186 
 1187   // number of relocations needed by a call trampoline stub
 1188   static uint reloc_call_trampoline() {
 1189     return 0; // no call trampolines on this platform
 1190   }
 1191 };
 1192 
 1193 class HandlerImpl {
 1194 
 1195  public:
 1196 
 1197   static int emit_exception_handler(C2_MacroAssembler *masm);
 1198   static int emit_deopt_handler(C2_MacroAssembler* masm);
 1199 
 1200   static uint size_exception_handler() {
 1201     return MacroAssembler::far_codestub_branch_size();
 1202   }
 1203 
 1204   static uint size_deopt_handler() {
 1205     // count one adr and one far branch instruction
 1206     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1207   }
 1208 };
 1209 
 1210 class Node::PD {
 1211 public:
 1212   enum NodeFlags {
 1213     _last_flag = Node::_last_flag
 1214   };
 1215 };
 1216 
 1217   bool is_CAS(int opcode, bool maybe_volatile);
 1218 
 1219   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1220 
 1221   bool unnecessary_acquire(const Node *barrier);
 1222   bool needs_acquiring_load(const Node *load);
 1223 
 1224   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1225 
 1226   bool unnecessary_release(const Node *barrier);
 1227   bool unnecessary_volatile(const Node *barrier);
 1228   bool needs_releasing_store(const Node *store);
 1229 
 1230   // predicate controlling translation of CompareAndSwapX
 1231   bool needs_acquiring_load_exclusive(const Node *load);
 1232 
 1233   // predicate controlling addressing modes
 1234   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1235 
 1236   // Convert BootTest condition to Assembler condition.
 1237   // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 1238   Assembler::Condition to_assembler_cond(BoolTest::mask cond);
 1239 %}
 1240 
 1241 source %{
 1242 
 1243   // Derived RegMask with conditionally allocatable registers
 1244 
 1245   void PhaseOutput::pd_perform_mach_node_analysis() {
 1246   }
 1247 
 1248   int MachNode::pd_alignment_required() const {
 1249     return 1;
 1250   }
 1251 
 1252   int MachNode::compute_padding(int current_offset) const {
 1253     return 0;
 1254   }
 1255 
 1256   RegMask _ANY_REG32_mask;
 1257   RegMask _ANY_REG_mask;
 1258   RegMask _PTR_REG_mask;
 1259   RegMask _NO_SPECIAL_REG32_mask;
 1260   RegMask _NO_SPECIAL_REG_mask;
 1261   RegMask _NO_SPECIAL_PTR_REG_mask;
 1262   RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
 1263 
 1264   void reg_mask_init() {
 1265     // We derive below RegMask(s) from the ones which are auto-generated from
 1266     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1267     // registers conditionally reserved.
 1268 
 1269     _ANY_REG32_mask = _ALL_REG32_mask;
 1270     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1271 
 1272     _ANY_REG_mask = _ALL_REG_mask;
 1273 
 1274     _PTR_REG_mask = _ALL_REG_mask;
 1275 
 1276     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1277     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1278 
 1279     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1280     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1281 
 1282     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1283     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1284 
 1285     // r27 is not allocatable when compressed oops is on and heapbase is not
 1286     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1287     if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
 1288       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1289       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1290       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1291     }
 1292 
 1293     // r29 is not allocatable when PreserveFramePointer is on
 1294     if (PreserveFramePointer) {
 1295       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1296       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1297       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1298     }
 1299 
 1300     _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
 1301     _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1302   }
 1303 
 1304   // Optimizaton of volatile gets and puts
 1305   // -------------------------------------
 1306   //
 1307   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1308   // use to implement volatile reads and writes. For a volatile read
 1309   // we simply need
 1310   //
 1311   //   ldar<x>
 1312   //
 1313   // and for a volatile write we need
 1314   //
 1315   //   stlr<x>
 1316   //
 1317   // Alternatively, we can implement them by pairing a normal
 1318   // load/store with a memory barrier. For a volatile read we need
 1319   //
 1320   //   ldr<x>
 1321   //   dmb ishld
 1322   //
 1323   // for a volatile write
 1324   //
 1325   //   dmb ish
 1326   //   str<x>
 1327   //   dmb ish
 1328   //
 1329   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1330   // sequences. These are normally translated to an instruction
 1331   // sequence like the following
 1332   //
 1333   //   dmb      ish
 1334   // retry:
 1335   //   ldxr<x>   rval raddr
 1336   //   cmp       rval rold
 1337   //   b.ne done
 1338   //   stlxr<x>  rval, rnew, rold
 1339   //   cbnz      rval retry
 1340   // done:
 1341   //   cset      r0, eq
 1342   //   dmb ishld
 1343   //
 1344   // Note that the exclusive store is already using an stlxr
 1345   // instruction. That is required to ensure visibility to other
 1346   // threads of the exclusive write (assuming it succeeds) before that
 1347   // of any subsequent writes.
 1348   //
 1349   // The following instruction sequence is an improvement on the above
 1350   //
 1351   // retry:
 1352   //   ldaxr<x>  rval raddr
 1353   //   cmp       rval rold
 1354   //   b.ne done
 1355   //   stlxr<x>  rval, rnew, rold
 1356   //   cbnz      rval retry
 1357   // done:
 1358   //   cset      r0, eq
 1359   //
 1360   // We don't need the leading dmb ish since the stlxr guarantees
 1361   // visibility of prior writes in the case that the swap is
 1362   // successful. Crucially we don't have to worry about the case where
 1363   // the swap is not successful since no valid program should be
 1364   // relying on visibility of prior changes by the attempting thread
 1365   // in the case where the CAS fails.
 1366   //
 1367   // Similarly, we don't need the trailing dmb ishld if we substitute
 1368   // an ldaxr instruction since that will provide all the guarantees we
 1369   // require regarding observation of changes made by other threads
 1370   // before any change to the CAS address observed by the load.
 1371   //
 1372   // In order to generate the desired instruction sequence we need to
 1373   // be able to identify specific 'signature' ideal graph node
 1374   // sequences which i) occur as a translation of a volatile reads or
 1375   // writes or CAS operations and ii) do not occur through any other
 1376   // translation or graph transformation. We can then provide
 1377   // alternative aldc matching rules which translate these node
 1378   // sequences to the desired machine code sequences. Selection of the
 1379   // alternative rules can be implemented by predicates which identify
 1380   // the relevant node sequences.
 1381   //
 1382   // The ideal graph generator translates a volatile read to the node
 1383   // sequence
 1384   //
 1385   //   LoadX[mo_acquire]
 1386   //   MemBarAcquire
 1387   //
 1388   // As a special case when using the compressed oops optimization we
 1389   // may also see this variant
 1390   //
 1391   //   LoadN[mo_acquire]
 1392   //   DecodeN
 1393   //   MemBarAcquire
 1394   //
 1395   // A volatile write is translated to the node sequence
 1396   //
 1397   //   MemBarRelease
 1398   //   StoreX[mo_release] {CardMark}-optional
 1399   //   MemBarVolatile
 1400   //
 1401   // n.b. the above node patterns are generated with a strict
 1402   // 'signature' configuration of input and output dependencies (see
 1403   // the predicates below for exact details). The card mark may be as
 1404   // simple as a few extra nodes or, in a few GC configurations, may
 1405   // include more complex control flow between the leading and
 1406   // trailing memory barriers. However, whatever the card mark
 1407   // configuration these signatures are unique to translated volatile
 1408   // reads/stores -- they will not appear as a result of any other
 1409   // bytecode translation or inlining nor as a consequence of
 1410   // optimizing transforms.
 1411   //
 1412   // We also want to catch inlined unsafe volatile gets and puts and
 1413   // be able to implement them using either ldar<x>/stlr<x> or some
 1414   // combination of ldr<x>/stlr<x> and dmb instructions.
 1415   //
 1416   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1417   // normal volatile put node sequence containing an extra cpuorder
 1418   // membar
 1419   //
 1420   //   MemBarRelease
 1421   //   MemBarCPUOrder
 1422   //   StoreX[mo_release] {CardMark}-optional
 1423   //   MemBarCPUOrder
 1424   //   MemBarVolatile
 1425   //
 1426   // n.b. as an aside, a cpuorder membar is not itself subject to
 1427   // matching and translation by adlc rules.  However, the rule
 1428   // predicates need to detect its presence in order to correctly
 1429   // select the desired adlc rules.
 1430   //
 1431   // Inlined unsafe volatile gets manifest as a slightly different
 1432   // node sequence to a normal volatile get because of the
 1433   // introduction of some CPUOrder memory barriers to bracket the
 1434   // Load. However, but the same basic skeleton of a LoadX feeding a
 1435   // MemBarAcquire, possibly through an optional DecodeN, is still
 1436   // present
 1437   //
 1438   //   MemBarCPUOrder
 1439   //        ||       \\
 1440   //   MemBarCPUOrder LoadX[mo_acquire]
 1441   //        ||            |
 1442   //        ||       {DecodeN} optional
 1443   //        ||       /
 1444   //     MemBarAcquire
 1445   //
 1446   // In this case the acquire membar does not directly depend on the
 1447   // load. However, we can be sure that the load is generated from an
 1448   // inlined unsafe volatile get if we see it dependent on this unique
 1449   // sequence of membar nodes. Similarly, given an acquire membar we
 1450   // can know that it was added because of an inlined unsafe volatile
 1451   // get if it is fed and feeds a cpuorder membar and if its feed
 1452   // membar also feeds an acquiring load.
 1453   //
 1454   // Finally an inlined (Unsafe) CAS operation is translated to the
 1455   // following ideal graph
 1456   //
 1457   //   MemBarRelease
 1458   //   MemBarCPUOrder
 1459   //   CompareAndSwapX {CardMark}-optional
 1460   //   MemBarCPUOrder
 1461   //   MemBarAcquire
 1462   //
 1463   // So, where we can identify these volatile read and write
 1464   // signatures we can choose to plant either of the above two code
 1465   // sequences. For a volatile read we can simply plant a normal
 1466   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1467   // also choose to inhibit translation of the MemBarAcquire and
 1468   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1469   //
 1470   // When we recognise a volatile store signature we can choose to
 1471   // plant at a dmb ish as a translation for the MemBarRelease, a
 1472   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1473   // Alternatively, we can inhibit translation of the MemBarRelease
 1474   // and MemBarVolatile and instead plant a simple stlr<x>
 1475   // instruction.
 1476   //
 1477   // when we recognise a CAS signature we can choose to plant a dmb
 1478   // ish as a translation for the MemBarRelease, the conventional
 1479   // macro-instruction sequence for the CompareAndSwap node (which
 1480   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1481   // Alternatively, we can elide generation of the dmb instructions
 1482   // and plant the alternative CompareAndSwap macro-instruction
 1483   // sequence (which uses ldaxr<x>).
 1484   //
 1485   // Of course, the above only applies when we see these signature
 1486   // configurations. We still want to plant dmb instructions in any
 1487   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1488   // MemBarVolatile. For example, at the end of a constructor which
 1489   // writes final/volatile fields we will see a MemBarRelease
 1490   // instruction and this needs a 'dmb ish' lest we risk the
 1491   // constructed object being visible without making the
 1492   // final/volatile field writes visible.
 1493   //
 1494   // n.b. the translation rules below which rely on detection of the
 1495   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1496   // If we see anything other than the signature configurations we
 1497   // always just translate the loads and stores to ldr<x> and str<x>
 1498   // and translate acquire, release and volatile membars to the
 1499   // relevant dmb instructions.
 1500   //
 1501 
 1502   // is_CAS(int opcode, bool maybe_volatile)
 1503   //
 1504   // return true if opcode is one of the possible CompareAndSwapX
 1505   // values otherwise false.
 1506 
 1507   bool is_CAS(int opcode, bool maybe_volatile)
 1508   {
 1509     switch(opcode) {
 1510       // We handle these
 1511     case Op_CompareAndSwapI:
 1512     case Op_CompareAndSwapL:
 1513     case Op_CompareAndSwapP:
 1514     case Op_CompareAndSwapN:
 1515     case Op_ShenandoahCompareAndSwapP:
 1516     case Op_ShenandoahCompareAndSwapN:
 1517     case Op_CompareAndSwapB:
 1518     case Op_CompareAndSwapS:
 1519     case Op_GetAndSetI:
 1520     case Op_GetAndSetL:
 1521     case Op_GetAndSetP:
 1522     case Op_GetAndSetN:
 1523     case Op_GetAndAddI:
 1524     case Op_GetAndAddL:
 1525       return true;
 1526     case Op_CompareAndExchangeI:
 1527     case Op_CompareAndExchangeN:
 1528     case Op_CompareAndExchangeB:
 1529     case Op_CompareAndExchangeS:
 1530     case Op_CompareAndExchangeL:
 1531     case Op_CompareAndExchangeP:
 1532     case Op_WeakCompareAndSwapB:
 1533     case Op_WeakCompareAndSwapS:
 1534     case Op_WeakCompareAndSwapI:
 1535     case Op_WeakCompareAndSwapL:
 1536     case Op_WeakCompareAndSwapP:
 1537     case Op_WeakCompareAndSwapN:
 1538     case Op_ShenandoahWeakCompareAndSwapP:
 1539     case Op_ShenandoahWeakCompareAndSwapN:
 1540     case Op_ShenandoahCompareAndExchangeP:
 1541     case Op_ShenandoahCompareAndExchangeN:
 1542       return maybe_volatile;
 1543     default:
 1544       return false;
 1545     }
 1546   }
 1547 
 1548   // helper to determine the maximum number of Phi nodes we may need to
 1549   // traverse when searching from a card mark membar for the merge mem
 1550   // feeding a trailing membar or vice versa
 1551 
 1552 // predicates controlling emit of ldr<x>/ldar<x>
 1553 
 1554 bool unnecessary_acquire(const Node *barrier)
 1555 {
 1556   assert(barrier->is_MemBar(), "expecting a membar");
 1557 
 1558   MemBarNode* mb = barrier->as_MemBar();
 1559 
 1560   if (mb->trailing_load()) {
 1561     return true;
 1562   }
 1563 
 1564   if (mb->trailing_load_store()) {
 1565     Node* load_store = mb->in(MemBarNode::Precedent);
 1566     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1567     return is_CAS(load_store->Opcode(), true);
 1568   }
 1569 
 1570   return false;
 1571 }
 1572 
 1573 bool needs_acquiring_load(const Node *n)
 1574 {
 1575   assert(n->is_Load(), "expecting a load");
 1576   LoadNode *ld = n->as_Load();
 1577   return ld->is_acquire();
 1578 }
 1579 
 1580 bool unnecessary_release(const Node *n)
 1581 {
 1582   assert((n->is_MemBar() &&
 1583           n->Opcode() == Op_MemBarRelease),
 1584          "expecting a release membar");
 1585 
 1586   MemBarNode *barrier = n->as_MemBar();
 1587   if (!barrier->leading()) {
 1588     return false;
 1589   } else {
 1590     Node* trailing = barrier->trailing_membar();
 1591     MemBarNode* trailing_mb = trailing->as_MemBar();
 1592     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1593     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1594 
 1595     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1596     if (mem->is_Store()) {
 1597       assert(mem->as_Store()->is_release(), "");
 1598       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1599       return true;
 1600     } else {
 1601       assert(mem->is_LoadStore(), "");
 1602       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1603       return is_CAS(mem->Opcode(), true);
 1604     }
 1605   }
 1606   return false;
 1607 }
 1608 
 1609 bool unnecessary_volatile(const Node *n)
 1610 {
 1611   // assert n->is_MemBar();
 1612   MemBarNode *mbvol = n->as_MemBar();
 1613 
 1614   bool release = mbvol->trailing_store();
 1615   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1616 #ifdef ASSERT
 1617   if (release) {
 1618     Node* leading = mbvol->leading_membar();
 1619     assert(leading->Opcode() == Op_MemBarRelease, "");
 1620     assert(leading->as_MemBar()->leading_store(), "");
 1621     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1622   }
 1623 #endif
 1624 
 1625   return release;
 1626 }
 1627 
 1628 // predicates controlling emit of str<x>/stlr<x>
 1629 
 1630 bool needs_releasing_store(const Node *n)
 1631 {
 1632   // assert n->is_Store();
 1633   StoreNode *st = n->as_Store();
 1634   return st->trailing_membar() != nullptr;
 1635 }
 1636 
 1637 // predicate controlling translation of CAS
 1638 //
 1639 // returns true if CAS needs to use an acquiring load otherwise false
 1640 
 1641 bool needs_acquiring_load_exclusive(const Node *n)
 1642 {
 1643   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1644   LoadStoreNode* ldst = n->as_LoadStore();
 1645   if (is_CAS(n->Opcode(), false)) {
 1646     assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
 1647   } else {
 1648     return ldst->trailing_membar() != nullptr;
 1649   }
 1650 
 1651   // so we can just return true here
 1652   return true;
 1653 }
 1654 
 1655 #define __ masm->
 1656 
 1657 // advance declarations for helper functions to convert register
 1658 // indices to register objects
 1659 
 1660 // the ad file has to provide implementations of certain methods
 1661 // expected by the generic code
 1662 //
 1663 // REQUIRED FUNCTIONALITY
 1664 
 1665 //=============================================================================
 1666 
 1667 // !!!!! Special hack to get all types of calls to specify the byte offset
 1668 //       from the start of the call to the point where the return address
 1669 //       will point.
 1670 
 1671 int MachCallStaticJavaNode::ret_addr_offset()
 1672 {
 1673   // call should be a simple bl
 1674   int off = 4;
 1675   return off;
 1676 }
 1677 
 1678 int MachCallDynamicJavaNode::ret_addr_offset()
 1679 {
 1680   return 16; // movz, movk, movk, bl
 1681 }
 1682 
 1683 int MachCallRuntimeNode::ret_addr_offset() {
 1684   // for generated stubs the call will be
 1685   //   bl(addr)
 1686   // or with far branches
 1687   //   bl(trampoline_stub)
 1688   // for real runtime callouts it will be six instructions
 1689   // see aarch64_enc_java_to_runtime
 1690   //   adr(rscratch2, retaddr)
 1691   //   str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
 1692   //   lea(rscratch1, RuntimeAddress(addr)
 1693   //   blr(rscratch1)
 1694   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1695   if (cb) {
 1696     return 1 * NativeInstruction::instruction_size;
 1697   } else {
 1698     return 6 * NativeInstruction::instruction_size;
 1699   }
 1700 }
 1701 
 1702 //=============================================================================
 1703 
 1704 #ifndef PRODUCT
 1705 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1706   st->print("BREAKPOINT");
 1707 }
 1708 #endif
 1709 
 1710 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1711   __ brk(0);
 1712 }
 1713 
 1714 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1715   return MachNode::size(ra_);
 1716 }
 1717 
 1718 //=============================================================================
 1719 
 1720 #ifndef PRODUCT
 1721   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1722     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1723   }
 1724 #endif
 1725 
 1726   void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
 1727     for (int i = 0; i < _count; i++) {
 1728       __ nop();
 1729     }
 1730   }
 1731 
 1732   uint MachNopNode::size(PhaseRegAlloc*) const {
 1733     return _count * NativeInstruction::instruction_size;
 1734   }
 1735 
 1736 //=============================================================================
 1737 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1738 
 1739 int ConstantTable::calculate_table_base_offset() const {
 1740   return 0;  // absolute addressing, no offset
 1741 }
 1742 
 1743 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1744 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1745   ShouldNotReachHere();
 1746 }
 1747 
 1748 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
 1749   // Empty encoding
 1750 }
 1751 
 1752 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1753   return 0;
 1754 }
 1755 
 1756 #ifndef PRODUCT
 1757 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1758   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1759 }
 1760 #endif
 1761 
 1762 #ifndef PRODUCT
 1763 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1764   Compile* C = ra_->C;
 1765 
 1766   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1767 
 1768   if (C->output()->need_stack_bang(framesize))
 1769     st->print("# stack bang size=%d\n\t", framesize);
 1770 
 1771   if (VM_Version::use_rop_protection()) {
 1772     st->print("ldr  zr, [lr]\n\t");
 1773     st->print("paciaz\n\t");
 1774   }
 1775   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1776     st->print("sub  sp, sp, #%d\n\t", framesize);
 1777     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1778     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1779   } else {
 1780     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1781     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1782     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1783     st->print("sub  sp, sp, rscratch1");
 1784   }
 1785   if (C->stub_function() == nullptr) {
 1786     st->print("\n\t");
 1787     st->print("ldr  rscratch1, [guard]\n\t");
 1788     st->print("dmb ishld\n\t");
 1789     st->print("ldr  rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
 1790     st->print("cmp  rscratch1, rscratch2\n\t");
 1791     st->print("b.eq skip");
 1792     st->print("\n\t");
 1793     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1794     st->print("b skip\n\t");
 1795     st->print("guard: int\n\t");
 1796     st->print("\n\t");
 1797     st->print("skip:\n\t");
 1798   }
 1799 }
 1800 #endif
 1801 
 1802 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1803   Compile* C = ra_->C;
 1804 
 1805   // n.b. frame size includes space for return pc and rfp
 1806   const int framesize = C->output()->frame_size_in_bytes();
 1807 
 1808   if (C->clinit_barrier_on_entry()) {
 1809     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1810 
 1811     Label L_skip_barrier;
 1812 
 1813     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1814     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1815     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1816     __ bind(L_skip_barrier);
 1817   }
 1818 
 1819   if (C->max_vector_size() > 0) {
 1820     __ reinitialize_ptrue();
 1821   }
 1822 
 1823   int bangsize = C->output()->bang_size_in_bytes();
 1824   if (C->output()->need_stack_bang(bangsize))
 1825     __ generate_stack_overflow_check(bangsize);
 1826 
 1827   __ build_frame(framesize);
 1828 
 1829   if (C->stub_function() == nullptr) {
 1830     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1831     // Dummy labels for just measuring the code size
 1832     Label dummy_slow_path;
 1833     Label dummy_continuation;
 1834     Label dummy_guard;
 1835     Label* slow_path = &dummy_slow_path;
 1836     Label* continuation = &dummy_continuation;
 1837     Label* guard = &dummy_guard;
 1838     if (!Compile::current()->output()->in_scratch_emit_size()) {
 1839       // Use real labels from actual stub when not emitting code for the purpose of measuring its size
 1840       C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
 1841       Compile::current()->output()->add_stub(stub);
 1842       slow_path = &stub->entry();
 1843       continuation = &stub->continuation();
 1844       guard = &stub->guard();
 1845     }
 1846     // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
 1847     bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
 1848   }
 1849 
 1850   if (VerifyStackAtCalls) {
 1851     Unimplemented();
 1852   }
 1853 
 1854   C->output()->set_frame_complete(__ offset());
 1855 
 1856   if (C->has_mach_constant_base_node()) {
 1857     // NOTE: We set the table base offset here because users might be
 1858     // emitted before MachConstantBaseNode.
 1859     ConstantTable& constant_table = C->output()->constant_table();
 1860     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1861   }
 1862 }
 1863 
 1864 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1865 {
 1866   return MachNode::size(ra_); // too many variables; just compute it
 1867                               // the hard way
 1868 }
 1869 
 1870 int MachPrologNode::reloc() const
 1871 {
 1872   return 0;
 1873 }
 1874 
 1875 //=============================================================================
 1876 
 1877 #ifndef PRODUCT
 1878 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1879   Compile* C = ra_->C;
 1880   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1881 
 1882   st->print("# pop frame %d\n\t",framesize);
 1883 
 1884   if (framesize == 0) {
 1885     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1886   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1887     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1888     st->print("add  sp, sp, #%d\n\t", framesize);
 1889   } else {
 1890     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1891     st->print("add  sp, sp, rscratch1\n\t");
 1892     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1893   }
 1894   if (VM_Version::use_rop_protection()) {
 1895     st->print("autiaz\n\t");
 1896     st->print("ldr  zr, [lr]\n\t");
 1897   }
 1898 
 1899   if (do_polling() && C->is_method_compilation()) {
 1900     st->print("# test polling word\n\t");
 1901     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1902     st->print("cmp  sp, rscratch1\n\t");
 1903     st->print("bhi #slow_path");
 1904   }
 1905 }
 1906 #endif
 1907 
 1908 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1909   Compile* C = ra_->C;
 1910   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1911 
 1912   __ remove_frame(framesize);
 1913 
 1914   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1915     __ reserved_stack_check();
 1916   }
 1917 
 1918   if (do_polling() && C->is_method_compilation()) {
 1919     Label dummy_label;
 1920     Label* code_stub = &dummy_label;
 1921     if (!C->output()->in_scratch_emit_size()) {
 1922       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
 1923       C->output()->add_stub(stub);
 1924       code_stub = &stub->entry();
 1925     }
 1926     __ relocate(relocInfo::poll_return_type);
 1927     __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
 1928   }
 1929 }
 1930 
 1931 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 1932   // Variable size. Determine dynamically.
 1933   return MachNode::size(ra_);
 1934 }
 1935 
 1936 int MachEpilogNode::reloc() const {
 1937   // Return number of relocatable values contained in this instruction.
 1938   return 1; // 1 for polling page.
 1939 }
 1940 
 1941 const Pipeline * MachEpilogNode::pipeline() const {
 1942   return MachNode::pipeline_class();
 1943 }
 1944 
 1945 //=============================================================================
 1946 
 1947 static enum RC rc_class(OptoReg::Name reg) {
 1948 
 1949   if (reg == OptoReg::Bad) {
 1950     return rc_bad;
 1951   }
 1952 
 1953   // we have 32 int registers * 2 halves
 1954   int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
 1955 
 1956   if (reg < slots_of_int_registers) {
 1957     return rc_int;
 1958   }
 1959 
 1960   // we have 32 float register * 8 halves
 1961   int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
 1962   if (reg < slots_of_int_registers + slots_of_float_registers) {
 1963     return rc_float;
 1964   }
 1965 
 1966   int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
 1967   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 1968     return rc_predicate;
 1969   }
 1970 
 1971   // Between predicate regs & stack is the flags.
 1972   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 1973 
 1974   return rc_stack;
 1975 }
 1976 
 1977 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 1978   Compile* C = ra_->C;
 1979 
 1980   // Get registers to move.
 1981   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 1982   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 1983   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 1984   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 1985 
 1986   enum RC src_hi_rc = rc_class(src_hi);
 1987   enum RC src_lo_rc = rc_class(src_lo);
 1988   enum RC dst_hi_rc = rc_class(dst_hi);
 1989   enum RC dst_lo_rc = rc_class(dst_lo);
 1990 
 1991   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 1992 
 1993   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 1994     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 1995            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 1996            "expected aligned-adjacent pairs");
 1997   }
 1998 
 1999   if (src_lo == dst_lo && src_hi == dst_hi) {
 2000     return 0;            // Self copy, no move.
 2001   }
 2002 
 2003   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 2004               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 2005   int src_offset = ra_->reg2offset(src_lo);
 2006   int dst_offset = ra_->reg2offset(dst_lo);
 2007 
 2008   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2009     uint ireg = ideal_reg();
 2010     if (ireg == Op_VecA && masm) {
 2011       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 2012       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2013         // stack->stack
 2014         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 2015                                                 sve_vector_reg_size_in_bytes);
 2016       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2017         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2018                             sve_vector_reg_size_in_bytes);
 2019       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2020         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2021                               sve_vector_reg_size_in_bytes);
 2022       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2023         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2024                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 2025                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 2026       } else {
 2027         ShouldNotReachHere();
 2028       }
 2029     } else if (masm) {
 2030       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 2031       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 2032       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2033         // stack->stack
 2034         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2035         if (ireg == Op_VecD) {
 2036           __ unspill(rscratch1, true, src_offset);
 2037           __ spill(rscratch1, true, dst_offset);
 2038         } else {
 2039           __ spill_copy128(src_offset, dst_offset);
 2040         }
 2041       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2042         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2043                ireg == Op_VecD ? __ T8B : __ T16B,
 2044                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2045       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2046         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2047                  ireg == Op_VecD ? __ D : __ Q,
 2048                  ra_->reg2offset(dst_lo));
 2049       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2050         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2051                    ireg == Op_VecD ? __ D : __ Q,
 2052                    ra_->reg2offset(src_lo));
 2053       } else {
 2054         ShouldNotReachHere();
 2055       }
 2056     }
 2057   } else if (masm) {
 2058     switch (src_lo_rc) {
 2059     case rc_int:
 2060       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2061         if (is64) {
 2062             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2063                    as_Register(Matcher::_regEncode[src_lo]));
 2064         } else {
 2065             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2066                     as_Register(Matcher::_regEncode[src_lo]));
 2067         }
 2068       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2069         if (is64) {
 2070             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2071                      as_Register(Matcher::_regEncode[src_lo]));
 2072         } else {
 2073             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2074                      as_Register(Matcher::_regEncode[src_lo]));
 2075         }
 2076       } else {                    // gpr --> stack spill
 2077         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2078         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2079       }
 2080       break;
 2081     case rc_float:
 2082       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2083         if (is64) {
 2084             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2085                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2086         } else {
 2087             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2088                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2089         }
 2090       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2091         if (is64) {
 2092             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2093                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2094         } else {
 2095             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2096                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2097         }
 2098       } else {                    // fpr --> stack spill
 2099         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2100         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2101                  is64 ? __ D : __ S, dst_offset);
 2102       }
 2103       break;
 2104     case rc_stack:
 2105       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2106         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2107       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2108         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2109                    is64 ? __ D : __ S, src_offset);
 2110       } else if (dst_lo_rc == rc_predicate) {
 2111         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2112                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2113       } else {                    // stack --> stack copy
 2114         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2115         if (ideal_reg() == Op_RegVectMask) {
 2116           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2117                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2118         } else {
 2119           __ unspill(rscratch1, is64, src_offset);
 2120           __ spill(rscratch1, is64, dst_offset);
 2121         }
 2122       }
 2123       break;
 2124     case rc_predicate:
 2125       if (dst_lo_rc == rc_predicate) {
 2126         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2127       } else if (dst_lo_rc == rc_stack) {
 2128         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2129                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2130       } else {
 2131         assert(false, "bad src and dst rc_class combination.");
 2132         ShouldNotReachHere();
 2133       }
 2134       break;
 2135     default:
 2136       assert(false, "bad rc_class for spill");
 2137       ShouldNotReachHere();
 2138     }
 2139   }
 2140 
 2141   if (st) {
 2142     st->print("spill ");
 2143     if (src_lo_rc == rc_stack) {
 2144       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2145     } else {
 2146       st->print("%s -> ", Matcher::regName[src_lo]);
 2147     }
 2148     if (dst_lo_rc == rc_stack) {
 2149       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2150     } else {
 2151       st->print("%s", Matcher::regName[dst_lo]);
 2152     }
 2153     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2154       int vsize = 0;
 2155       switch (ideal_reg()) {
 2156       case Op_VecD:
 2157         vsize = 64;
 2158         break;
 2159       case Op_VecX:
 2160         vsize = 128;
 2161         break;
 2162       case Op_VecA:
 2163         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2164         break;
 2165       default:
 2166         assert(false, "bad register type for spill");
 2167         ShouldNotReachHere();
 2168       }
 2169       st->print("\t# vector spill size = %d", vsize);
 2170     } else if (ideal_reg() == Op_RegVectMask) {
 2171       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2172       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2173       st->print("\t# predicate spill size = %d", vsize);
 2174     } else {
 2175       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2176     }
 2177   }
 2178 
 2179   return 0;
 2180 
 2181 }
 2182 
 2183 #ifndef PRODUCT
 2184 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2185   if (!ra_)
 2186     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2187   else
 2188     implementation(nullptr, ra_, false, st);
 2189 }
 2190 #endif
 2191 
 2192 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 2193   implementation(masm, ra_, false, nullptr);
 2194 }
 2195 
 2196 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2197   return MachNode::size(ra_);
 2198 }
 2199 
 2200 //=============================================================================
 2201 
 2202 #ifndef PRODUCT
 2203 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2204   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2205   int reg = ra_->get_reg_first(this);
 2206   st->print("add %s, rsp, #%d]\t# box lock",
 2207             Matcher::regName[reg], offset);
 2208 }
 2209 #endif
 2210 
 2211 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 2212   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2213   int reg    = ra_->get_encode(this);
 2214 
 2215   // This add will handle any 24-bit signed offset. 24 bits allows an
 2216   // 8 megabyte stack frame.
 2217   __ add(as_Register(reg), sp, offset);
 2218 }
 2219 
 2220 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2221   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2222   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2223 
 2224   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2225     return NativeInstruction::instruction_size;
 2226   } else {
 2227     return 2 * NativeInstruction::instruction_size;
 2228   }
 2229 }
 2230 
 2231 //=============================================================================
 2232 
 2233 #ifndef PRODUCT
 2234 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2235 {
 2236   st->print_cr("# MachUEPNode");
 2237   if (UseCompressedClassPointers) {
 2238     st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2239     st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
 2240     st->print_cr("\tcmpw rscratch1, r10");
 2241   } else {
 2242     st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2243     st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
 2244     st->print_cr("\tcmp rscratch1, r10");
 2245   }
 2246   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2247 }
 2248 #endif
 2249 
 2250 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 2251 {
 2252   __ ic_check(InteriorEntryAlignment);
 2253 }
 2254 
 2255 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2256 {
 2257   return MachNode::size(ra_);
 2258 }
 2259 
 2260 // REQUIRED EMIT CODE
 2261 
 2262 //=============================================================================
 2263 
 2264 // Emit exception handler code.
 2265 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
 2266 {
 2267   // mov rscratch1 #exception_blob_entry_point
 2268   // br rscratch1
 2269   // Note that the code buffer's insts_mark is always relative to insts.
 2270   // That's why we must use the macroassembler to generate a handler.
 2271   address base = __ start_a_stub(size_exception_handler());
 2272   if (base == nullptr) {
 2273     ciEnv::current()->record_failure("CodeCache is full");
 2274     return 0;  // CodeBuffer::expand failed
 2275   }
 2276   int offset = __ offset();
 2277   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2278   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2279   __ end_a_stub();
 2280   return offset;
 2281 }
 2282 
 2283 // Emit deopt handler code.
 2284 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
 2285 {
 2286   // Note that the code buffer's insts_mark is always relative to insts.
 2287   // That's why we must use the macroassembler to generate a handler.
 2288   address base = __ start_a_stub(size_deopt_handler());
 2289   if (base == nullptr) {
 2290     ciEnv::current()->record_failure("CodeCache is full");
 2291     return 0;  // CodeBuffer::expand failed
 2292   }
 2293   int offset = __ offset();
 2294 
 2295   __ adr(lr, __ pc());
 2296   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2297 
 2298   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2299   __ end_a_stub();
 2300   return offset;
 2301 }
 2302 
 2303 // REQUIRED MATCHER CODE
 2304 
 2305 //=============================================================================
 2306 
 2307 bool Matcher::match_rule_supported(int opcode) {
 2308   if (!has_match_rule(opcode))
 2309     return false;
 2310 
 2311   switch (opcode) {
 2312     case Op_OnSpinWait:
 2313       return VM_Version::supports_on_spin_wait();
 2314     case Op_CacheWB:
 2315     case Op_CacheWBPreSync:
 2316     case Op_CacheWBPostSync:
 2317       if (!VM_Version::supports_data_cache_line_flush()) {
 2318         return false;
 2319       }
 2320       break;
 2321     case Op_ExpandBits:
 2322     case Op_CompressBits:
 2323       if (!VM_Version::supports_svebitperm()) {
 2324         return false;
 2325       }
 2326       break;
 2327     case Op_FmaF:
 2328     case Op_FmaD:
 2329     case Op_FmaVF:
 2330     case Op_FmaVD:
 2331       if (!UseFMA) {
 2332         return false;
 2333       }
 2334       break;
 2335     case Op_FmaHF:
 2336       // UseFMA flag also needs to be checked along with FEAT_FP16
 2337       if (!UseFMA || !is_feat_fp16_supported()) {
 2338         return false;
 2339       }
 2340       break;
 2341     case Op_AddHF:
 2342     case Op_SubHF:
 2343     case Op_MulHF:
 2344     case Op_DivHF:
 2345     case Op_MinHF:
 2346     case Op_MaxHF:
 2347     case Op_SqrtHF:
 2348       // Half-precision floating point scalar operations require FEAT_FP16
 2349       // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
 2350       // features are supported.
 2351       if (!is_feat_fp16_supported()) {
 2352         return false;
 2353       }
 2354       break;
 2355   }
 2356 
 2357   return true; // Per default match rules are supported.
 2358 }
 2359 
 2360 const RegMask* Matcher::predicate_reg_mask(void) {
 2361   return &_PR_REG_mask;
 2362 }
 2363 
 2364 bool Matcher::supports_vector_calling_convention(void) {
 2365   return EnableVectorSupport;
 2366 }
 2367 
 2368 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2369   assert(EnableVectorSupport, "sanity");
 2370   int lo = V0_num;
 2371   int hi = V0_H_num;
 2372   if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
 2373     hi = V0_K_num;
 2374   }
 2375   return OptoRegPair(hi, lo);
 2376 }
 2377 
 2378 // Is this branch offset short enough that a short branch can be used?
 2379 //
 2380 // NOTE: If the platform does not provide any short branch variants, then
 2381 //       this method should return false for offset 0.
 2382 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2383   // The passed offset is relative to address of the branch.
 2384 
 2385   return (-32768 <= offset && offset < 32768);
 2386 }
 2387 
 2388 // Vector width in bytes.
 2389 int Matcher::vector_width_in_bytes(BasicType bt) {
 2390   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2391   int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
 2392   // Minimum 2 values in vector
 2393   if (size < 2*type2aelembytes(bt)) size = 0;
 2394   // But never < 4
 2395   if (size < 4) size = 0;
 2396   return size;
 2397 }
 2398 
 2399 // Limits on vector size (number of elements) loaded into vector.
 2400 int Matcher::max_vector_size(const BasicType bt) {
 2401   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2402 }
 2403 
 2404 int Matcher::min_vector_size(const BasicType bt) {
 2405   // Usually, the shortest vector length supported by AArch64 ISA and
 2406   // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
 2407   // vectors in a few special cases.
 2408   int size;
 2409   switch(bt) {
 2410     case T_BOOLEAN:
 2411       // Load/store a vector mask with only 2 elements for vector types
 2412       // such as "2I/2F/2L/2D".
 2413       size = 2;
 2414       break;
 2415     case T_BYTE:
 2416       // Generate a "4B" vector, to support vector cast between "8B/16B"
 2417       // and "4S/4I/4L/4F/4D".
 2418       size = 4;
 2419       break;
 2420     case T_SHORT:
 2421       // Generate a "2S" vector, to support vector cast between "4S/8S"
 2422       // and "2I/2L/2F/2D".
 2423       size = 2;
 2424       break;
 2425     default:
 2426       // Limit the min vector length to 64-bit.
 2427       size = 8 / type2aelembytes(bt);
 2428       // The number of elements in a vector should be at least 2.
 2429       size = MAX2(size, 2);
 2430   }
 2431 
 2432   int max_size = max_vector_size(bt);
 2433   return MIN2(size, max_size);
 2434 }
 2435 
 2436 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
 2437   return Matcher::max_vector_size(bt);
 2438 }
 2439 
 2440 // Actual max scalable vector register length.
 2441 int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2442   return Matcher::max_vector_size(bt);
 2443 }
 2444 
 2445 // Vector ideal reg.
 2446 uint Matcher::vector_ideal_reg(int len) {
 2447   if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
 2448     return Op_VecA;
 2449   }
 2450   switch(len) {
 2451     // For 16-bit/32-bit mask vector, reuse VecD.
 2452     case  2:
 2453     case  4:
 2454     case  8: return Op_VecD;
 2455     case 16: return Op_VecX;
 2456   }
 2457   ShouldNotReachHere();
 2458   return 0;
 2459 }
 2460 
 2461 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
 2462   assert(Matcher::is_generic_vector(generic_opnd), "not generic");
 2463   switch (ideal_reg) {
 2464     case Op_VecA: return new vecAOper();
 2465     case Op_VecD: return new vecDOper();
 2466     case Op_VecX: return new vecXOper();
 2467   }
 2468   ShouldNotReachHere();
 2469   return nullptr;
 2470 }
 2471 
 2472 bool Matcher::is_reg2reg_move(MachNode* m) {
 2473   return false;
 2474 }
 2475 
 2476 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2477   return opnd->opcode() == VREG;
 2478 }
 2479 
 2480 // Return whether or not this register is ever used as an argument.
 2481 // This function is used on startup to build the trampoline stubs in
 2482 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2483 // call in the trampoline, and arguments in those registers not be
 2484 // available to the callee.
 2485 bool Matcher::can_be_java_arg(int reg)
 2486 {
 2487   return
 2488     reg ==  R0_num || reg == R0_H_num ||
 2489     reg ==  R1_num || reg == R1_H_num ||
 2490     reg ==  R2_num || reg == R2_H_num ||
 2491     reg ==  R3_num || reg == R3_H_num ||
 2492     reg ==  R4_num || reg == R4_H_num ||
 2493     reg ==  R5_num || reg == R5_H_num ||
 2494     reg ==  R6_num || reg == R6_H_num ||
 2495     reg ==  R7_num || reg == R7_H_num ||
 2496     reg ==  V0_num || reg == V0_H_num ||
 2497     reg ==  V1_num || reg == V1_H_num ||
 2498     reg ==  V2_num || reg == V2_H_num ||
 2499     reg ==  V3_num || reg == V3_H_num ||
 2500     reg ==  V4_num || reg == V4_H_num ||
 2501     reg ==  V5_num || reg == V5_H_num ||
 2502     reg ==  V6_num || reg == V6_H_num ||
 2503     reg ==  V7_num || reg == V7_H_num;
 2504 }
 2505 
 2506 bool Matcher::is_spillable_arg(int reg)
 2507 {
 2508   return can_be_java_arg(reg);
 2509 }
 2510 
 2511 uint Matcher::int_pressure_limit()
 2512 {
 2513   // JDK-8183543: When taking the number of available registers as int
 2514   // register pressure threshold, the jtreg test:
 2515   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2516   // failed due to C2 compilation failure with
 2517   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2518   //
 2519   // A derived pointer is live at CallNode and then is flagged by RA
 2520   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2521   // derived pointers and lastly fail to spill after reaching maximum
 2522   // number of iterations. Lowering the default pressure threshold to
 2523   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2524   // a high register pressure area of the code so that split_DEF can
 2525   // generate DefinitionSpillCopy for the derived pointer.
 2526   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2527   if (!PreserveFramePointer) {
 2528     // When PreserveFramePointer is off, frame pointer is allocatable,
 2529     // but different from other SOC registers, it is excluded from
 2530     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2531     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2532     // See check_pressure_at_fatproj().
 2533     default_int_pressure_threshold--;
 2534   }
 2535   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2536 }
 2537 
 2538 uint Matcher::float_pressure_limit()
 2539 {
 2540   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2541   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2542 }
 2543 
 2544 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2545   return false;
 2546 }
 2547 
 2548 RegMask Matcher::divI_proj_mask() {
 2549   ShouldNotReachHere();
 2550   return RegMask();
 2551 }
 2552 
 2553 // Register for MODI projection of divmodI.
 2554 RegMask Matcher::modI_proj_mask() {
 2555   ShouldNotReachHere();
 2556   return RegMask();
 2557 }
 2558 
 2559 // Register for DIVL projection of divmodL.
 2560 RegMask Matcher::divL_proj_mask() {
 2561   ShouldNotReachHere();
 2562   return RegMask();
 2563 }
 2564 
 2565 // Register for MODL projection of divmodL.
 2566 RegMask Matcher::modL_proj_mask() {
 2567   ShouldNotReachHere();
 2568   return RegMask();
 2569 }
 2570 
 2571 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2572   return FP_REG_mask();
 2573 }
 2574 
 2575 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2576   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2577     Node* u = addp->fast_out(i);
 2578     if (u->is_LoadStore()) {
 2579       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2580       // instructions) only take register indirect as an operand, so
 2581       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2582       // must fail.
 2583       return false;
 2584     }
 2585     if (u->is_Mem()) {
 2586       int opsize = u->as_Mem()->memory_size();
 2587       assert(opsize > 0, "unexpected memory operand size");
 2588       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2589         return false;
 2590       }
 2591     }
 2592   }
 2593   return true;
 2594 }
 2595 
 2596 // Convert BootTest condition to Assembler condition.
 2597 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 2598 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
 2599   Assembler::Condition result;
 2600   switch(cond) {
 2601     case BoolTest::eq:
 2602       result = Assembler::EQ; break;
 2603     case BoolTest::ne:
 2604       result = Assembler::NE; break;
 2605     case BoolTest::le:
 2606       result = Assembler::LE; break;
 2607     case BoolTest::ge:
 2608       result = Assembler::GE; break;
 2609     case BoolTest::lt:
 2610       result = Assembler::LT; break;
 2611     case BoolTest::gt:
 2612       result = Assembler::GT; break;
 2613     case BoolTest::ule:
 2614       result = Assembler::LS; break;
 2615     case BoolTest::uge:
 2616       result = Assembler::HS; break;
 2617     case BoolTest::ult:
 2618       result = Assembler::LO; break;
 2619     case BoolTest::ugt:
 2620       result = Assembler::HI; break;
 2621     case BoolTest::overflow:
 2622       result = Assembler::VS; break;
 2623     case BoolTest::no_overflow:
 2624       result = Assembler::VC; break;
 2625     default:
 2626       ShouldNotReachHere();
 2627       return Assembler::Condition(-1);
 2628   }
 2629 
 2630   // Check conversion
 2631   if (cond & BoolTest::unsigned_compare) {
 2632     assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
 2633   } else {
 2634     assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
 2635   }
 2636 
 2637   return result;
 2638 }
 2639 
 2640 // Binary src (Replicate con)
 2641 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
 2642   if (n == nullptr || m == nullptr) {
 2643     return false;
 2644   }
 2645 
 2646   if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
 2647     return false;
 2648   }
 2649 
 2650   Node* imm_node = m->in(1);
 2651   if (!imm_node->is_Con()) {
 2652     return false;
 2653   }
 2654 
 2655   const Type* t = imm_node->bottom_type();
 2656   if (!(t->isa_int() || t->isa_long())) {
 2657     return false;
 2658   }
 2659 
 2660   switch (n->Opcode()) {
 2661   case Op_AndV:
 2662   case Op_OrV:
 2663   case Op_XorV: {
 2664     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
 2665     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2666     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2667   }
 2668   case Op_AddVB:
 2669     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2670   case Op_AddVS:
 2671   case Op_AddVI:
 2672     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2673   case Op_AddVL:
 2674     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2675   default:
 2676     return false;
 2677   }
 2678 }
 2679 
 2680 // (XorV src (Replicate m1))
 2681 // (XorVMask src (MaskAll m1))
 2682 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
 2683   if (n != nullptr && m != nullptr) {
 2684     return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
 2685            VectorNode::is_all_ones_vector(m);
 2686   }
 2687   return false;
 2688 }
 2689 
 2690 // Should the matcher clone input 'm' of node 'n'?
 2691 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2692   if (is_vshift_con_pattern(n, m) ||
 2693       is_vector_bitwise_not_pattern(n, m) ||
 2694       is_valid_sve_arith_imm_pattern(n, m) ||
 2695       is_encode_and_store_pattern(n, m)) {
 2696     mstack.push(m, Visit);
 2697     return true;
 2698   }
 2699   return false;
 2700 }
 2701 
 2702 // Should the Matcher clone shifts on addressing modes, expecting them
 2703 // to be subsumed into complex addressing expressions or compute them
 2704 // into registers?
 2705 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2706 
 2707   // Loads and stores with indirect memory input (e.g., volatile loads and
 2708   // stores) do not subsume the input into complex addressing expressions. If
 2709   // the addressing expression is input to at least one such load or store, do
 2710   // not clone the addressing expression. Query needs_acquiring_load and
 2711   // needs_releasing_store as a proxy for indirect memory input, as it is not
 2712   // possible to directly query for indirect memory input at this stage.
 2713   for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
 2714     Node* n = m->fast_out(i);
 2715     if (n->is_Load() && needs_acquiring_load(n)) {
 2716       return false;
 2717     }
 2718     if (n->is_Store() && needs_releasing_store(n)) {
 2719       return false;
 2720     }
 2721   }
 2722 
 2723   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2724     return true;
 2725   }
 2726 
 2727   Node *off = m->in(AddPNode::Offset);
 2728   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2729       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2730       // Are there other uses besides address expressions?
 2731       !is_visited(off)) {
 2732     address_visited.set(off->_idx); // Flag as address_visited
 2733     mstack.push(off->in(2), Visit);
 2734     Node *conv = off->in(1);
 2735     if (conv->Opcode() == Op_ConvI2L &&
 2736         // Are there other uses besides address expressions?
 2737         !is_visited(conv)) {
 2738       address_visited.set(conv->_idx); // Flag as address_visited
 2739       mstack.push(conv->in(1), Pre_Visit);
 2740     } else {
 2741       mstack.push(conv, Pre_Visit);
 2742     }
 2743     address_visited.test_set(m->_idx); // Flag as address_visited
 2744     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2745     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2746     return true;
 2747   } else if (off->Opcode() == Op_ConvI2L &&
 2748              // Are there other uses besides address expressions?
 2749              !is_visited(off)) {
 2750     address_visited.test_set(m->_idx); // Flag as address_visited
 2751     address_visited.set(off->_idx); // Flag as address_visited
 2752     mstack.push(off->in(1), Pre_Visit);
 2753     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2754     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2755     return true;
 2756   }
 2757   return false;
 2758 }
 2759 
 2760 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2761   {                                                                     \
 2762     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2763     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2764     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2765     __ INSN(REG, as_Register(BASE));                                    \
 2766   }
 2767 
 2768 
 2769 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2770   {
 2771     Address::extend scale;
 2772 
 2773     // Hooboy, this is fugly.  We need a way to communicate to the
 2774     // encoder that the index needs to be sign extended, so we have to
 2775     // enumerate all the cases.
 2776     switch (opcode) {
 2777     case INDINDEXSCALEDI2L:
 2778     case INDINDEXSCALEDI2LN:
 2779     case INDINDEXI2L:
 2780     case INDINDEXI2LN:
 2781       scale = Address::sxtw(size);
 2782       break;
 2783     default:
 2784       scale = Address::lsl(size);
 2785     }
 2786 
 2787     if (index == -1) {
 2788       return Address(base, disp);
 2789     } else {
 2790       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2791       return Address(base, as_Register(index), scale);
 2792     }
 2793   }
 2794 
 2795 
 2796 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2797 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2798 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2799 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2800                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2801 
 2802   // Used for all non-volatile memory accesses.  The use of
 2803   // $mem->opcode() to discover whether this pattern uses sign-extended
 2804   // offsets is something of a kludge.
 2805   static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
 2806                         Register reg, int opcode,
 2807                         Register base, int index, int scale, int disp,
 2808                         int size_in_memory)
 2809   {
 2810     Address addr = mem2address(opcode, base, index, scale, disp);
 2811     if (addr.getMode() == Address::base_plus_offset) {
 2812       /* Fix up any out-of-range offsets. */
 2813       assert_different_registers(rscratch1, base);
 2814       assert_different_registers(rscratch1, reg);
 2815       addr = __ legitimize_address(addr, size_in_memory, rscratch1);
 2816     }
 2817     (masm->*insn)(reg, addr);
 2818   }
 2819 
 2820   static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
 2821                         FloatRegister reg, int opcode,
 2822                         Register base, int index, int size, int disp,
 2823                         int size_in_memory)
 2824   {
 2825     Address::extend scale;
 2826 
 2827     switch (opcode) {
 2828     case INDINDEXSCALEDI2L:
 2829     case INDINDEXSCALEDI2LN:
 2830       scale = Address::sxtw(size);
 2831       break;
 2832     default:
 2833       scale = Address::lsl(size);
 2834     }
 2835 
 2836     if (index == -1) {
 2837       // Fix up any out-of-range offsets.
 2838       assert_different_registers(rscratch1, base);
 2839       Address addr = Address(base, disp);
 2840       addr = __ legitimize_address(addr, size_in_memory, rscratch1);
 2841       (masm->*insn)(reg, addr);
 2842     } else {
 2843       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2844       (masm->*insn)(reg, Address(base, as_Register(index), scale));
 2845     }
 2846   }
 2847 
 2848   static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
 2849                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2850                         int opcode, Register base, int index, int size, int disp)
 2851   {
 2852     if (index == -1) {
 2853       (masm->*insn)(reg, T, Address(base, disp));
 2854     } else {
 2855       assert(disp == 0, "unsupported address mode");
 2856       (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2857     }
 2858   }
 2859 
 2860 %}
 2861 
 2862 
 2863 
 2864 //----------ENCODING BLOCK-----------------------------------------------------
 2865 // This block specifies the encoding classes used by the compiler to
 2866 // output byte streams.  Encoding classes are parameterized macros
 2867 // used by Machine Instruction Nodes in order to generate the bit
 2868 // encoding of the instruction.  Operands specify their base encoding
 2869 // interface with the interface keyword.  There are currently
 2870 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2871 // COND_INTER.  REG_INTER causes an operand to generate a function
 2872 // which returns its register number when queried.  CONST_INTER causes
 2873 // an operand to generate a function which returns the value of the
 2874 // constant when queried.  MEMORY_INTER causes an operand to generate
 2875 // four functions which return the Base Register, the Index Register,
 2876 // the Scale Value, and the Offset Value of the operand when queried.
 2877 // COND_INTER causes an operand to generate six functions which return
 2878 // the encoding code (ie - encoding bits for the instruction)
 2879 // associated with each basic boolean condition for a conditional
 2880 // instruction.
 2881 //
 2882 // Instructions specify two basic values for encoding.  Again, a
 2883 // function is available to check if the constant displacement is an
 2884 // oop. They use the ins_encode keyword to specify their encoding
 2885 // classes (which must be a sequence of enc_class names, and their
 2886 // parameters, specified in the encoding block), and they use the
 2887 // opcode keyword to specify, in order, their primary, secondary, and
 2888 // tertiary opcode.  Only the opcode sections which a particular
 2889 // instruction needs for encoding need to be specified.
 2890 encode %{
 2891   // Build emit functions for each basic byte or larger field in the
 2892   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2893   // from C++ code in the enc_class source block.  Emit functions will
 2894   // live in the main source block for now.  In future, we can
 2895   // generalize this by adding a syntax that specifies the sizes of
 2896   // fields in an order, so that the adlc can build the emit functions
 2897   // automagically
 2898 
 2899   // catch all for unimplemented encodings
 2900   enc_class enc_unimplemented %{
 2901     __ unimplemented("C2 catch all");
 2902   %}
 2903 
 2904   // BEGIN Non-volatile memory access
 2905 
 2906   // This encoding class is generated automatically from ad_encode.m4.
 2907   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2908   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2909     Register dst_reg = as_Register($dst$$reg);
 2910     loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2911                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2912   %}
 2913 
 2914   // This encoding class is generated automatically from ad_encode.m4.
 2915   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2916   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2917     Register dst_reg = as_Register($dst$$reg);
 2918     loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2919                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2920   %}
 2921 
 2922   // This encoding class is generated automatically from ad_encode.m4.
 2923   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2924   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2925     Register dst_reg = as_Register($dst$$reg);
 2926     loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2927                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2928   %}
 2929 
 2930   // This encoding class is generated automatically from ad_encode.m4.
 2931   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2932   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2933     Register dst_reg = as_Register($dst$$reg);
 2934     loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2935                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2936   %}
 2937 
 2938   // This encoding class is generated automatically from ad_encode.m4.
 2939   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2940   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2941     Register dst_reg = as_Register($dst$$reg);
 2942     loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2943                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2944   %}
 2945 
 2946   // This encoding class is generated automatically from ad_encode.m4.
 2947   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2948   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2949     Register dst_reg = as_Register($dst$$reg);
 2950     loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2951                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2952   %}
 2953 
 2954   // This encoding class is generated automatically from ad_encode.m4.
 2955   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2956   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2957     Register dst_reg = as_Register($dst$$reg);
 2958     loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2959                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2960   %}
 2961 
 2962   // This encoding class is generated automatically from ad_encode.m4.
 2963   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2964   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2965     Register dst_reg = as_Register($dst$$reg);
 2966     loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2967                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2968   %}
 2969 
 2970   // This encoding class is generated automatically from ad_encode.m4.
 2971   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2972   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2973     Register dst_reg = as_Register($dst$$reg);
 2974     loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2975                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2976   %}
 2977 
 2978   // This encoding class is generated automatically from ad_encode.m4.
 2979   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2980   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2981     Register dst_reg = as_Register($dst$$reg);
 2982     loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2983                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2984   %}
 2985 
 2986   // This encoding class is generated automatically from ad_encode.m4.
 2987   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2988   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2989     Register dst_reg = as_Register($dst$$reg);
 2990     loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2991                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2992   %}
 2993 
 2994   // This encoding class is generated automatically from ad_encode.m4.
 2995   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2996   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2997     Register dst_reg = as_Register($dst$$reg);
 2998     loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2999                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3000   %}
 3001 
 3002   // This encoding class is generated automatically from ad_encode.m4.
 3003   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3004   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 3005     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3006     loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 3007                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3008   %}
 3009 
 3010   // This encoding class is generated automatically from ad_encode.m4.
 3011   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3012   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 3013     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3014     loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 3015                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3016   %}
 3017 
 3018   // This encoding class is generated automatically from ad_encode.m4.
 3019   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3020   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 3021     Register src_reg = as_Register($src$$reg);
 3022     loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
 3023                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3024   %}
 3025 
 3026   // This encoding class is generated automatically from ad_encode.m4.
 3027   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3028   enc_class aarch64_enc_strb0(memory1 mem) %{
 3029     loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3030                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3031   %}
 3032 
 3033   // This encoding class is generated automatically from ad_encode.m4.
 3034   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3035   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 3036     Register src_reg = as_Register($src$$reg);
 3037     loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
 3038                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3039   %}
 3040 
 3041   // This encoding class is generated automatically from ad_encode.m4.
 3042   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3043   enc_class aarch64_enc_strh0(memory2 mem) %{
 3044     loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
 3045                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3046   %}
 3047 
 3048   // This encoding class is generated automatically from ad_encode.m4.
 3049   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3050   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 3051     Register src_reg = as_Register($src$$reg);
 3052     loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
 3053                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3054   %}
 3055 
 3056   // This encoding class is generated automatically from ad_encode.m4.
 3057   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3058   enc_class aarch64_enc_strw0(memory4 mem) %{
 3059     loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
 3060                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3061   %}
 3062 
 3063   // This encoding class is generated automatically from ad_encode.m4.
 3064   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3065   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 3066     Register src_reg = as_Register($src$$reg);
 3067     // we sometimes get asked to store the stack pointer into the
 3068     // current thread -- we cannot do that directly on AArch64
 3069     if (src_reg == r31_sp) {
 3070       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3071       __ mov(rscratch2, sp);
 3072       src_reg = rscratch2;
 3073     }
 3074     loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
 3075                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3076   %}
 3077 
 3078   // This encoding class is generated automatically from ad_encode.m4.
 3079   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3080   enc_class aarch64_enc_str0(memory8 mem) %{
 3081     loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
 3082                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3083   %}
 3084 
 3085   // This encoding class is generated automatically from ad_encode.m4.
 3086   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3087   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3088     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3089     loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
 3090                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3091   %}
 3092 
 3093   // This encoding class is generated automatically from ad_encode.m4.
 3094   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3095   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3096     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3097     loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
 3098                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3099   %}
 3100 
 3101   // This encoding class is generated automatically from ad_encode.m4.
 3102   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3103   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3104       __ membar(Assembler::StoreStore);
 3105       loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3106                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3107   %}
 3108 
 3109   // END Non-volatile memory access
 3110 
 3111   // Vector loads and stores
 3112   enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
 3113     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3114     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3115        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3116   %}
 3117 
 3118   enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
 3119     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3120     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3121        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3122   %}
 3123 
 3124   enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
 3125     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3126     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3127        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3128   %}
 3129 
 3130   enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
 3131     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3132     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3133        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3134   %}
 3135 
 3136   enc_class aarch64_enc_strvH(vReg src, memory mem) %{
 3137     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3138     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
 3139        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3140   %}
 3141 
 3142   enc_class aarch64_enc_strvS(vReg src, memory mem) %{
 3143     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3144     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
 3145        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3146   %}
 3147 
 3148   enc_class aarch64_enc_strvD(vReg src, memory mem) %{
 3149     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3150     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
 3151        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3152   %}
 3153 
 3154   enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
 3155     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3156     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3157        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3158   %}
 3159 
 3160   // volatile loads and stores
 3161 
 3162   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3163     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3164                  rscratch1, stlrb);
 3165   %}
 3166 
 3167   enc_class aarch64_enc_stlrb0(memory mem) %{
 3168     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3169                  rscratch1, stlrb);
 3170   %}
 3171 
 3172   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3173     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3174                  rscratch1, stlrh);
 3175   %}
 3176 
 3177   enc_class aarch64_enc_stlrh0(memory mem) %{
 3178     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3179                  rscratch1, stlrh);
 3180   %}
 3181 
 3182   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3183     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3184                  rscratch1, stlrw);
 3185   %}
 3186 
 3187   enc_class aarch64_enc_stlrw0(memory mem) %{
 3188     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3189                  rscratch1, stlrw);
 3190   %}
 3191 
 3192   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3193     Register dst_reg = as_Register($dst$$reg);
 3194     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3195              rscratch1, ldarb);
 3196     __ sxtbw(dst_reg, dst_reg);
 3197   %}
 3198 
 3199   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3200     Register dst_reg = as_Register($dst$$reg);
 3201     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3202              rscratch1, ldarb);
 3203     __ sxtb(dst_reg, dst_reg);
 3204   %}
 3205 
 3206   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3207     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3208              rscratch1, ldarb);
 3209   %}
 3210 
 3211   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3212     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3213              rscratch1, ldarb);
 3214   %}
 3215 
 3216   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3217     Register dst_reg = as_Register($dst$$reg);
 3218     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3219              rscratch1, ldarh);
 3220     __ sxthw(dst_reg, dst_reg);
 3221   %}
 3222 
 3223   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3224     Register dst_reg = as_Register($dst$$reg);
 3225     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3226              rscratch1, ldarh);
 3227     __ sxth(dst_reg, dst_reg);
 3228   %}
 3229 
 3230   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3231     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3232              rscratch1, ldarh);
 3233   %}
 3234 
 3235   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3236     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3237              rscratch1, ldarh);
 3238   %}
 3239 
 3240   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3241     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3242              rscratch1, ldarw);
 3243   %}
 3244 
 3245   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3246     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3247              rscratch1, ldarw);
 3248   %}
 3249 
 3250   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3251     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3252              rscratch1, ldar);
 3253   %}
 3254 
 3255   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3256     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3257              rscratch1, ldarw);
 3258     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3259   %}
 3260 
 3261   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3262     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3263              rscratch1, ldar);
 3264     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3265   %}
 3266 
 3267   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3268     Register src_reg = as_Register($src$$reg);
 3269     // we sometimes get asked to store the stack pointer into the
 3270     // current thread -- we cannot do that directly on AArch64
 3271     if (src_reg == r31_sp) {
 3272       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3273       __ mov(rscratch2, sp);
 3274       src_reg = rscratch2;
 3275     }
 3276     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3277                  rscratch1, stlr);
 3278   %}
 3279 
 3280   enc_class aarch64_enc_stlr0(memory mem) %{
 3281     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3282                  rscratch1, stlr);
 3283   %}
 3284 
 3285   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3286     {
 3287       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3288       __ fmovs(rscratch2, src_reg);
 3289     }
 3290     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3291                  rscratch1, stlrw);
 3292   %}
 3293 
 3294   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3295     {
 3296       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3297       __ fmovd(rscratch2, src_reg);
 3298     }
 3299     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3300                  rscratch1, stlr);
 3301   %}
 3302 
 3303   // synchronized read/update encodings
 3304 
 3305   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3306     Register dst_reg = as_Register($dst$$reg);
 3307     Register base = as_Register($mem$$base);
 3308     int index = $mem$$index;
 3309     int scale = $mem$$scale;
 3310     int disp = $mem$$disp;
 3311     if (index == -1) {
 3312        if (disp != 0) {
 3313         __ lea(rscratch1, Address(base, disp));
 3314         __ ldaxr(dst_reg, rscratch1);
 3315       } else {
 3316         // TODO
 3317         // should we ever get anything other than this case?
 3318         __ ldaxr(dst_reg, base);
 3319       }
 3320     } else {
 3321       Register index_reg = as_Register(index);
 3322       if (disp == 0) {
 3323         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3324         __ ldaxr(dst_reg, rscratch1);
 3325       } else {
 3326         __ lea(rscratch1, Address(base, disp));
 3327         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3328         __ ldaxr(dst_reg, rscratch1);
 3329       }
 3330     }
 3331   %}
 3332 
 3333   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3334     Register src_reg = as_Register($src$$reg);
 3335     Register base = as_Register($mem$$base);
 3336     int index = $mem$$index;
 3337     int scale = $mem$$scale;
 3338     int disp = $mem$$disp;
 3339     if (index == -1) {
 3340        if (disp != 0) {
 3341         __ lea(rscratch2, Address(base, disp));
 3342         __ stlxr(rscratch1, src_reg, rscratch2);
 3343       } else {
 3344         // TODO
 3345         // should we ever get anything other than this case?
 3346         __ stlxr(rscratch1, src_reg, base);
 3347       }
 3348     } else {
 3349       Register index_reg = as_Register(index);
 3350       if (disp == 0) {
 3351         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3352         __ stlxr(rscratch1, src_reg, rscratch2);
 3353       } else {
 3354         __ lea(rscratch2, Address(base, disp));
 3355         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3356         __ stlxr(rscratch1, src_reg, rscratch2);
 3357       }
 3358     }
 3359     __ cmpw(rscratch1, zr);
 3360   %}
 3361 
 3362   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3363     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3364     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3365                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3366                /*weak*/ false, noreg);
 3367   %}
 3368 
 3369   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3370     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3371     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3372                Assembler::word, /*acquire*/ false, /*release*/ true,
 3373                /*weak*/ false, noreg);
 3374   %}
 3375 
 3376   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3377     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3378     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3379                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3380                /*weak*/ false, noreg);
 3381   %}
 3382 
 3383   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3384     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3385     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3386                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3387                /*weak*/ false, noreg);
 3388   %}
 3389 
 3390 
 3391   // The only difference between aarch64_enc_cmpxchg and
 3392   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3393   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3394   // lock.
 3395   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3396     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3397     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3398                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3399                /*weak*/ false, noreg);
 3400   %}
 3401 
 3402   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3403     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3404     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3405                Assembler::word, /*acquire*/ true, /*release*/ true,
 3406                /*weak*/ false, noreg);
 3407   %}
 3408 
 3409   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3410     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3411     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3412                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3413                /*weak*/ false, noreg);
 3414   %}
 3415 
 3416   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3417     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3418     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3419                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3420                /*weak*/ false, noreg);
 3421   %}
 3422 
 3423   // auxiliary used for CompareAndSwapX to set result register
 3424   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3425     Register res_reg = as_Register($res$$reg);
 3426     __ cset(res_reg, Assembler::EQ);
 3427   %}
 3428 
 3429   // prefetch encodings
 3430 
 3431   enc_class aarch64_enc_prefetchw(memory mem) %{
 3432     Register base = as_Register($mem$$base);
 3433     int index = $mem$$index;
 3434     int scale = $mem$$scale;
 3435     int disp = $mem$$disp;
 3436     if (index == -1) {
 3437       // Fix up any out-of-range offsets.
 3438       assert_different_registers(rscratch1, base);
 3439       Address addr = Address(base, disp);
 3440       addr = __ legitimize_address(addr, 8, rscratch1);
 3441       __ prfm(addr, PSTL1KEEP);
 3442     } else {
 3443       Register index_reg = as_Register(index);
 3444       if (disp == 0) {
 3445         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3446       } else {
 3447         __ lea(rscratch1, Address(base, disp));
 3448 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3449       }
 3450     }
 3451   %}
 3452 
 3453   // mov encodings
 3454 
 3455   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3456     uint32_t con = (uint32_t)$src$$constant;
 3457     Register dst_reg = as_Register($dst$$reg);
 3458     if (con == 0) {
 3459       __ movw(dst_reg, zr);
 3460     } else {
 3461       __ movw(dst_reg, con);
 3462     }
 3463   %}
 3464 
 3465   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3466     Register dst_reg = as_Register($dst$$reg);
 3467     uint64_t con = (uint64_t)$src$$constant;
 3468     if (con == 0) {
 3469       __ mov(dst_reg, zr);
 3470     } else {
 3471       __ mov(dst_reg, con);
 3472     }
 3473   %}
 3474 
 3475   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3476     Register dst_reg = as_Register($dst$$reg);
 3477     address con = (address)$src$$constant;
 3478     if (con == nullptr || con == (address)1) {
 3479       ShouldNotReachHere();
 3480     } else {
 3481       relocInfo::relocType rtype = $src->constant_reloc();
 3482       if (rtype == relocInfo::oop_type) {
 3483         __ movoop(dst_reg, (jobject)con);
 3484       } else if (rtype == relocInfo::metadata_type) {
 3485         __ mov_metadata(dst_reg, (Metadata*)con);
 3486       } else {
 3487         assert(rtype == relocInfo::none, "unexpected reloc type");
 3488         if (! __ is_valid_AArch64_address(con) ||
 3489             con < (address)(uintptr_t)os::vm_page_size()) {
 3490           __ mov(dst_reg, con);
 3491         } else {
 3492           uint64_t offset;
 3493           __ adrp(dst_reg, con, offset);
 3494           __ add(dst_reg, dst_reg, offset);
 3495         }
 3496       }
 3497     }
 3498   %}
 3499 
 3500   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3501     Register dst_reg = as_Register($dst$$reg);
 3502     __ mov(dst_reg, zr);
 3503   %}
 3504 
 3505   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3506     Register dst_reg = as_Register($dst$$reg);
 3507     __ mov(dst_reg, (uint64_t)1);
 3508   %}
 3509 
 3510   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3511     Register dst_reg = as_Register($dst$$reg);
 3512     address con = (address)$src$$constant;
 3513     if (con == nullptr) {
 3514       ShouldNotReachHere();
 3515     } else {
 3516       relocInfo::relocType rtype = $src->constant_reloc();
 3517       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3518       __ set_narrow_oop(dst_reg, (jobject)con);
 3519     }
 3520   %}
 3521 
 3522   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3523     Register dst_reg = as_Register($dst$$reg);
 3524     __ mov(dst_reg, zr);
 3525   %}
 3526 
 3527   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3528     Register dst_reg = as_Register($dst$$reg);
 3529     address con = (address)$src$$constant;
 3530     if (con == nullptr) {
 3531       ShouldNotReachHere();
 3532     } else {
 3533       relocInfo::relocType rtype = $src->constant_reloc();
 3534       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3535       __ set_narrow_klass(dst_reg, (Klass *)con);
 3536     }
 3537   %}
 3538 
 3539   // arithmetic encodings
 3540 
 3541   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3542     Register dst_reg = as_Register($dst$$reg);
 3543     Register src_reg = as_Register($src1$$reg);
 3544     int32_t con = (int32_t)$src2$$constant;
 3545     // add has primary == 0, subtract has primary == 1
 3546     if ($primary) { con = -con; }
 3547     if (con < 0) {
 3548       __ subw(dst_reg, src_reg, -con);
 3549     } else {
 3550       __ addw(dst_reg, src_reg, con);
 3551     }
 3552   %}
 3553 
 3554   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3555     Register dst_reg = as_Register($dst$$reg);
 3556     Register src_reg = as_Register($src1$$reg);
 3557     int32_t con = (int32_t)$src2$$constant;
 3558     // add has primary == 0, subtract has primary == 1
 3559     if ($primary) { con = -con; }
 3560     if (con < 0) {
 3561       __ sub(dst_reg, src_reg, -con);
 3562     } else {
 3563       __ add(dst_reg, src_reg, con);
 3564     }
 3565   %}
 3566 
 3567   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3568    Register dst_reg = as_Register($dst$$reg);
 3569    Register src1_reg = as_Register($src1$$reg);
 3570    Register src2_reg = as_Register($src2$$reg);
 3571     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3572   %}
 3573 
 3574   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3575    Register dst_reg = as_Register($dst$$reg);
 3576    Register src1_reg = as_Register($src1$$reg);
 3577    Register src2_reg = as_Register($src2$$reg);
 3578     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3579   %}
 3580 
 3581   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3582    Register dst_reg = as_Register($dst$$reg);
 3583    Register src1_reg = as_Register($src1$$reg);
 3584    Register src2_reg = as_Register($src2$$reg);
 3585     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3586   %}
 3587 
 3588   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3589    Register dst_reg = as_Register($dst$$reg);
 3590    Register src1_reg = as_Register($src1$$reg);
 3591    Register src2_reg = as_Register($src2$$reg);
 3592     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3593   %}
 3594 
 3595   // compare instruction encodings
 3596 
 3597   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3598     Register reg1 = as_Register($src1$$reg);
 3599     Register reg2 = as_Register($src2$$reg);
 3600     __ cmpw(reg1, reg2);
 3601   %}
 3602 
 3603   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3604     Register reg = as_Register($src1$$reg);
 3605     int32_t val = $src2$$constant;
 3606     if (val >= 0) {
 3607       __ subsw(zr, reg, val);
 3608     } else {
 3609       __ addsw(zr, reg, -val);
 3610     }
 3611   %}
 3612 
 3613   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3614     Register reg1 = as_Register($src1$$reg);
 3615     uint32_t val = (uint32_t)$src2$$constant;
 3616     __ movw(rscratch1, val);
 3617     __ cmpw(reg1, rscratch1);
 3618   %}
 3619 
 3620   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3621     Register reg1 = as_Register($src1$$reg);
 3622     Register reg2 = as_Register($src2$$reg);
 3623     __ cmp(reg1, reg2);
 3624   %}
 3625 
 3626   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3627     Register reg = as_Register($src1$$reg);
 3628     int64_t val = $src2$$constant;
 3629     if (val >= 0) {
 3630       __ subs(zr, reg, val);
 3631     } else if (val != -val) {
 3632       __ adds(zr, reg, -val);
 3633     } else {
 3634     // aargh, Long.MIN_VALUE is a special case
 3635       __ orr(rscratch1, zr, (uint64_t)val);
 3636       __ subs(zr, reg, rscratch1);
 3637     }
 3638   %}
 3639 
 3640   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3641     Register reg1 = as_Register($src1$$reg);
 3642     uint64_t val = (uint64_t)$src2$$constant;
 3643     __ mov(rscratch1, val);
 3644     __ cmp(reg1, rscratch1);
 3645   %}
 3646 
 3647   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3648     Register reg1 = as_Register($src1$$reg);
 3649     Register reg2 = as_Register($src2$$reg);
 3650     __ cmp(reg1, reg2);
 3651   %}
 3652 
 3653   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3654     Register reg1 = as_Register($src1$$reg);
 3655     Register reg2 = as_Register($src2$$reg);
 3656     __ cmpw(reg1, reg2);
 3657   %}
 3658 
 3659   enc_class aarch64_enc_testp(iRegP src) %{
 3660     Register reg = as_Register($src$$reg);
 3661     __ cmp(reg, zr);
 3662   %}
 3663 
 3664   enc_class aarch64_enc_testn(iRegN src) %{
 3665     Register reg = as_Register($src$$reg);
 3666     __ cmpw(reg, zr);
 3667   %}
 3668 
 3669   enc_class aarch64_enc_b(label lbl) %{
 3670     Label *L = $lbl$$label;
 3671     __ b(*L);
 3672   %}
 3673 
 3674   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3675     Label *L = $lbl$$label;
 3676     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3677   %}
 3678 
 3679   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3680     Label *L = $lbl$$label;
 3681     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3682   %}
 3683 
 3684   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3685   %{
 3686      Register sub_reg = as_Register($sub$$reg);
 3687      Register super_reg = as_Register($super$$reg);
 3688      Register temp_reg = as_Register($temp$$reg);
 3689      Register result_reg = as_Register($result$$reg);
 3690 
 3691      Label miss;
 3692      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3693                                      nullptr, &miss,
 3694                                      /*set_cond_codes:*/ true);
 3695      if ($primary) {
 3696        __ mov(result_reg, zr);
 3697      }
 3698      __ bind(miss);
 3699   %}
 3700 
 3701   enc_class aarch64_enc_java_static_call(method meth) %{
 3702     address addr = (address)$meth$$method;
 3703     address call;
 3704     if (!_method) {
 3705       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3706       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
 3707       if (call == nullptr) {
 3708         ciEnv::current()->record_failure("CodeCache is full");
 3709         return;
 3710       }
 3711     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 3712       // The NOP here is purely to ensure that eliding a call to
 3713       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 3714       __ nop();
 3715       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 3716     } else {
 3717       int method_index = resolved_method_index(masm);
 3718       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3719                                                   : static_call_Relocation::spec(method_index);
 3720       call = __ trampoline_call(Address(addr, rspec));
 3721       if (call == nullptr) {
 3722         ciEnv::current()->record_failure("CodeCache is full");
 3723         return;
 3724       }
 3725       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 3726         // Calls of the same statically bound method can share
 3727         // a stub to the interpreter.
 3728         __ code()->shared_stub_to_interp_for(_method, call - __ begin());
 3729       } else {
 3730         // Emit stub for static call
 3731         address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
 3732         if (stub == nullptr) {
 3733           ciEnv::current()->record_failure("CodeCache is full");
 3734           return;
 3735         }
 3736       }
 3737     }
 3738 
 3739     __ post_call_nop();
 3740 
 3741     // Only non uncommon_trap calls need to reinitialize ptrue.
 3742     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3743       __ reinitialize_ptrue();
 3744     }
 3745   %}
 3746 
 3747   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3748     int method_index = resolved_method_index(masm);
 3749     address call = __ ic_call((address)$meth$$method, method_index);
 3750     if (call == nullptr) {
 3751       ciEnv::current()->record_failure("CodeCache is full");
 3752       return;
 3753     }
 3754     __ post_call_nop();
 3755     if (Compile::current()->max_vector_size() > 0) {
 3756       __ reinitialize_ptrue();
 3757     }
 3758   %}
 3759 
 3760   enc_class aarch64_enc_call_epilog() %{
 3761     if (VerifyStackAtCalls) {
 3762       // Check that stack depth is unchanged: find majik cookie on stack
 3763       __ call_Unimplemented();
 3764     }
 3765   %}
 3766 
 3767   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3768     // some calls to generated routines (arraycopy code) are scheduled
 3769     // by C2 as runtime calls. if so we can call them using a br (they
 3770     // will be in a reachable segment) otherwise we have to use a blr
 3771     // which loads the absolute address into a register.
 3772     address entry = (address)$meth$$method;
 3773     CodeBlob *cb = CodeCache::find_blob(entry);
 3774     if (cb) {
 3775       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3776       if (call == nullptr) {
 3777         ciEnv::current()->record_failure("CodeCache is full");
 3778         return;
 3779       }
 3780       __ post_call_nop();
 3781     } else {
 3782       Label retaddr;
 3783       // Make the anchor frame walkable
 3784       __ adr(rscratch2, retaddr);
 3785       __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
 3786       __ lea(rscratch1, RuntimeAddress(entry));
 3787       __ blr(rscratch1);
 3788       __ bind(retaddr);
 3789       __ post_call_nop();
 3790     }
 3791     if (Compile::current()->max_vector_size() > 0) {
 3792       __ reinitialize_ptrue();
 3793     }
 3794   %}
 3795 
 3796   enc_class aarch64_enc_rethrow() %{
 3797     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3798   %}
 3799 
 3800   enc_class aarch64_enc_ret() %{
 3801 #ifdef ASSERT
 3802     if (Compile::current()->max_vector_size() > 0) {
 3803       __ verify_ptrue();
 3804     }
 3805 #endif
 3806     __ ret(lr);
 3807   %}
 3808 
 3809   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3810     Register target_reg = as_Register($jump_target$$reg);
 3811     __ br(target_reg);
 3812   %}
 3813 
 3814   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3815     Register target_reg = as_Register($jump_target$$reg);
 3816     // exception oop should be in r0
 3817     // ret addr has been popped into lr
 3818     // callee expects it in r3
 3819     __ mov(r3, lr);
 3820     __ br(target_reg);
 3821   %}
 3822 
 3823 %}
 3824 
 3825 //----------FRAME--------------------------------------------------------------
 3826 // Definition of frame structure and management information.
 3827 //
 3828 //  S T A C K   L A Y O U T    Allocators stack-slot number
 3829 //                             |   (to get allocators register number
 3830 //  G  Owned by    |        |  v    add OptoReg::stack0())
 3831 //  r   CALLER     |        |
 3832 //  o     |        +--------+      pad to even-align allocators stack-slot
 3833 //  w     V        |  pad0  |        numbers; owned by CALLER
 3834 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 3835 //  h     ^        |   in   |  5
 3836 //        |        |  args  |  4   Holes in incoming args owned by SELF
 3837 //  |     |        |        |  3
 3838 //  |     |        +--------+
 3839 //  V     |        | old out|      Empty on Intel, window on Sparc
 3840 //        |    old |preserve|      Must be even aligned.
 3841 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 3842 //        |        |   in   |  3   area for Intel ret address
 3843 //     Owned by    |preserve|      Empty on Sparc.
 3844 //       SELF      +--------+
 3845 //        |        |  pad2  |  2   pad to align old SP
 3846 //        |        +--------+  1
 3847 //        |        | locks  |  0
 3848 //        |        +--------+----> OptoReg::stack0(), even aligned
 3849 //        |        |  pad1  | 11   pad to align new SP
 3850 //        |        +--------+
 3851 //        |        |        | 10
 3852 //        |        | spills |  9   spills
 3853 //        V        |        |  8   (pad0 slot for callee)
 3854 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 3855 //        ^        |  out   |  7
 3856 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 3857 //     Owned by    +--------+
 3858 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 3859 //        |    new |preserve|      Must be even-aligned.
 3860 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 3861 //        |        |        |
 3862 //
 3863 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 3864 //         known from SELF's arguments and the Java calling convention.
 3865 //         Region 6-7 is determined per call site.
 3866 // Note 2: If the calling convention leaves holes in the incoming argument
 3867 //         area, those holes are owned by SELF.  Holes in the outgoing area
 3868 //         are owned by the CALLEE.  Holes should not be necessary in the
 3869 //         incoming area, as the Java calling convention is completely under
 3870 //         the control of the AD file.  Doubles can be sorted and packed to
 3871 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 3872 //         varargs C calling conventions.
 3873 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 3874 //         even aligned with pad0 as needed.
 3875 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 3876 //           (the latter is true on Intel but is it false on AArch64?)
 3877 //         region 6-11 is even aligned; it may be padded out more so that
 3878 //         the region from SP to FP meets the minimum stack alignment.
 3879 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 3880 //         alignment.  Region 11, pad1, may be dynamically extended so that
 3881 //         SP meets the minimum alignment.
 3882 
 3883 frame %{
 3884   // These three registers define part of the calling convention
 3885   // between compiled code and the interpreter.
 3886 
 3887   // Inline Cache Register or Method for I2C.
 3888   inline_cache_reg(R12);
 3889 
 3890   // Number of stack slots consumed by locking an object
 3891   sync_stack_slots(2);
 3892 
 3893   // Compiled code's Frame Pointer
 3894   frame_pointer(R31);
 3895 
 3896   // Interpreter stores its frame pointer in a register which is
 3897   // stored to the stack by I2CAdaptors.
 3898   // I2CAdaptors convert from interpreted java to compiled java.
 3899   interpreter_frame_pointer(R29);
 3900 
 3901   // Stack alignment requirement
 3902   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 3903 
 3904   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 3905   // for calls to C.  Supports the var-args backing area for register parms.
 3906   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 3907 
 3908   // The after-PROLOG location of the return address.  Location of
 3909   // return address specifies a type (REG or STACK) and a number
 3910   // representing the register number (i.e. - use a register name) or
 3911   // stack slot.
 3912   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 3913   // Otherwise, it is above the locks and verification slot and alignment word
 3914   // TODO this may well be correct but need to check why that - 2 is there
 3915   // ppc port uses 0 but we definitely need to allow for fixed_slots
 3916   // which folds in the space used for monitors
 3917   return_addr(STACK - 2 +
 3918               align_up((Compile::current()->in_preserve_stack_slots() +
 3919                         Compile::current()->fixed_slots()),
 3920                        stack_alignment_in_slots()));
 3921 
 3922   // Location of compiled Java return values.  Same as C for now.
 3923   return_value
 3924   %{
 3925     // TODO do we allow ideal_reg == Op_RegN???
 3926     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 3927            "only return normal values");
 3928 
 3929     static const int lo[Op_RegL + 1] = { // enum name
 3930       0,                                 // Op_Node
 3931       0,                                 // Op_Set
 3932       R0_num,                            // Op_RegN
 3933       R0_num,                            // Op_RegI
 3934       R0_num,                            // Op_RegP
 3935       V0_num,                            // Op_RegF
 3936       V0_num,                            // Op_RegD
 3937       R0_num                             // Op_RegL
 3938     };
 3939 
 3940     static const int hi[Op_RegL + 1] = { // enum name
 3941       0,                                 // Op_Node
 3942       0,                                 // Op_Set
 3943       OptoReg::Bad,                      // Op_RegN
 3944       OptoReg::Bad,                      // Op_RegI
 3945       R0_H_num,                          // Op_RegP
 3946       OptoReg::Bad,                      // Op_RegF
 3947       V0_H_num,                          // Op_RegD
 3948       R0_H_num                           // Op_RegL
 3949     };
 3950 
 3951     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 3952   %}
 3953 %}
 3954 
 3955 //----------ATTRIBUTES---------------------------------------------------------
 3956 //----------Operand Attributes-------------------------------------------------
 3957 op_attrib op_cost(1);        // Required cost attribute
 3958 
 3959 //----------Instruction Attributes---------------------------------------------
 3960 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 3961 ins_attrib ins_size(32);        // Required size attribute (in bits)
 3962 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 3963                                 // a non-matching short branch variant
 3964                                 // of some long branch?
 3965 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 3966                                 // be a power of 2) specifies the
 3967                                 // alignment that some part of the
 3968                                 // instruction (not necessarily the
 3969                                 // start) requires.  If > 1, a
 3970                                 // compute_padding() function must be
 3971                                 // provided for the instruction
 3972 
 3973 // Whether this node is expanded during code emission into a sequence of
 3974 // instructions and the first instruction can perform an implicit null check.
 3975 ins_attrib ins_is_late_expanded_null_check_candidate(false);
 3976 
 3977 //----------OPERANDS-----------------------------------------------------------
 3978 // Operand definitions must precede instruction definitions for correct parsing
 3979 // in the ADLC because operands constitute user defined types which are used in
 3980 // instruction definitions.
 3981 
 3982 //----------Simple Operands----------------------------------------------------
 3983 
 3984 // Integer operands 32 bit
 3985 // 32 bit immediate
 3986 operand immI()
 3987 %{
 3988   match(ConI);
 3989 
 3990   op_cost(0);
 3991   format %{ %}
 3992   interface(CONST_INTER);
 3993 %}
 3994 
 3995 // 32 bit zero
 3996 operand immI0()
 3997 %{
 3998   predicate(n->get_int() == 0);
 3999   match(ConI);
 4000 
 4001   op_cost(0);
 4002   format %{ %}
 4003   interface(CONST_INTER);
 4004 %}
 4005 
 4006 // 32 bit unit increment
 4007 operand immI_1()
 4008 %{
 4009   predicate(n->get_int() == 1);
 4010   match(ConI);
 4011 
 4012   op_cost(0);
 4013   format %{ %}
 4014   interface(CONST_INTER);
 4015 %}
 4016 
 4017 // 32 bit unit decrement
 4018 operand immI_M1()
 4019 %{
 4020   predicate(n->get_int() == -1);
 4021   match(ConI);
 4022 
 4023   op_cost(0);
 4024   format %{ %}
 4025   interface(CONST_INTER);
 4026 %}
 4027 
 4028 // Shift values for add/sub extension shift
 4029 operand immIExt()
 4030 %{
 4031   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4032   match(ConI);
 4033 
 4034   op_cost(0);
 4035   format %{ %}
 4036   interface(CONST_INTER);
 4037 %}
 4038 
 4039 operand immI_gt_1()
 4040 %{
 4041   predicate(n->get_int() > 1);
 4042   match(ConI);
 4043 
 4044   op_cost(0);
 4045   format %{ %}
 4046   interface(CONST_INTER);
 4047 %}
 4048 
 4049 operand immI_le_4()
 4050 %{
 4051   predicate(n->get_int() <= 4);
 4052   match(ConI);
 4053 
 4054   op_cost(0);
 4055   format %{ %}
 4056   interface(CONST_INTER);
 4057 %}
 4058 
 4059 operand immI_16()
 4060 %{
 4061   predicate(n->get_int() == 16);
 4062   match(ConI);
 4063 
 4064   op_cost(0);
 4065   format %{ %}
 4066   interface(CONST_INTER);
 4067 %}
 4068 
 4069 operand immI_24()
 4070 %{
 4071   predicate(n->get_int() == 24);
 4072   match(ConI);
 4073 
 4074   op_cost(0);
 4075   format %{ %}
 4076   interface(CONST_INTER);
 4077 %}
 4078 
 4079 operand immI_32()
 4080 %{
 4081   predicate(n->get_int() == 32);
 4082   match(ConI);
 4083 
 4084   op_cost(0);
 4085   format %{ %}
 4086   interface(CONST_INTER);
 4087 %}
 4088 
 4089 operand immI_48()
 4090 %{
 4091   predicate(n->get_int() == 48);
 4092   match(ConI);
 4093 
 4094   op_cost(0);
 4095   format %{ %}
 4096   interface(CONST_INTER);
 4097 %}
 4098 
 4099 operand immI_56()
 4100 %{
 4101   predicate(n->get_int() == 56);
 4102   match(ConI);
 4103 
 4104   op_cost(0);
 4105   format %{ %}
 4106   interface(CONST_INTER);
 4107 %}
 4108 
 4109 operand immI_255()
 4110 %{
 4111   predicate(n->get_int() == 255);
 4112   match(ConI);
 4113 
 4114   op_cost(0);
 4115   format %{ %}
 4116   interface(CONST_INTER);
 4117 %}
 4118 
 4119 operand immI_65535()
 4120 %{
 4121   predicate(n->get_int() == 65535);
 4122   match(ConI);
 4123 
 4124   op_cost(0);
 4125   format %{ %}
 4126   interface(CONST_INTER);
 4127 %}
 4128 
 4129 operand immI_positive()
 4130 %{
 4131   predicate(n->get_int() > 0);
 4132   match(ConI);
 4133 
 4134   op_cost(0);
 4135   format %{ %}
 4136   interface(CONST_INTER);
 4137 %}
 4138 
 4139 // BoolTest condition for signed compare
 4140 operand immI_cmp_cond()
 4141 %{
 4142   predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
 4143   match(ConI);
 4144 
 4145   op_cost(0);
 4146   format %{ %}
 4147   interface(CONST_INTER);
 4148 %}
 4149 
 4150 // BoolTest condition for unsigned compare
 4151 operand immI_cmpU_cond()
 4152 %{
 4153   predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
 4154   match(ConI);
 4155 
 4156   op_cost(0);
 4157   format %{ %}
 4158   interface(CONST_INTER);
 4159 %}
 4160 
 4161 operand immL_255()
 4162 %{
 4163   predicate(n->get_long() == 255L);
 4164   match(ConL);
 4165 
 4166   op_cost(0);
 4167   format %{ %}
 4168   interface(CONST_INTER);
 4169 %}
 4170 
 4171 operand immL_65535()
 4172 %{
 4173   predicate(n->get_long() == 65535L);
 4174   match(ConL);
 4175 
 4176   op_cost(0);
 4177   format %{ %}
 4178   interface(CONST_INTER);
 4179 %}
 4180 
 4181 operand immL_4294967295()
 4182 %{
 4183   predicate(n->get_long() == 4294967295L);
 4184   match(ConL);
 4185 
 4186   op_cost(0);
 4187   format %{ %}
 4188   interface(CONST_INTER);
 4189 %}
 4190 
 4191 operand immL_bitmask()
 4192 %{
 4193   predicate((n->get_long() != 0)
 4194             && ((n->get_long() & 0xc000000000000000l) == 0)
 4195             && is_power_of_2(n->get_long() + 1));
 4196   match(ConL);
 4197 
 4198   op_cost(0);
 4199   format %{ %}
 4200   interface(CONST_INTER);
 4201 %}
 4202 
 4203 operand immI_bitmask()
 4204 %{
 4205   predicate((n->get_int() != 0)
 4206             && ((n->get_int() & 0xc0000000) == 0)
 4207             && is_power_of_2(n->get_int() + 1));
 4208   match(ConI);
 4209 
 4210   op_cost(0);
 4211   format %{ %}
 4212   interface(CONST_INTER);
 4213 %}
 4214 
 4215 operand immL_positive_bitmaskI()
 4216 %{
 4217   predicate((n->get_long() != 0)
 4218             && ((julong)n->get_long() < 0x80000000ULL)
 4219             && is_power_of_2(n->get_long() + 1));
 4220   match(ConL);
 4221 
 4222   op_cost(0);
 4223   format %{ %}
 4224   interface(CONST_INTER);
 4225 %}
 4226 
 4227 // Scale values for scaled offset addressing modes (up to long but not quad)
 4228 operand immIScale()
 4229 %{
 4230   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4231   match(ConI);
 4232 
 4233   op_cost(0);
 4234   format %{ %}
 4235   interface(CONST_INTER);
 4236 %}
 4237 
 4238 // 5 bit signed integer
 4239 operand immI5()
 4240 %{
 4241   predicate(Assembler::is_simm(n->get_int(), 5));
 4242   match(ConI);
 4243 
 4244   op_cost(0);
 4245   format %{ %}
 4246   interface(CONST_INTER);
 4247 %}
 4248 
 4249 // 7 bit unsigned integer
 4250 operand immIU7()
 4251 %{
 4252   predicate(Assembler::is_uimm(n->get_int(), 7));
 4253   match(ConI);
 4254 
 4255   op_cost(0);
 4256   format %{ %}
 4257   interface(CONST_INTER);
 4258 %}
 4259 
 4260 // Offset for scaled or unscaled immediate loads and stores
 4261 operand immIOffset()
 4262 %{
 4263   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4264   match(ConI);
 4265 
 4266   op_cost(0);
 4267   format %{ %}
 4268   interface(CONST_INTER);
 4269 %}
 4270 
 4271 operand immIOffset1()
 4272 %{
 4273   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4274   match(ConI);
 4275 
 4276   op_cost(0);
 4277   format %{ %}
 4278   interface(CONST_INTER);
 4279 %}
 4280 
 4281 operand immIOffset2()
 4282 %{
 4283   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4284   match(ConI);
 4285 
 4286   op_cost(0);
 4287   format %{ %}
 4288   interface(CONST_INTER);
 4289 %}
 4290 
 4291 operand immIOffset4()
 4292 %{
 4293   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4294   match(ConI);
 4295 
 4296   op_cost(0);
 4297   format %{ %}
 4298   interface(CONST_INTER);
 4299 %}
 4300 
 4301 operand immIOffset8()
 4302 %{
 4303   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4304   match(ConI);
 4305 
 4306   op_cost(0);
 4307   format %{ %}
 4308   interface(CONST_INTER);
 4309 %}
 4310 
 4311 operand immIOffset16()
 4312 %{
 4313   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4314   match(ConI);
 4315 
 4316   op_cost(0);
 4317   format %{ %}
 4318   interface(CONST_INTER);
 4319 %}
 4320 
 4321 operand immLOffset()
 4322 %{
 4323   predicate(n->get_long() >= -256 && n->get_long() <= 65520);
 4324   match(ConL);
 4325 
 4326   op_cost(0);
 4327   format %{ %}
 4328   interface(CONST_INTER);
 4329 %}
 4330 
 4331 operand immLoffset1()
 4332 %{
 4333   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4334   match(ConL);
 4335 
 4336   op_cost(0);
 4337   format %{ %}
 4338   interface(CONST_INTER);
 4339 %}
 4340 
 4341 operand immLoffset2()
 4342 %{
 4343   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4344   match(ConL);
 4345 
 4346   op_cost(0);
 4347   format %{ %}
 4348   interface(CONST_INTER);
 4349 %}
 4350 
 4351 operand immLoffset4()
 4352 %{
 4353   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4354   match(ConL);
 4355 
 4356   op_cost(0);
 4357   format %{ %}
 4358   interface(CONST_INTER);
 4359 %}
 4360 
 4361 operand immLoffset8()
 4362 %{
 4363   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4364   match(ConL);
 4365 
 4366   op_cost(0);
 4367   format %{ %}
 4368   interface(CONST_INTER);
 4369 %}
 4370 
 4371 operand immLoffset16()
 4372 %{
 4373   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4374   match(ConL);
 4375 
 4376   op_cost(0);
 4377   format %{ %}
 4378   interface(CONST_INTER);
 4379 %}
 4380 
 4381 // 5 bit signed long integer
 4382 operand immL5()
 4383 %{
 4384   predicate(Assembler::is_simm(n->get_long(), 5));
 4385   match(ConL);
 4386 
 4387   op_cost(0);
 4388   format %{ %}
 4389   interface(CONST_INTER);
 4390 %}
 4391 
 4392 // 7 bit unsigned long integer
 4393 operand immLU7()
 4394 %{
 4395   predicate(Assembler::is_uimm(n->get_long(), 7));
 4396   match(ConL);
 4397 
 4398   op_cost(0);
 4399   format %{ %}
 4400   interface(CONST_INTER);
 4401 %}
 4402 
 4403 // 8 bit signed value.
 4404 operand immI8()
 4405 %{
 4406   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4407   match(ConI);
 4408 
 4409   op_cost(0);
 4410   format %{ %}
 4411   interface(CONST_INTER);
 4412 %}
 4413 
 4414 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4415 operand immIDupV()
 4416 %{
 4417   predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
 4418   match(ConI);
 4419 
 4420   op_cost(0);
 4421   format %{ %}
 4422   interface(CONST_INTER);
 4423 %}
 4424 
 4425 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4426 operand immLDupV()
 4427 %{
 4428   predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
 4429   match(ConL);
 4430 
 4431   op_cost(0);
 4432   format %{ %}
 4433   interface(CONST_INTER);
 4434 %}
 4435 
 4436 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4437 operand immHDupV()
 4438 %{
 4439   predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
 4440   match(ConH);
 4441 
 4442   op_cost(0);
 4443   format %{ %}
 4444   interface(CONST_INTER);
 4445 %}
 4446 
 4447 // 8 bit integer valid for vector add sub immediate
 4448 operand immBAddSubV()
 4449 %{
 4450   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4451   match(ConI);
 4452 
 4453   op_cost(0);
 4454   format %{ %}
 4455   interface(CONST_INTER);
 4456 %}
 4457 
 4458 // 32 bit integer valid for add sub immediate
 4459 operand immIAddSub()
 4460 %{
 4461   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4462   match(ConI);
 4463   op_cost(0);
 4464   format %{ %}
 4465   interface(CONST_INTER);
 4466 %}
 4467 
 4468 // 32 bit integer valid for vector add sub immediate
 4469 operand immIAddSubV()
 4470 %{
 4471   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4472   match(ConI);
 4473 
 4474   op_cost(0);
 4475   format %{ %}
 4476   interface(CONST_INTER);
 4477 %}
 4478 
 4479 // 32 bit unsigned integer valid for logical immediate
 4480 
 4481 operand immBLog()
 4482 %{
 4483   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4484   match(ConI);
 4485 
 4486   op_cost(0);
 4487   format %{ %}
 4488   interface(CONST_INTER);
 4489 %}
 4490 
 4491 operand immSLog()
 4492 %{
 4493   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4494   match(ConI);
 4495 
 4496   op_cost(0);
 4497   format %{ %}
 4498   interface(CONST_INTER);
 4499 %}
 4500 
 4501 operand immILog()
 4502 %{
 4503   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4504   match(ConI);
 4505 
 4506   op_cost(0);
 4507   format %{ %}
 4508   interface(CONST_INTER);
 4509 %}
 4510 
 4511 // Integer operands 64 bit
 4512 // 64 bit immediate
 4513 operand immL()
 4514 %{
 4515   match(ConL);
 4516 
 4517   op_cost(0);
 4518   format %{ %}
 4519   interface(CONST_INTER);
 4520 %}
 4521 
 4522 // 64 bit zero
 4523 operand immL0()
 4524 %{
 4525   predicate(n->get_long() == 0);
 4526   match(ConL);
 4527 
 4528   op_cost(0);
 4529   format %{ %}
 4530   interface(CONST_INTER);
 4531 %}
 4532 
 4533 // 64 bit unit decrement
 4534 operand immL_M1()
 4535 %{
 4536   predicate(n->get_long() == -1);
 4537   match(ConL);
 4538 
 4539   op_cost(0);
 4540   format %{ %}
 4541   interface(CONST_INTER);
 4542 %}
 4543 
 4544 // 64 bit integer valid for add sub immediate
 4545 operand immLAddSub()
 4546 %{
 4547   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4548   match(ConL);
 4549   op_cost(0);
 4550   format %{ %}
 4551   interface(CONST_INTER);
 4552 %}
 4553 
 4554 // 64 bit integer valid for addv subv immediate
 4555 operand immLAddSubV()
 4556 %{
 4557   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4558   match(ConL);
 4559 
 4560   op_cost(0);
 4561   format %{ %}
 4562   interface(CONST_INTER);
 4563 %}
 4564 
 4565 // 64 bit integer valid for logical immediate
 4566 operand immLLog()
 4567 %{
 4568   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4569   match(ConL);
 4570   op_cost(0);
 4571   format %{ %}
 4572   interface(CONST_INTER);
 4573 %}
 4574 
 4575 // Long Immediate: low 32-bit mask
 4576 operand immL_32bits()
 4577 %{
 4578   predicate(n->get_long() == 0xFFFFFFFFL);
 4579   match(ConL);
 4580   op_cost(0);
 4581   format %{ %}
 4582   interface(CONST_INTER);
 4583 %}
 4584 
 4585 // Pointer operands
 4586 // Pointer Immediate
 4587 operand immP()
 4588 %{
 4589   match(ConP);
 4590 
 4591   op_cost(0);
 4592   format %{ %}
 4593   interface(CONST_INTER);
 4594 %}
 4595 
 4596 // nullptr Pointer Immediate
 4597 operand immP0()
 4598 %{
 4599   predicate(n->get_ptr() == 0);
 4600   match(ConP);
 4601 
 4602   op_cost(0);
 4603   format %{ %}
 4604   interface(CONST_INTER);
 4605 %}
 4606 
 4607 // Pointer Immediate One
 4608 // this is used in object initialization (initial object header)
 4609 operand immP_1()
 4610 %{
 4611   predicate(n->get_ptr() == 1);
 4612   match(ConP);
 4613 
 4614   op_cost(0);
 4615   format %{ %}
 4616   interface(CONST_INTER);
 4617 %}
 4618 
 4619 // Float and Double operands
 4620 // Double Immediate
 4621 operand immD()
 4622 %{
 4623   match(ConD);
 4624   op_cost(0);
 4625   format %{ %}
 4626   interface(CONST_INTER);
 4627 %}
 4628 
 4629 // Double Immediate: +0.0d
 4630 operand immD0()
 4631 %{
 4632   predicate(jlong_cast(n->getd()) == 0);
 4633   match(ConD);
 4634 
 4635   op_cost(0);
 4636   format %{ %}
 4637   interface(CONST_INTER);
 4638 %}
 4639 
 4640 // constant 'double +0.0'.
 4641 operand immDPacked()
 4642 %{
 4643   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4644   match(ConD);
 4645   op_cost(0);
 4646   format %{ %}
 4647   interface(CONST_INTER);
 4648 %}
 4649 
 4650 // Float Immediate
 4651 operand immF()
 4652 %{
 4653   match(ConF);
 4654   op_cost(0);
 4655   format %{ %}
 4656   interface(CONST_INTER);
 4657 %}
 4658 
 4659 // Float Immediate: +0.0f.
 4660 operand immF0()
 4661 %{
 4662   predicate(jint_cast(n->getf()) == 0);
 4663   match(ConF);
 4664 
 4665   op_cost(0);
 4666   format %{ %}
 4667   interface(CONST_INTER);
 4668 %}
 4669 
 4670 // Half Float (FP16) Immediate
 4671 operand immH()
 4672 %{
 4673   match(ConH);
 4674   op_cost(0);
 4675   format %{ %}
 4676   interface(CONST_INTER);
 4677 %}
 4678 
 4679 //
 4680 operand immFPacked()
 4681 %{
 4682   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4683   match(ConF);
 4684   op_cost(0);
 4685   format %{ %}
 4686   interface(CONST_INTER);
 4687 %}
 4688 
 4689 // Narrow pointer operands
 4690 // Narrow Pointer Immediate
 4691 operand immN()
 4692 %{
 4693   match(ConN);
 4694 
 4695   op_cost(0);
 4696   format %{ %}
 4697   interface(CONST_INTER);
 4698 %}
 4699 
 4700 // Narrow nullptr Pointer Immediate
 4701 operand immN0()
 4702 %{
 4703   predicate(n->get_narrowcon() == 0);
 4704   match(ConN);
 4705 
 4706   op_cost(0);
 4707   format %{ %}
 4708   interface(CONST_INTER);
 4709 %}
 4710 
 4711 operand immNKlass()
 4712 %{
 4713   match(ConNKlass);
 4714 
 4715   op_cost(0);
 4716   format %{ %}
 4717   interface(CONST_INTER);
 4718 %}
 4719 
 4720 // Integer 32 bit Register Operands
 4721 // Integer 32 bitRegister (excludes SP)
 4722 operand iRegI()
 4723 %{
 4724   constraint(ALLOC_IN_RC(any_reg32));
 4725   match(RegI);
 4726   match(iRegINoSp);
 4727   op_cost(0);
 4728   format %{ %}
 4729   interface(REG_INTER);
 4730 %}
 4731 
 4732 // Integer 32 bit Register not Special
 4733 operand iRegINoSp()
 4734 %{
 4735   constraint(ALLOC_IN_RC(no_special_reg32));
 4736   match(RegI);
 4737   op_cost(0);
 4738   format %{ %}
 4739   interface(REG_INTER);
 4740 %}
 4741 
 4742 // Integer 64 bit Register Operands
 4743 // Integer 64 bit Register (includes SP)
 4744 operand iRegL()
 4745 %{
 4746   constraint(ALLOC_IN_RC(any_reg));
 4747   match(RegL);
 4748   match(iRegLNoSp);
 4749   op_cost(0);
 4750   format %{ %}
 4751   interface(REG_INTER);
 4752 %}
 4753 
 4754 // Integer 64 bit Register not Special
 4755 operand iRegLNoSp()
 4756 %{
 4757   constraint(ALLOC_IN_RC(no_special_reg));
 4758   match(RegL);
 4759   match(iRegL_R0);
 4760   format %{ %}
 4761   interface(REG_INTER);
 4762 %}
 4763 
 4764 // Pointer Register Operands
 4765 // Pointer Register
 4766 operand iRegP()
 4767 %{
 4768   constraint(ALLOC_IN_RC(ptr_reg));
 4769   match(RegP);
 4770   match(iRegPNoSp);
 4771   match(iRegP_R0);
 4772   //match(iRegP_R2);
 4773   //match(iRegP_R4);
 4774   match(iRegP_R5);
 4775   match(thread_RegP);
 4776   op_cost(0);
 4777   format %{ %}
 4778   interface(REG_INTER);
 4779 %}
 4780 
 4781 // Pointer 64 bit Register not Special
 4782 operand iRegPNoSp()
 4783 %{
 4784   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4785   match(RegP);
 4786   // match(iRegP);
 4787   // match(iRegP_R0);
 4788   // match(iRegP_R2);
 4789   // match(iRegP_R4);
 4790   // match(iRegP_R5);
 4791   // match(thread_RegP);
 4792   op_cost(0);
 4793   format %{ %}
 4794   interface(REG_INTER);
 4795 %}
 4796 
 4797 // This operand is not allowed to use rfp even if
 4798 // rfp is not used to hold the frame pointer.
 4799 operand iRegPNoSpNoRfp()
 4800 %{
 4801   constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
 4802   match(RegP);
 4803   match(iRegPNoSp);
 4804   op_cost(0);
 4805   format %{ %}
 4806   interface(REG_INTER);
 4807 %}
 4808 
 4809 // Pointer 64 bit Register R0 only
 4810 operand iRegP_R0()
 4811 %{
 4812   constraint(ALLOC_IN_RC(r0_reg));
 4813   match(RegP);
 4814   // match(iRegP);
 4815   match(iRegPNoSp);
 4816   op_cost(0);
 4817   format %{ %}
 4818   interface(REG_INTER);
 4819 %}
 4820 
 4821 // Pointer 64 bit Register R1 only
 4822 operand iRegP_R1()
 4823 %{
 4824   constraint(ALLOC_IN_RC(r1_reg));
 4825   match(RegP);
 4826   // match(iRegP);
 4827   match(iRegPNoSp);
 4828   op_cost(0);
 4829   format %{ %}
 4830   interface(REG_INTER);
 4831 %}
 4832 
 4833 // Pointer 64 bit Register R2 only
 4834 operand iRegP_R2()
 4835 %{
 4836   constraint(ALLOC_IN_RC(r2_reg));
 4837   match(RegP);
 4838   // match(iRegP);
 4839   match(iRegPNoSp);
 4840   op_cost(0);
 4841   format %{ %}
 4842   interface(REG_INTER);
 4843 %}
 4844 
 4845 // Pointer 64 bit Register R3 only
 4846 operand iRegP_R3()
 4847 %{
 4848   constraint(ALLOC_IN_RC(r3_reg));
 4849   match(RegP);
 4850   // match(iRegP);
 4851   match(iRegPNoSp);
 4852   op_cost(0);
 4853   format %{ %}
 4854   interface(REG_INTER);
 4855 %}
 4856 
 4857 // Pointer 64 bit Register R4 only
 4858 operand iRegP_R4()
 4859 %{
 4860   constraint(ALLOC_IN_RC(r4_reg));
 4861   match(RegP);
 4862   // match(iRegP);
 4863   match(iRegPNoSp);
 4864   op_cost(0);
 4865   format %{ %}
 4866   interface(REG_INTER);
 4867 %}
 4868 
 4869 // Pointer 64 bit Register R5 only
 4870 operand iRegP_R5()
 4871 %{
 4872   constraint(ALLOC_IN_RC(r5_reg));
 4873   match(RegP);
 4874   // match(iRegP);
 4875   match(iRegPNoSp);
 4876   op_cost(0);
 4877   format %{ %}
 4878   interface(REG_INTER);
 4879 %}
 4880 
 4881 // Pointer 64 bit Register R10 only
 4882 operand iRegP_R10()
 4883 %{
 4884   constraint(ALLOC_IN_RC(r10_reg));
 4885   match(RegP);
 4886   // match(iRegP);
 4887   match(iRegPNoSp);
 4888   op_cost(0);
 4889   format %{ %}
 4890   interface(REG_INTER);
 4891 %}
 4892 
 4893 // Long 64 bit Register R0 only
 4894 operand iRegL_R0()
 4895 %{
 4896   constraint(ALLOC_IN_RC(r0_reg));
 4897   match(RegL);
 4898   match(iRegLNoSp);
 4899   op_cost(0);
 4900   format %{ %}
 4901   interface(REG_INTER);
 4902 %}
 4903 
 4904 // Long 64 bit Register R11 only
 4905 operand iRegL_R11()
 4906 %{
 4907   constraint(ALLOC_IN_RC(r11_reg));
 4908   match(RegL);
 4909   match(iRegLNoSp);
 4910   op_cost(0);
 4911   format %{ %}
 4912   interface(REG_INTER);
 4913 %}
 4914 
 4915 // Register R0 only
 4916 operand iRegI_R0()
 4917 %{
 4918   constraint(ALLOC_IN_RC(int_r0_reg));
 4919   match(RegI);
 4920   match(iRegINoSp);
 4921   op_cost(0);
 4922   format %{ %}
 4923   interface(REG_INTER);
 4924 %}
 4925 
 4926 // Register R2 only
 4927 operand iRegI_R2()
 4928 %{
 4929   constraint(ALLOC_IN_RC(int_r2_reg));
 4930   match(RegI);
 4931   match(iRegINoSp);
 4932   op_cost(0);
 4933   format %{ %}
 4934   interface(REG_INTER);
 4935 %}
 4936 
 4937 // Register R3 only
 4938 operand iRegI_R3()
 4939 %{
 4940   constraint(ALLOC_IN_RC(int_r3_reg));
 4941   match(RegI);
 4942   match(iRegINoSp);
 4943   op_cost(0);
 4944   format %{ %}
 4945   interface(REG_INTER);
 4946 %}
 4947 
 4948 
 4949 // Register R4 only
 4950 operand iRegI_R4()
 4951 %{
 4952   constraint(ALLOC_IN_RC(int_r4_reg));
 4953   match(RegI);
 4954   match(iRegINoSp);
 4955   op_cost(0);
 4956   format %{ %}
 4957   interface(REG_INTER);
 4958 %}
 4959 
 4960 
 4961 // Pointer Register Operands
 4962 // Narrow Pointer Register
 4963 operand iRegN()
 4964 %{
 4965   constraint(ALLOC_IN_RC(any_reg32));
 4966   match(RegN);
 4967   match(iRegNNoSp);
 4968   op_cost(0);
 4969   format %{ %}
 4970   interface(REG_INTER);
 4971 %}
 4972 
 4973 // Integer 64 bit Register not Special
 4974 operand iRegNNoSp()
 4975 %{
 4976   constraint(ALLOC_IN_RC(no_special_reg32));
 4977   match(RegN);
 4978   op_cost(0);
 4979   format %{ %}
 4980   interface(REG_INTER);
 4981 %}
 4982 
 4983 // Float Register
 4984 // Float register operands
 4985 operand vRegF()
 4986 %{
 4987   constraint(ALLOC_IN_RC(float_reg));
 4988   match(RegF);
 4989 
 4990   op_cost(0);
 4991   format %{ %}
 4992   interface(REG_INTER);
 4993 %}
 4994 
 4995 // Double Register
 4996 // Double register operands
 4997 operand vRegD()
 4998 %{
 4999   constraint(ALLOC_IN_RC(double_reg));
 5000   match(RegD);
 5001 
 5002   op_cost(0);
 5003   format %{ %}
 5004   interface(REG_INTER);
 5005 %}
 5006 
 5007 // Generic vector class. This will be used for
 5008 // all vector operands, including NEON and SVE.
 5009 operand vReg()
 5010 %{
 5011   constraint(ALLOC_IN_RC(dynamic));
 5012   match(VecA);
 5013   match(VecD);
 5014   match(VecX);
 5015 
 5016   op_cost(0);
 5017   format %{ %}
 5018   interface(REG_INTER);
 5019 %}
 5020 
 5021 operand vReg_V10()
 5022 %{
 5023   constraint(ALLOC_IN_RC(v10_veca_reg));
 5024   match(vReg);
 5025 
 5026   op_cost(0);
 5027   format %{ %}
 5028   interface(REG_INTER);
 5029 %}
 5030 
 5031 operand vReg_V11()
 5032 %{
 5033   constraint(ALLOC_IN_RC(v11_veca_reg));
 5034   match(vReg);
 5035 
 5036   op_cost(0);
 5037   format %{ %}
 5038   interface(REG_INTER);
 5039 %}
 5040 
 5041 operand vReg_V12()
 5042 %{
 5043   constraint(ALLOC_IN_RC(v12_veca_reg));
 5044   match(vReg);
 5045 
 5046   op_cost(0);
 5047   format %{ %}
 5048   interface(REG_INTER);
 5049 %}
 5050 
 5051 operand vReg_V13()
 5052 %{
 5053   constraint(ALLOC_IN_RC(v13_veca_reg));
 5054   match(vReg);
 5055 
 5056   op_cost(0);
 5057   format %{ %}
 5058   interface(REG_INTER);
 5059 %}
 5060 
 5061 operand vReg_V17()
 5062 %{
 5063   constraint(ALLOC_IN_RC(v17_veca_reg));
 5064   match(vReg);
 5065 
 5066   op_cost(0);
 5067   format %{ %}
 5068   interface(REG_INTER);
 5069 %}
 5070 
 5071 operand vReg_V18()
 5072 %{
 5073   constraint(ALLOC_IN_RC(v18_veca_reg));
 5074   match(vReg);
 5075 
 5076   op_cost(0);
 5077   format %{ %}
 5078   interface(REG_INTER);
 5079 %}
 5080 
 5081 operand vReg_V23()
 5082 %{
 5083   constraint(ALLOC_IN_RC(v23_veca_reg));
 5084   match(vReg);
 5085 
 5086   op_cost(0);
 5087   format %{ %}
 5088   interface(REG_INTER);
 5089 %}
 5090 
 5091 operand vReg_V24()
 5092 %{
 5093   constraint(ALLOC_IN_RC(v24_veca_reg));
 5094   match(vReg);
 5095 
 5096   op_cost(0);
 5097   format %{ %}
 5098   interface(REG_INTER);
 5099 %}
 5100 
 5101 operand vecA()
 5102 %{
 5103   constraint(ALLOC_IN_RC(vectora_reg));
 5104   match(VecA);
 5105 
 5106   op_cost(0);
 5107   format %{ %}
 5108   interface(REG_INTER);
 5109 %}
 5110 
 5111 operand vecD()
 5112 %{
 5113   constraint(ALLOC_IN_RC(vectord_reg));
 5114   match(VecD);
 5115 
 5116   op_cost(0);
 5117   format %{ %}
 5118   interface(REG_INTER);
 5119 %}
 5120 
 5121 operand vecX()
 5122 %{
 5123   constraint(ALLOC_IN_RC(vectorx_reg));
 5124   match(VecX);
 5125 
 5126   op_cost(0);
 5127   format %{ %}
 5128   interface(REG_INTER);
 5129 %}
 5130 
 5131 operand vRegD_V0()
 5132 %{
 5133   constraint(ALLOC_IN_RC(v0_reg));
 5134   match(RegD);
 5135   op_cost(0);
 5136   format %{ %}
 5137   interface(REG_INTER);
 5138 %}
 5139 
 5140 operand vRegD_V1()
 5141 %{
 5142   constraint(ALLOC_IN_RC(v1_reg));
 5143   match(RegD);
 5144   op_cost(0);
 5145   format %{ %}
 5146   interface(REG_INTER);
 5147 %}
 5148 
 5149 operand vRegD_V2()
 5150 %{
 5151   constraint(ALLOC_IN_RC(v2_reg));
 5152   match(RegD);
 5153   op_cost(0);
 5154   format %{ %}
 5155   interface(REG_INTER);
 5156 %}
 5157 
 5158 operand vRegD_V3()
 5159 %{
 5160   constraint(ALLOC_IN_RC(v3_reg));
 5161   match(RegD);
 5162   op_cost(0);
 5163   format %{ %}
 5164   interface(REG_INTER);
 5165 %}
 5166 
 5167 operand vRegD_V4()
 5168 %{
 5169   constraint(ALLOC_IN_RC(v4_reg));
 5170   match(RegD);
 5171   op_cost(0);
 5172   format %{ %}
 5173   interface(REG_INTER);
 5174 %}
 5175 
 5176 operand vRegD_V5()
 5177 %{
 5178   constraint(ALLOC_IN_RC(v5_reg));
 5179   match(RegD);
 5180   op_cost(0);
 5181   format %{ %}
 5182   interface(REG_INTER);
 5183 %}
 5184 
 5185 operand vRegD_V6()
 5186 %{
 5187   constraint(ALLOC_IN_RC(v6_reg));
 5188   match(RegD);
 5189   op_cost(0);
 5190   format %{ %}
 5191   interface(REG_INTER);
 5192 %}
 5193 
 5194 operand vRegD_V7()
 5195 %{
 5196   constraint(ALLOC_IN_RC(v7_reg));
 5197   match(RegD);
 5198   op_cost(0);
 5199   format %{ %}
 5200   interface(REG_INTER);
 5201 %}
 5202 
 5203 operand vRegD_V12()
 5204 %{
 5205   constraint(ALLOC_IN_RC(v12_reg));
 5206   match(RegD);
 5207   op_cost(0);
 5208   format %{ %}
 5209   interface(REG_INTER);
 5210 %}
 5211 
 5212 operand vRegD_V13()
 5213 %{
 5214   constraint(ALLOC_IN_RC(v13_reg));
 5215   match(RegD);
 5216   op_cost(0);
 5217   format %{ %}
 5218   interface(REG_INTER);
 5219 %}
 5220 
 5221 operand pReg()
 5222 %{
 5223   constraint(ALLOC_IN_RC(pr_reg));
 5224   match(RegVectMask);
 5225   match(pRegGov);
 5226   op_cost(0);
 5227   format %{ %}
 5228   interface(REG_INTER);
 5229 %}
 5230 
 5231 operand pRegGov()
 5232 %{
 5233   constraint(ALLOC_IN_RC(gov_pr));
 5234   match(RegVectMask);
 5235   match(pReg);
 5236   op_cost(0);
 5237   format %{ %}
 5238   interface(REG_INTER);
 5239 %}
 5240 
 5241 operand pRegGov_P0()
 5242 %{
 5243   constraint(ALLOC_IN_RC(p0_reg));
 5244   match(RegVectMask);
 5245   op_cost(0);
 5246   format %{ %}
 5247   interface(REG_INTER);
 5248 %}
 5249 
 5250 operand pRegGov_P1()
 5251 %{
 5252   constraint(ALLOC_IN_RC(p1_reg));
 5253   match(RegVectMask);
 5254   op_cost(0);
 5255   format %{ %}
 5256   interface(REG_INTER);
 5257 %}
 5258 
 5259 // Flags register, used as output of signed compare instructions
 5260 
 5261 // note that on AArch64 we also use this register as the output for
 5262 // for floating point compare instructions (CmpF CmpD). this ensures
 5263 // that ordered inequality tests use GT, GE, LT or LE none of which
 5264 // pass through cases where the result is unordered i.e. one or both
 5265 // inputs to the compare is a NaN. this means that the ideal code can
 5266 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5267 // (where the comparison should always fail). EQ and NE tests are
 5268 // always generated in ideal code so that unordered folds into the NE
 5269 // case, matching the behaviour of AArch64 NE.
 5270 //
 5271 // This differs from x86 where the outputs of FP compares use a
 5272 // special FP flags registers and where compares based on this
 5273 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5274 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5275 // to explicitly handle the unordered case in branches. x86 also has
 5276 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5277 
 5278 operand rFlagsReg()
 5279 %{
 5280   constraint(ALLOC_IN_RC(int_flags));
 5281   match(RegFlags);
 5282 
 5283   op_cost(0);
 5284   format %{ "RFLAGS" %}
 5285   interface(REG_INTER);
 5286 %}
 5287 
 5288 // Flags register, used as output of unsigned compare instructions
 5289 operand rFlagsRegU()
 5290 %{
 5291   constraint(ALLOC_IN_RC(int_flags));
 5292   match(RegFlags);
 5293 
 5294   op_cost(0);
 5295   format %{ "RFLAGSU" %}
 5296   interface(REG_INTER);
 5297 %}
 5298 
 5299 // Special Registers
 5300 
 5301 // Method Register
 5302 operand inline_cache_RegP(iRegP reg)
 5303 %{
 5304   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5305   match(reg);
 5306   match(iRegPNoSp);
 5307   op_cost(0);
 5308   format %{ %}
 5309   interface(REG_INTER);
 5310 %}
 5311 
 5312 // Thread Register
 5313 operand thread_RegP(iRegP reg)
 5314 %{
 5315   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5316   match(reg);
 5317   op_cost(0);
 5318   format %{ %}
 5319   interface(REG_INTER);
 5320 %}
 5321 
 5322 //----------Memory Operands----------------------------------------------------
 5323 
 5324 operand indirect(iRegP reg)
 5325 %{
 5326   constraint(ALLOC_IN_RC(ptr_reg));
 5327   match(reg);
 5328   op_cost(0);
 5329   format %{ "[$reg]" %}
 5330   interface(MEMORY_INTER) %{
 5331     base($reg);
 5332     index(0xffffffff);
 5333     scale(0x0);
 5334     disp(0x0);
 5335   %}
 5336 %}
 5337 
 5338 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5339 %{
 5340   constraint(ALLOC_IN_RC(ptr_reg));
 5341   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5342   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5343   op_cost(0);
 5344   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5345   interface(MEMORY_INTER) %{
 5346     base($reg);
 5347     index($ireg);
 5348     scale($scale);
 5349     disp(0x0);
 5350   %}
 5351 %}
 5352 
 5353 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5354 %{
 5355   constraint(ALLOC_IN_RC(ptr_reg));
 5356   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5357   match(AddP reg (LShiftL lreg scale));
 5358   op_cost(0);
 5359   format %{ "$reg, $lreg lsl($scale)" %}
 5360   interface(MEMORY_INTER) %{
 5361     base($reg);
 5362     index($lreg);
 5363     scale($scale);
 5364     disp(0x0);
 5365   %}
 5366 %}
 5367 
 5368 operand indIndexI2L(iRegP reg, iRegI ireg)
 5369 %{
 5370   constraint(ALLOC_IN_RC(ptr_reg));
 5371   match(AddP reg (ConvI2L ireg));
 5372   op_cost(0);
 5373   format %{ "$reg, $ireg, 0, I2L" %}
 5374   interface(MEMORY_INTER) %{
 5375     base($reg);
 5376     index($ireg);
 5377     scale(0x0);
 5378     disp(0x0);
 5379   %}
 5380 %}
 5381 
 5382 operand indIndex(iRegP reg, iRegL lreg)
 5383 %{
 5384   constraint(ALLOC_IN_RC(ptr_reg));
 5385   match(AddP reg lreg);
 5386   op_cost(0);
 5387   format %{ "$reg, $lreg" %}
 5388   interface(MEMORY_INTER) %{
 5389     base($reg);
 5390     index($lreg);
 5391     scale(0x0);
 5392     disp(0x0);
 5393   %}
 5394 %}
 5395 
 5396 operand indOffI1(iRegP reg, immIOffset1 off)
 5397 %{
 5398   constraint(ALLOC_IN_RC(ptr_reg));
 5399   match(AddP reg off);
 5400   op_cost(0);
 5401   format %{ "[$reg, $off]" %}
 5402   interface(MEMORY_INTER) %{
 5403     base($reg);
 5404     index(0xffffffff);
 5405     scale(0x0);
 5406     disp($off);
 5407   %}
 5408 %}
 5409 
 5410 operand indOffI2(iRegP reg, immIOffset2 off)
 5411 %{
 5412   constraint(ALLOC_IN_RC(ptr_reg));
 5413   match(AddP reg off);
 5414   op_cost(0);
 5415   format %{ "[$reg, $off]" %}
 5416   interface(MEMORY_INTER) %{
 5417     base($reg);
 5418     index(0xffffffff);
 5419     scale(0x0);
 5420     disp($off);
 5421   %}
 5422 %}
 5423 
 5424 operand indOffI4(iRegP reg, immIOffset4 off)
 5425 %{
 5426   constraint(ALLOC_IN_RC(ptr_reg));
 5427   match(AddP reg off);
 5428   op_cost(0);
 5429   format %{ "[$reg, $off]" %}
 5430   interface(MEMORY_INTER) %{
 5431     base($reg);
 5432     index(0xffffffff);
 5433     scale(0x0);
 5434     disp($off);
 5435   %}
 5436 %}
 5437 
 5438 operand indOffI8(iRegP reg, immIOffset8 off)
 5439 %{
 5440   constraint(ALLOC_IN_RC(ptr_reg));
 5441   match(AddP reg off);
 5442   op_cost(0);
 5443   format %{ "[$reg, $off]" %}
 5444   interface(MEMORY_INTER) %{
 5445     base($reg);
 5446     index(0xffffffff);
 5447     scale(0x0);
 5448     disp($off);
 5449   %}
 5450 %}
 5451 
 5452 operand indOffI16(iRegP reg, immIOffset16 off)
 5453 %{
 5454   constraint(ALLOC_IN_RC(ptr_reg));
 5455   match(AddP reg off);
 5456   op_cost(0);
 5457   format %{ "[$reg, $off]" %}
 5458   interface(MEMORY_INTER) %{
 5459     base($reg);
 5460     index(0xffffffff);
 5461     scale(0x0);
 5462     disp($off);
 5463   %}
 5464 %}
 5465 
 5466 operand indOffL1(iRegP reg, immLoffset1 off)
 5467 %{
 5468   constraint(ALLOC_IN_RC(ptr_reg));
 5469   match(AddP reg off);
 5470   op_cost(0);
 5471   format %{ "[$reg, $off]" %}
 5472   interface(MEMORY_INTER) %{
 5473     base($reg);
 5474     index(0xffffffff);
 5475     scale(0x0);
 5476     disp($off);
 5477   %}
 5478 %}
 5479 
 5480 operand indOffL2(iRegP reg, immLoffset2 off)
 5481 %{
 5482   constraint(ALLOC_IN_RC(ptr_reg));
 5483   match(AddP reg off);
 5484   op_cost(0);
 5485   format %{ "[$reg, $off]" %}
 5486   interface(MEMORY_INTER) %{
 5487     base($reg);
 5488     index(0xffffffff);
 5489     scale(0x0);
 5490     disp($off);
 5491   %}
 5492 %}
 5493 
 5494 operand indOffL4(iRegP reg, immLoffset4 off)
 5495 %{
 5496   constraint(ALLOC_IN_RC(ptr_reg));
 5497   match(AddP reg off);
 5498   op_cost(0);
 5499   format %{ "[$reg, $off]" %}
 5500   interface(MEMORY_INTER) %{
 5501     base($reg);
 5502     index(0xffffffff);
 5503     scale(0x0);
 5504     disp($off);
 5505   %}
 5506 %}
 5507 
 5508 operand indOffL8(iRegP reg, immLoffset8 off)
 5509 %{
 5510   constraint(ALLOC_IN_RC(ptr_reg));
 5511   match(AddP reg off);
 5512   op_cost(0);
 5513   format %{ "[$reg, $off]" %}
 5514   interface(MEMORY_INTER) %{
 5515     base($reg);
 5516     index(0xffffffff);
 5517     scale(0x0);
 5518     disp($off);
 5519   %}
 5520 %}
 5521 
 5522 operand indOffL16(iRegP reg, immLoffset16 off)
 5523 %{
 5524   constraint(ALLOC_IN_RC(ptr_reg));
 5525   match(AddP reg off);
 5526   op_cost(0);
 5527   format %{ "[$reg, $off]" %}
 5528   interface(MEMORY_INTER) %{
 5529     base($reg);
 5530     index(0xffffffff);
 5531     scale(0x0);
 5532     disp($off);
 5533   %}
 5534 %}
 5535 
 5536 operand indirectX2P(iRegL reg)
 5537 %{
 5538   constraint(ALLOC_IN_RC(ptr_reg));
 5539   match(CastX2P reg);
 5540   op_cost(0);
 5541   format %{ "[$reg]\t# long -> ptr" %}
 5542   interface(MEMORY_INTER) %{
 5543     base($reg);
 5544     index(0xffffffff);
 5545     scale(0x0);
 5546     disp(0x0);
 5547   %}
 5548 %}
 5549 
 5550 operand indOffX2P(iRegL reg, immLOffset off)
 5551 %{
 5552   constraint(ALLOC_IN_RC(ptr_reg));
 5553   match(AddP (CastX2P reg) off);
 5554   op_cost(0);
 5555   format %{ "[$reg, $off]\t# long -> ptr" %}
 5556   interface(MEMORY_INTER) %{
 5557     base($reg);
 5558     index(0xffffffff);
 5559     scale(0x0);
 5560     disp($off);
 5561   %}
 5562 %}
 5563 
 5564 operand indirectN(iRegN reg)
 5565 %{
 5566   predicate(CompressedOops::shift() == 0);
 5567   constraint(ALLOC_IN_RC(ptr_reg));
 5568   match(DecodeN reg);
 5569   op_cost(0);
 5570   format %{ "[$reg]\t# narrow" %}
 5571   interface(MEMORY_INTER) %{
 5572     base($reg);
 5573     index(0xffffffff);
 5574     scale(0x0);
 5575     disp(0x0);
 5576   %}
 5577 %}
 5578 
 5579 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5580 %{
 5581   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5582   constraint(ALLOC_IN_RC(ptr_reg));
 5583   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5584   op_cost(0);
 5585   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5586   interface(MEMORY_INTER) %{
 5587     base($reg);
 5588     index($ireg);
 5589     scale($scale);
 5590     disp(0x0);
 5591   %}
 5592 %}
 5593 
 5594 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5595 %{
 5596   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5597   constraint(ALLOC_IN_RC(ptr_reg));
 5598   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5599   op_cost(0);
 5600   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5601   interface(MEMORY_INTER) %{
 5602     base($reg);
 5603     index($lreg);
 5604     scale($scale);
 5605     disp(0x0);
 5606   %}
 5607 %}
 5608 
 5609 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5610 %{
 5611   predicate(CompressedOops::shift() == 0);
 5612   constraint(ALLOC_IN_RC(ptr_reg));
 5613   match(AddP (DecodeN reg) (ConvI2L ireg));
 5614   op_cost(0);
 5615   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5616   interface(MEMORY_INTER) %{
 5617     base($reg);
 5618     index($ireg);
 5619     scale(0x0);
 5620     disp(0x0);
 5621   %}
 5622 %}
 5623 
 5624 operand indIndexN(iRegN reg, iRegL lreg)
 5625 %{
 5626   predicate(CompressedOops::shift() == 0);
 5627   constraint(ALLOC_IN_RC(ptr_reg));
 5628   match(AddP (DecodeN reg) lreg);
 5629   op_cost(0);
 5630   format %{ "$reg, $lreg\t# narrow" %}
 5631   interface(MEMORY_INTER) %{
 5632     base($reg);
 5633     index($lreg);
 5634     scale(0x0);
 5635     disp(0x0);
 5636   %}
 5637 %}
 5638 
 5639 operand indOffIN(iRegN reg, immIOffset off)
 5640 %{
 5641   predicate(CompressedOops::shift() == 0);
 5642   constraint(ALLOC_IN_RC(ptr_reg));
 5643   match(AddP (DecodeN reg) off);
 5644   op_cost(0);
 5645   format %{ "[$reg, $off]\t# narrow" %}
 5646   interface(MEMORY_INTER) %{
 5647     base($reg);
 5648     index(0xffffffff);
 5649     scale(0x0);
 5650     disp($off);
 5651   %}
 5652 %}
 5653 
 5654 operand indOffLN(iRegN reg, immLOffset off)
 5655 %{
 5656   predicate(CompressedOops::shift() == 0);
 5657   constraint(ALLOC_IN_RC(ptr_reg));
 5658   match(AddP (DecodeN reg) off);
 5659   op_cost(0);
 5660   format %{ "[$reg, $off]\t# narrow" %}
 5661   interface(MEMORY_INTER) %{
 5662     base($reg);
 5663     index(0xffffffff);
 5664     scale(0x0);
 5665     disp($off);
 5666   %}
 5667 %}
 5668 
 5669 
 5670 //----------Special Memory Operands--------------------------------------------
 5671 // Stack Slot Operand - This operand is used for loading and storing temporary
 5672 //                      values on the stack where a match requires a value to
 5673 //                      flow through memory.
 5674 operand stackSlotP(sRegP reg)
 5675 %{
 5676   constraint(ALLOC_IN_RC(stack_slots));
 5677   op_cost(100);
 5678   // No match rule because this operand is only generated in matching
 5679   // match(RegP);
 5680   format %{ "[$reg]" %}
 5681   interface(MEMORY_INTER) %{
 5682     base(0x1e);  // RSP
 5683     index(0x0);  // No Index
 5684     scale(0x0);  // No Scale
 5685     disp($reg);  // Stack Offset
 5686   %}
 5687 %}
 5688 
 5689 operand stackSlotI(sRegI reg)
 5690 %{
 5691   constraint(ALLOC_IN_RC(stack_slots));
 5692   // No match rule because this operand is only generated in matching
 5693   // match(RegI);
 5694   format %{ "[$reg]" %}
 5695   interface(MEMORY_INTER) %{
 5696     base(0x1e);  // RSP
 5697     index(0x0);  // No Index
 5698     scale(0x0);  // No Scale
 5699     disp($reg);  // Stack Offset
 5700   %}
 5701 %}
 5702 
 5703 operand stackSlotF(sRegF reg)
 5704 %{
 5705   constraint(ALLOC_IN_RC(stack_slots));
 5706   // No match rule because this operand is only generated in matching
 5707   // match(RegF);
 5708   format %{ "[$reg]" %}
 5709   interface(MEMORY_INTER) %{
 5710     base(0x1e);  // RSP
 5711     index(0x0);  // No Index
 5712     scale(0x0);  // No Scale
 5713     disp($reg);  // Stack Offset
 5714   %}
 5715 %}
 5716 
 5717 operand stackSlotD(sRegD reg)
 5718 %{
 5719   constraint(ALLOC_IN_RC(stack_slots));
 5720   // No match rule because this operand is only generated in matching
 5721   // match(RegD);
 5722   format %{ "[$reg]" %}
 5723   interface(MEMORY_INTER) %{
 5724     base(0x1e);  // RSP
 5725     index(0x0);  // No Index
 5726     scale(0x0);  // No Scale
 5727     disp($reg);  // Stack Offset
 5728   %}
 5729 %}
 5730 
 5731 operand stackSlotL(sRegL reg)
 5732 %{
 5733   constraint(ALLOC_IN_RC(stack_slots));
 5734   // No match rule because this operand is only generated in matching
 5735   // match(RegL);
 5736   format %{ "[$reg]" %}
 5737   interface(MEMORY_INTER) %{
 5738     base(0x1e);  // RSP
 5739     index(0x0);  // No Index
 5740     scale(0x0);  // No Scale
 5741     disp($reg);  // Stack Offset
 5742   %}
 5743 %}
 5744 
 5745 // Operands for expressing Control Flow
 5746 // NOTE: Label is a predefined operand which should not be redefined in
 5747 //       the AD file. It is generically handled within the ADLC.
 5748 
 5749 //----------Conditional Branch Operands----------------------------------------
 5750 // Comparison Op  - This is the operation of the comparison, and is limited to
 5751 //                  the following set of codes:
 5752 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 5753 //
 5754 // Other attributes of the comparison, such as unsignedness, are specified
 5755 // by the comparison instruction that sets a condition code flags register.
 5756 // That result is represented by a flags operand whose subtype is appropriate
 5757 // to the unsignedness (etc.) of the comparison.
 5758 //
 5759 // Later, the instruction which matches both the Comparison Op (a Bool) and
 5760 // the flags (produced by the Cmp) specifies the coding of the comparison op
 5761 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 5762 
 5763 // used for signed integral comparisons and fp comparisons
 5764 
 5765 operand cmpOp()
 5766 %{
 5767   match(Bool);
 5768 
 5769   format %{ "" %}
 5770   interface(COND_INTER) %{
 5771     equal(0x0, "eq");
 5772     not_equal(0x1, "ne");
 5773     less(0xb, "lt");
 5774     greater_equal(0xa, "ge");
 5775     less_equal(0xd, "le");
 5776     greater(0xc, "gt");
 5777     overflow(0x6, "vs");
 5778     no_overflow(0x7, "vc");
 5779   %}
 5780 %}
 5781 
 5782 // used for unsigned integral comparisons
 5783 
 5784 operand cmpOpU()
 5785 %{
 5786   match(Bool);
 5787 
 5788   format %{ "" %}
 5789   interface(COND_INTER) %{
 5790     equal(0x0, "eq");
 5791     not_equal(0x1, "ne");
 5792     less(0x3, "lo");
 5793     greater_equal(0x2, "hs");
 5794     less_equal(0x9, "ls");
 5795     greater(0x8, "hi");
 5796     overflow(0x6, "vs");
 5797     no_overflow(0x7, "vc");
 5798   %}
 5799 %}
 5800 
 5801 // used for certain integral comparisons which can be
 5802 // converted to cbxx or tbxx instructions
 5803 
 5804 operand cmpOpEqNe()
 5805 %{
 5806   match(Bool);
 5807   op_cost(0);
 5808   predicate(n->as_Bool()->_test._test == BoolTest::ne
 5809             || n->as_Bool()->_test._test == BoolTest::eq);
 5810 
 5811   format %{ "" %}
 5812   interface(COND_INTER) %{
 5813     equal(0x0, "eq");
 5814     not_equal(0x1, "ne");
 5815     less(0xb, "lt");
 5816     greater_equal(0xa, "ge");
 5817     less_equal(0xd, "le");
 5818     greater(0xc, "gt");
 5819     overflow(0x6, "vs");
 5820     no_overflow(0x7, "vc");
 5821   %}
 5822 %}
 5823 
 5824 // used for certain integral comparisons which can be
 5825 // converted to cbxx or tbxx instructions
 5826 
 5827 operand cmpOpLtGe()
 5828 %{
 5829   match(Bool);
 5830   op_cost(0);
 5831 
 5832   predicate(n->as_Bool()->_test._test == BoolTest::lt
 5833             || n->as_Bool()->_test._test == BoolTest::ge);
 5834 
 5835   format %{ "" %}
 5836   interface(COND_INTER) %{
 5837     equal(0x0, "eq");
 5838     not_equal(0x1, "ne");
 5839     less(0xb, "lt");
 5840     greater_equal(0xa, "ge");
 5841     less_equal(0xd, "le");
 5842     greater(0xc, "gt");
 5843     overflow(0x6, "vs");
 5844     no_overflow(0x7, "vc");
 5845   %}
 5846 %}
 5847 
 5848 // used for certain unsigned integral comparisons which can be
 5849 // converted to cbxx or tbxx instructions
 5850 
 5851 operand cmpOpUEqNeLeGt()
 5852 %{
 5853   match(Bool);
 5854   op_cost(0);
 5855 
 5856   predicate(n->as_Bool()->_test._test == BoolTest::eq ||
 5857             n->as_Bool()->_test._test == BoolTest::ne ||
 5858             n->as_Bool()->_test._test == BoolTest::le ||
 5859             n->as_Bool()->_test._test == BoolTest::gt);
 5860 
 5861   format %{ "" %}
 5862   interface(COND_INTER) %{
 5863     equal(0x0, "eq");
 5864     not_equal(0x1, "ne");
 5865     less(0x3, "lo");
 5866     greater_equal(0x2, "hs");
 5867     less_equal(0x9, "ls");
 5868     greater(0x8, "hi");
 5869     overflow(0x6, "vs");
 5870     no_overflow(0x7, "vc");
 5871   %}
 5872 %}
 5873 
 5874 // Special operand allowing long args to int ops to be truncated for free
 5875 
 5876 operand iRegL2I(iRegL reg) %{
 5877 
 5878   op_cost(0);
 5879 
 5880   match(ConvL2I reg);
 5881 
 5882   format %{ "l2i($reg)" %}
 5883 
 5884   interface(REG_INTER)
 5885 %}
 5886 
 5887 operand iRegL2P(iRegL reg) %{
 5888 
 5889   op_cost(0);
 5890 
 5891   match(CastX2P reg);
 5892 
 5893   format %{ "l2p($reg)" %}
 5894 
 5895   interface(REG_INTER)
 5896 %}
 5897 
 5898 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 5899 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 5900 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 5901 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 5902 
 5903 //----------OPERAND CLASSES----------------------------------------------------
 5904 // Operand Classes are groups of operands that are used as to simplify
 5905 // instruction definitions by not requiring the AD writer to specify
 5906 // separate instructions for every form of operand when the
 5907 // instruction accepts multiple operand types with the same basic
 5908 // encoding and format. The classic case of this is memory operands.
 5909 
 5910 // memory is used to define read/write location for load/store
 5911 // instruction defs. we can turn a memory op into an Address
 5912 
 5913 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 5914                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
 5915 
 5916 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 5917                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
 5918 
 5919 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 5920                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
 5921 
 5922 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 5923                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
 5924 
 5925 // All of the memory operands. For the pipeline description.
 5926 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 5927                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 5928                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
 5929 
 5930 
 5931 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 5932 // operations. it allows the src to be either an iRegI or a (ConvL2I
 5933 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 5934 // can be elided because the 32-bit instruction will just employ the
 5935 // lower 32 bits anyway.
 5936 //
 5937 // n.b. this does not elide all L2I conversions. if the truncated
 5938 // value is consumed by more than one operation then the ConvL2I
 5939 // cannot be bundled into the consuming nodes so an l2i gets planted
 5940 // (actually a movw $dst $src) and the downstream instructions consume
 5941 // the result of the l2i as an iRegI input. That's a shame since the
 5942 // movw is actually redundant but its not too costly.
 5943 
 5944 opclass iRegIorL2I(iRegI, iRegL2I);
 5945 opclass iRegPorL2P(iRegP, iRegL2P);
 5946 
 5947 //----------PIPELINE-----------------------------------------------------------
 5948 // Rules which define the behavior of the target architectures pipeline.
 5949 
 5950 // For specific pipelines, eg A53, define the stages of that pipeline
 5951 //pipe_desc(ISS, EX1, EX2, WR);
 5952 #define ISS S0
 5953 #define EX1 S1
 5954 #define EX2 S2
 5955 #define WR  S3
 5956 
 5957 // Integer ALU reg operation
 5958 pipeline %{
 5959 
 5960 attributes %{
 5961   // ARM instructions are of fixed length
 5962   fixed_size_instructions;        // Fixed size instructions TODO does
 5963   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 5964   // ARM instructions come in 32-bit word units
 5965   instruction_unit_size = 4;         // An instruction is 4 bytes long
 5966   instruction_fetch_unit_size = 64;  // The processor fetches one line
 5967   instruction_fetch_units = 1;       // of 64 bytes
 5968 %}
 5969 
 5970 // We don't use an actual pipeline model so don't care about resources
 5971 // or description. we do use pipeline classes to introduce fixed
 5972 // latencies
 5973 
 5974 //----------RESOURCES----------------------------------------------------------
 5975 // Resources are the functional units available to the machine
 5976 
 5977 resources( INS0, INS1, INS01 = INS0 | INS1,
 5978            ALU0, ALU1, ALU = ALU0 | ALU1,
 5979            MAC,
 5980            DIV,
 5981            BRANCH,
 5982            LDST,
 5983            NEON_FP);
 5984 
 5985 //----------PIPELINE DESCRIPTION-----------------------------------------------
 5986 // Pipeline Description specifies the stages in the machine's pipeline
 5987 
 5988 // Define the pipeline as a generic 6 stage pipeline
 5989 pipe_desc(S0, S1, S2, S3, S4, S5);
 5990 
 5991 //----------PIPELINE CLASSES---------------------------------------------------
 5992 // Pipeline Classes describe the stages in which input and output are
 5993 // referenced by the hardware pipeline.
 5994 
 5995 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 5996 %{
 5997   single_instruction;
 5998   src1   : S1(read);
 5999   src2   : S2(read);
 6000   dst    : S5(write);
 6001   INS01  : ISS;
 6002   NEON_FP : S5;
 6003 %}
 6004 
 6005 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6006 %{
 6007   single_instruction;
 6008   src1   : S1(read);
 6009   src2   : S2(read);
 6010   dst    : S5(write);
 6011   INS01  : ISS;
 6012   NEON_FP : S5;
 6013 %}
 6014 
 6015 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6016 %{
 6017   single_instruction;
 6018   src    : S1(read);
 6019   dst    : S5(write);
 6020   INS01  : ISS;
 6021   NEON_FP : S5;
 6022 %}
 6023 
 6024 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6025 %{
 6026   single_instruction;
 6027   src    : S1(read);
 6028   dst    : S5(write);
 6029   INS01  : ISS;
 6030   NEON_FP : S5;
 6031 %}
 6032 
 6033 pipe_class fp_d2f(vRegF dst, vRegD src)
 6034 %{
 6035   single_instruction;
 6036   src    : S1(read);
 6037   dst    : S5(write);
 6038   INS01  : ISS;
 6039   NEON_FP : S5;
 6040 %}
 6041 
 6042 pipe_class fp_f2d(vRegD dst, vRegF src)
 6043 %{
 6044   single_instruction;
 6045   src    : S1(read);
 6046   dst    : S5(write);
 6047   INS01  : ISS;
 6048   NEON_FP : S5;
 6049 %}
 6050 
 6051 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6052 %{
 6053   single_instruction;
 6054   src    : S1(read);
 6055   dst    : S5(write);
 6056   INS01  : ISS;
 6057   NEON_FP : S5;
 6058 %}
 6059 
 6060 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6061 %{
 6062   single_instruction;
 6063   src    : S1(read);
 6064   dst    : S5(write);
 6065   INS01  : ISS;
 6066   NEON_FP : S5;
 6067 %}
 6068 
 6069 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6070 %{
 6071   single_instruction;
 6072   src    : S1(read);
 6073   dst    : S5(write);
 6074   INS01  : ISS;
 6075   NEON_FP : S5;
 6076 %}
 6077 
 6078 pipe_class fp_l2f(vRegF dst, iRegL src)
 6079 %{
 6080   single_instruction;
 6081   src    : S1(read);
 6082   dst    : S5(write);
 6083   INS01  : ISS;
 6084   NEON_FP : S5;
 6085 %}
 6086 
 6087 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6088 %{
 6089   single_instruction;
 6090   src    : S1(read);
 6091   dst    : S5(write);
 6092   INS01  : ISS;
 6093   NEON_FP : S5;
 6094 %}
 6095 
 6096 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6097 %{
 6098   single_instruction;
 6099   src    : S1(read);
 6100   dst    : S5(write);
 6101   INS01  : ISS;
 6102   NEON_FP : S5;
 6103 %}
 6104 
 6105 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6106 %{
 6107   single_instruction;
 6108   src    : S1(read);
 6109   dst    : S5(write);
 6110   INS01  : ISS;
 6111   NEON_FP : S5;
 6112 %}
 6113 
 6114 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6115 %{
 6116   single_instruction;
 6117   src    : S1(read);
 6118   dst    : S5(write);
 6119   INS01  : ISS;
 6120   NEON_FP : S5;
 6121 %}
 6122 
 6123 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6124 %{
 6125   single_instruction;
 6126   src1   : S1(read);
 6127   src2   : S2(read);
 6128   dst    : S5(write);
 6129   INS0   : ISS;
 6130   NEON_FP : S5;
 6131 %}
 6132 
 6133 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6134 %{
 6135   single_instruction;
 6136   src1   : S1(read);
 6137   src2   : S2(read);
 6138   dst    : S5(write);
 6139   INS0   : ISS;
 6140   NEON_FP : S5;
 6141 %}
 6142 
 6143 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6144 %{
 6145   single_instruction;
 6146   cr     : S1(read);
 6147   src1   : S1(read);
 6148   src2   : S1(read);
 6149   dst    : S3(write);
 6150   INS01  : ISS;
 6151   NEON_FP : S3;
 6152 %}
 6153 
 6154 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6155 %{
 6156   single_instruction;
 6157   cr     : S1(read);
 6158   src1   : S1(read);
 6159   src2   : S1(read);
 6160   dst    : S3(write);
 6161   INS01  : ISS;
 6162   NEON_FP : S3;
 6163 %}
 6164 
 6165 pipe_class fp_imm_s(vRegF dst)
 6166 %{
 6167   single_instruction;
 6168   dst    : S3(write);
 6169   INS01  : ISS;
 6170   NEON_FP : S3;
 6171 %}
 6172 
 6173 pipe_class fp_imm_d(vRegD dst)
 6174 %{
 6175   single_instruction;
 6176   dst    : S3(write);
 6177   INS01  : ISS;
 6178   NEON_FP : S3;
 6179 %}
 6180 
 6181 pipe_class fp_load_constant_s(vRegF dst)
 6182 %{
 6183   single_instruction;
 6184   dst    : S4(write);
 6185   INS01  : ISS;
 6186   NEON_FP : S4;
 6187 %}
 6188 
 6189 pipe_class fp_load_constant_d(vRegD dst)
 6190 %{
 6191   single_instruction;
 6192   dst    : S4(write);
 6193   INS01  : ISS;
 6194   NEON_FP : S4;
 6195 %}
 6196 
 6197 //------- Integer ALU operations --------------------------
 6198 
 6199 // Integer ALU reg-reg operation
 6200 // Operands needed in EX1, result generated in EX2
 6201 // Eg.  ADD     x0, x1, x2
 6202 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6203 %{
 6204   single_instruction;
 6205   dst    : EX2(write);
 6206   src1   : EX1(read);
 6207   src2   : EX1(read);
 6208   INS01  : ISS; // Dual issue as instruction 0 or 1
 6209   ALU    : EX2;
 6210 %}
 6211 
 6212 // Integer ALU reg-reg operation with constant shift
 6213 // Shifted register must be available in LATE_ISS instead of EX1
 6214 // Eg.  ADD     x0, x1, x2, LSL #2
 6215 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6216 %{
 6217   single_instruction;
 6218   dst    : EX2(write);
 6219   src1   : EX1(read);
 6220   src2   : ISS(read);
 6221   INS01  : ISS;
 6222   ALU    : EX2;
 6223 %}
 6224 
 6225 // Integer ALU reg operation with constant shift
 6226 // Eg.  LSL     x0, x1, #shift
 6227 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6228 %{
 6229   single_instruction;
 6230   dst    : EX2(write);
 6231   src1   : ISS(read);
 6232   INS01  : ISS;
 6233   ALU    : EX2;
 6234 %}
 6235 
 6236 // Integer ALU reg-reg operation with variable shift
 6237 // Both operands must be available in LATE_ISS instead of EX1
 6238 // Result is available in EX1 instead of EX2
 6239 // Eg.  LSLV    x0, x1, x2
 6240 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6241 %{
 6242   single_instruction;
 6243   dst    : EX1(write);
 6244   src1   : ISS(read);
 6245   src2   : ISS(read);
 6246   INS01  : ISS;
 6247   ALU    : EX1;
 6248 %}
 6249 
 6250 // Integer ALU reg-reg operation with extract
 6251 // As for _vshift above, but result generated in EX2
 6252 // Eg.  EXTR    x0, x1, x2, #N
 6253 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6254 %{
 6255   single_instruction;
 6256   dst    : EX2(write);
 6257   src1   : ISS(read);
 6258   src2   : ISS(read);
 6259   INS1   : ISS; // Can only dual issue as Instruction 1
 6260   ALU    : EX1;
 6261 %}
 6262 
 6263 // Integer ALU reg operation
 6264 // Eg.  NEG     x0, x1
 6265 pipe_class ialu_reg(iRegI dst, iRegI src)
 6266 %{
 6267   single_instruction;
 6268   dst    : EX2(write);
 6269   src    : EX1(read);
 6270   INS01  : ISS;
 6271   ALU    : EX2;
 6272 %}
 6273 
 6274 // Integer ALU reg mmediate operation
 6275 // Eg.  ADD     x0, x1, #N
 6276 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6277 %{
 6278   single_instruction;
 6279   dst    : EX2(write);
 6280   src1   : EX1(read);
 6281   INS01  : ISS;
 6282   ALU    : EX2;
 6283 %}
 6284 
 6285 // Integer ALU immediate operation (no source operands)
 6286 // Eg.  MOV     x0, #N
 6287 pipe_class ialu_imm(iRegI dst)
 6288 %{
 6289   single_instruction;
 6290   dst    : EX1(write);
 6291   INS01  : ISS;
 6292   ALU    : EX1;
 6293 %}
 6294 
 6295 //------- Compare operation -------------------------------
 6296 
 6297 // Compare reg-reg
 6298 // Eg.  CMP     x0, x1
 6299 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6300 %{
 6301   single_instruction;
 6302 //  fixed_latency(16);
 6303   cr     : EX2(write);
 6304   op1    : EX1(read);
 6305   op2    : EX1(read);
 6306   INS01  : ISS;
 6307   ALU    : EX2;
 6308 %}
 6309 
 6310 // Compare reg-reg
 6311 // Eg.  CMP     x0, #N
 6312 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6313 %{
 6314   single_instruction;
 6315 //  fixed_latency(16);
 6316   cr     : EX2(write);
 6317   op1    : EX1(read);
 6318   INS01  : ISS;
 6319   ALU    : EX2;
 6320 %}
 6321 
 6322 //------- Conditional instructions ------------------------
 6323 
 6324 // Conditional no operands
 6325 // Eg.  CSINC   x0, zr, zr, <cond>
 6326 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6327 %{
 6328   single_instruction;
 6329   cr     : EX1(read);
 6330   dst    : EX2(write);
 6331   INS01  : ISS;
 6332   ALU    : EX2;
 6333 %}
 6334 
 6335 // Conditional 2 operand
 6336 // EG.  CSEL    X0, X1, X2, <cond>
 6337 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6338 %{
 6339   single_instruction;
 6340   cr     : EX1(read);
 6341   src1   : EX1(read);
 6342   src2   : EX1(read);
 6343   dst    : EX2(write);
 6344   INS01  : ISS;
 6345   ALU    : EX2;
 6346 %}
 6347 
 6348 // Conditional 2 operand
 6349 // EG.  CSEL    X0, X1, X2, <cond>
 6350 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6351 %{
 6352   single_instruction;
 6353   cr     : EX1(read);
 6354   src    : EX1(read);
 6355   dst    : EX2(write);
 6356   INS01  : ISS;
 6357   ALU    : EX2;
 6358 %}
 6359 
 6360 //------- Multiply pipeline operations --------------------
 6361 
 6362 // Multiply reg-reg
 6363 // Eg.  MUL     w0, w1, w2
 6364 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6365 %{
 6366   single_instruction;
 6367   dst    : WR(write);
 6368   src1   : ISS(read);
 6369   src2   : ISS(read);
 6370   INS01  : ISS;
 6371   MAC    : WR;
 6372 %}
 6373 
 6374 // Multiply accumulate
 6375 // Eg.  MADD    w0, w1, w2, w3
 6376 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6377 %{
 6378   single_instruction;
 6379   dst    : WR(write);
 6380   src1   : ISS(read);
 6381   src2   : ISS(read);
 6382   src3   : ISS(read);
 6383   INS01  : ISS;
 6384   MAC    : WR;
 6385 %}
 6386 
 6387 // Eg.  MUL     w0, w1, w2
 6388 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6389 %{
 6390   single_instruction;
 6391   fixed_latency(3); // Maximum latency for 64 bit mul
 6392   dst    : WR(write);
 6393   src1   : ISS(read);
 6394   src2   : ISS(read);
 6395   INS01  : ISS;
 6396   MAC    : WR;
 6397 %}
 6398 
 6399 // Multiply accumulate
 6400 // Eg.  MADD    w0, w1, w2, w3
 6401 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6402 %{
 6403   single_instruction;
 6404   fixed_latency(3); // Maximum latency for 64 bit mul
 6405   dst    : WR(write);
 6406   src1   : ISS(read);
 6407   src2   : ISS(read);
 6408   src3   : ISS(read);
 6409   INS01  : ISS;
 6410   MAC    : WR;
 6411 %}
 6412 
 6413 //------- Divide pipeline operations --------------------
 6414 
 6415 // Eg.  SDIV    w0, w1, w2
 6416 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6417 %{
 6418   single_instruction;
 6419   fixed_latency(8); // Maximum latency for 32 bit divide
 6420   dst    : WR(write);
 6421   src1   : ISS(read);
 6422   src2   : ISS(read);
 6423   INS0   : ISS; // Can only dual issue as instruction 0
 6424   DIV    : WR;
 6425 %}
 6426 
 6427 // Eg.  SDIV    x0, x1, x2
 6428 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6429 %{
 6430   single_instruction;
 6431   fixed_latency(16); // Maximum latency for 64 bit divide
 6432   dst    : WR(write);
 6433   src1   : ISS(read);
 6434   src2   : ISS(read);
 6435   INS0   : ISS; // Can only dual issue as instruction 0
 6436   DIV    : WR;
 6437 %}
 6438 
 6439 //------- Load pipeline operations ------------------------
 6440 
 6441 // Load - prefetch
 6442 // Eg.  PFRM    <mem>
 6443 pipe_class iload_prefetch(memory mem)
 6444 %{
 6445   single_instruction;
 6446   mem    : ISS(read);
 6447   INS01  : ISS;
 6448   LDST   : WR;
 6449 %}
 6450 
 6451 // Load - reg, mem
 6452 // Eg.  LDR     x0, <mem>
 6453 pipe_class iload_reg_mem(iRegI dst, memory mem)
 6454 %{
 6455   single_instruction;
 6456   dst    : WR(write);
 6457   mem    : ISS(read);
 6458   INS01  : ISS;
 6459   LDST   : WR;
 6460 %}
 6461 
 6462 // Load - reg, reg
 6463 // Eg.  LDR     x0, [sp, x1]
 6464 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 6465 %{
 6466   single_instruction;
 6467   dst    : WR(write);
 6468   src    : ISS(read);
 6469   INS01  : ISS;
 6470   LDST   : WR;
 6471 %}
 6472 
 6473 //------- Store pipeline operations -----------------------
 6474 
 6475 // Store - zr, mem
 6476 // Eg.  STR     zr, <mem>
 6477 pipe_class istore_mem(memory mem)
 6478 %{
 6479   single_instruction;
 6480   mem    : ISS(read);
 6481   INS01  : ISS;
 6482   LDST   : WR;
 6483 %}
 6484 
 6485 // Store - reg, mem
 6486 // Eg.  STR     x0, <mem>
 6487 pipe_class istore_reg_mem(iRegI src, memory mem)
 6488 %{
 6489   single_instruction;
 6490   mem    : ISS(read);
 6491   src    : EX2(read);
 6492   INS01  : ISS;
 6493   LDST   : WR;
 6494 %}
 6495 
 6496 // Store - reg, reg
 6497 // Eg. STR      x0, [sp, x1]
 6498 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 6499 %{
 6500   single_instruction;
 6501   dst    : ISS(read);
 6502   src    : EX2(read);
 6503   INS01  : ISS;
 6504   LDST   : WR;
 6505 %}
 6506 
 6507 //------- Store pipeline operations -----------------------
 6508 
 6509 // Branch
 6510 pipe_class pipe_branch()
 6511 %{
 6512   single_instruction;
 6513   INS01  : ISS;
 6514   BRANCH : EX1;
 6515 %}
 6516 
 6517 // Conditional branch
 6518 pipe_class pipe_branch_cond(rFlagsReg cr)
 6519 %{
 6520   single_instruction;
 6521   cr     : EX1(read);
 6522   INS01  : ISS;
 6523   BRANCH : EX1;
 6524 %}
 6525 
 6526 // Compare & Branch
 6527 // EG.  CBZ/CBNZ
 6528 pipe_class pipe_cmp_branch(iRegI op1)
 6529 %{
 6530   single_instruction;
 6531   op1    : EX1(read);
 6532   INS01  : ISS;
 6533   BRANCH : EX1;
 6534 %}
 6535 
 6536 //------- Synchronisation operations ----------------------
 6537 
 6538 // Any operation requiring serialization.
 6539 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 6540 pipe_class pipe_serial()
 6541 %{
 6542   single_instruction;
 6543   force_serialization;
 6544   fixed_latency(16);
 6545   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6546   LDST   : WR;
 6547 %}
 6548 
 6549 // Generic big/slow expanded idiom - also serialized
 6550 pipe_class pipe_slow()
 6551 %{
 6552   instruction_count(10);
 6553   multiple_bundles;
 6554   force_serialization;
 6555   fixed_latency(16);
 6556   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6557   LDST   : WR;
 6558 %}
 6559 
 6560 // Empty pipeline class
 6561 pipe_class pipe_class_empty()
 6562 %{
 6563   single_instruction;
 6564   fixed_latency(0);
 6565 %}
 6566 
 6567 // Default pipeline class.
 6568 pipe_class pipe_class_default()
 6569 %{
 6570   single_instruction;
 6571   fixed_latency(2);
 6572 %}
 6573 
 6574 // Pipeline class for compares.
 6575 pipe_class pipe_class_compare()
 6576 %{
 6577   single_instruction;
 6578   fixed_latency(16);
 6579 %}
 6580 
 6581 // Pipeline class for memory operations.
 6582 pipe_class pipe_class_memory()
 6583 %{
 6584   single_instruction;
 6585   fixed_latency(16);
 6586 %}
 6587 
 6588 // Pipeline class for call.
 6589 pipe_class pipe_class_call()
 6590 %{
 6591   single_instruction;
 6592   fixed_latency(100);
 6593 %}
 6594 
 6595 // Define the class for the Nop node.
 6596 define %{
 6597    MachNop = pipe_class_empty;
 6598 %}
 6599 
 6600 %}
 6601 //----------INSTRUCTIONS-------------------------------------------------------
 6602 //
 6603 // match      -- States which machine-independent subtree may be replaced
 6604 //               by this instruction.
 6605 // ins_cost   -- The estimated cost of this instruction is used by instruction
 6606 //               selection to identify a minimum cost tree of machine
 6607 //               instructions that matches a tree of machine-independent
 6608 //               instructions.
 6609 // format     -- A string providing the disassembly for this instruction.
 6610 //               The value of an instruction's operand may be inserted
 6611 //               by referring to it with a '$' prefix.
 6612 // opcode     -- Three instruction opcodes may be provided.  These are referred
 6613 //               to within an encode class as $primary, $secondary, and $tertiary
 6614 //               rrspectively.  The primary opcode is commonly used to
 6615 //               indicate the type of machine instruction, while secondary
 6616 //               and tertiary are often used for prefix options or addressing
 6617 //               modes.
 6618 // ins_encode -- A list of encode classes with parameters. The encode class
 6619 //               name must have been defined in an 'enc_class' specification
 6620 //               in the encode section of the architecture description.
 6621 
 6622 // ============================================================================
 6623 // Memory (Load/Store) Instructions
 6624 
 6625 // Load Instructions
 6626 
 6627 // Load Byte (8 bit signed)
 6628 instruct loadB(iRegINoSp dst, memory1 mem)
 6629 %{
 6630   match(Set dst (LoadB mem));
 6631   predicate(!needs_acquiring_load(n));
 6632 
 6633   ins_cost(4 * INSN_COST);
 6634   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 6635 
 6636   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 6637 
 6638   ins_pipe(iload_reg_mem);
 6639 %}
 6640 
 6641 // Load Byte (8 bit signed) into long
 6642 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 6643 %{
 6644   match(Set dst (ConvI2L (LoadB mem)));
 6645   predicate(!needs_acquiring_load(n->in(1)));
 6646 
 6647   ins_cost(4 * INSN_COST);
 6648   format %{ "ldrsb  $dst, $mem\t# byte" %}
 6649 
 6650   ins_encode(aarch64_enc_ldrsb(dst, mem));
 6651 
 6652   ins_pipe(iload_reg_mem);
 6653 %}
 6654 
 6655 // Load Byte (8 bit unsigned)
 6656 instruct loadUB(iRegINoSp dst, memory1 mem)
 6657 %{
 6658   match(Set dst (LoadUB mem));
 6659   predicate(!needs_acquiring_load(n));
 6660 
 6661   ins_cost(4 * INSN_COST);
 6662   format %{ "ldrbw  $dst, $mem\t# byte" %}
 6663 
 6664   ins_encode(aarch64_enc_ldrb(dst, mem));
 6665 
 6666   ins_pipe(iload_reg_mem);
 6667 %}
 6668 
 6669 // Load Byte (8 bit unsigned) into long
 6670 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 6671 %{
 6672   match(Set dst (ConvI2L (LoadUB mem)));
 6673   predicate(!needs_acquiring_load(n->in(1)));
 6674 
 6675   ins_cost(4 * INSN_COST);
 6676   format %{ "ldrb  $dst, $mem\t# byte" %}
 6677 
 6678   ins_encode(aarch64_enc_ldrb(dst, mem));
 6679 
 6680   ins_pipe(iload_reg_mem);
 6681 %}
 6682 
 6683 // Load Short (16 bit signed)
 6684 instruct loadS(iRegINoSp dst, memory2 mem)
 6685 %{
 6686   match(Set dst (LoadS mem));
 6687   predicate(!needs_acquiring_load(n));
 6688 
 6689   ins_cost(4 * INSN_COST);
 6690   format %{ "ldrshw  $dst, $mem\t# short" %}
 6691 
 6692   ins_encode(aarch64_enc_ldrshw(dst, mem));
 6693 
 6694   ins_pipe(iload_reg_mem);
 6695 %}
 6696 
 6697 // Load Short (16 bit signed) into long
 6698 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 6699 %{
 6700   match(Set dst (ConvI2L (LoadS mem)));
 6701   predicate(!needs_acquiring_load(n->in(1)));
 6702 
 6703   ins_cost(4 * INSN_COST);
 6704   format %{ "ldrsh  $dst, $mem\t# short" %}
 6705 
 6706   ins_encode(aarch64_enc_ldrsh(dst, mem));
 6707 
 6708   ins_pipe(iload_reg_mem);
 6709 %}
 6710 
 6711 // Load Char (16 bit unsigned)
 6712 instruct loadUS(iRegINoSp dst, memory2 mem)
 6713 %{
 6714   match(Set dst (LoadUS mem));
 6715   predicate(!needs_acquiring_load(n));
 6716 
 6717   ins_cost(4 * INSN_COST);
 6718   format %{ "ldrh  $dst, $mem\t# short" %}
 6719 
 6720   ins_encode(aarch64_enc_ldrh(dst, mem));
 6721 
 6722   ins_pipe(iload_reg_mem);
 6723 %}
 6724 
 6725 // Load Short/Char (16 bit unsigned) into long
 6726 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 6727 %{
 6728   match(Set dst (ConvI2L (LoadUS mem)));
 6729   predicate(!needs_acquiring_load(n->in(1)));
 6730 
 6731   ins_cost(4 * INSN_COST);
 6732   format %{ "ldrh  $dst, $mem\t# short" %}
 6733 
 6734   ins_encode(aarch64_enc_ldrh(dst, mem));
 6735 
 6736   ins_pipe(iload_reg_mem);
 6737 %}
 6738 
 6739 // Load Integer (32 bit signed)
 6740 instruct loadI(iRegINoSp dst, memory4 mem)
 6741 %{
 6742   match(Set dst (LoadI mem));
 6743   predicate(!needs_acquiring_load(n));
 6744 
 6745   ins_cost(4 * INSN_COST);
 6746   format %{ "ldrw  $dst, $mem\t# int" %}
 6747 
 6748   ins_encode(aarch64_enc_ldrw(dst, mem));
 6749 
 6750   ins_pipe(iload_reg_mem);
 6751 %}
 6752 
 6753 // Load Integer (32 bit signed) into long
 6754 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 6755 %{
 6756   match(Set dst (ConvI2L (LoadI mem)));
 6757   predicate(!needs_acquiring_load(n->in(1)));
 6758 
 6759   ins_cost(4 * INSN_COST);
 6760   format %{ "ldrsw  $dst, $mem\t# int" %}
 6761 
 6762   ins_encode(aarch64_enc_ldrsw(dst, mem));
 6763 
 6764   ins_pipe(iload_reg_mem);
 6765 %}
 6766 
 6767 // Load Integer (32 bit unsigned) into long
 6768 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 6769 %{
 6770   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 6771   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 6772 
 6773   ins_cost(4 * INSN_COST);
 6774   format %{ "ldrw  $dst, $mem\t# int" %}
 6775 
 6776   ins_encode(aarch64_enc_ldrw(dst, mem));
 6777 
 6778   ins_pipe(iload_reg_mem);
 6779 %}
 6780 
 6781 // Load Long (64 bit signed)
 6782 instruct loadL(iRegLNoSp dst, memory8 mem)
 6783 %{
 6784   match(Set dst (LoadL mem));
 6785   predicate(!needs_acquiring_load(n));
 6786 
 6787   ins_cost(4 * INSN_COST);
 6788   format %{ "ldr  $dst, $mem\t# int" %}
 6789 
 6790   ins_encode(aarch64_enc_ldr(dst, mem));
 6791 
 6792   ins_pipe(iload_reg_mem);
 6793 %}
 6794 
 6795 // Load Range
 6796 instruct loadRange(iRegINoSp dst, memory4 mem)
 6797 %{
 6798   match(Set dst (LoadRange mem));
 6799 
 6800   ins_cost(4 * INSN_COST);
 6801   format %{ "ldrw  $dst, $mem\t# range" %}
 6802 
 6803   ins_encode(aarch64_enc_ldrw(dst, mem));
 6804 
 6805   ins_pipe(iload_reg_mem);
 6806 %}
 6807 
 6808 // Load Pointer
 6809 instruct loadP(iRegPNoSp dst, memory8 mem)
 6810 %{
 6811   match(Set dst (LoadP mem));
 6812   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 6813 
 6814   ins_cost(4 * INSN_COST);
 6815   format %{ "ldr  $dst, $mem\t# ptr" %}
 6816 
 6817   ins_encode(aarch64_enc_ldr(dst, mem));
 6818 
 6819   ins_pipe(iload_reg_mem);
 6820 %}
 6821 
 6822 // Load Compressed Pointer
 6823 instruct loadN(iRegNNoSp dst, memory4 mem)
 6824 %{
 6825   match(Set dst (LoadN mem));
 6826   predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
 6827 
 6828   ins_cost(4 * INSN_COST);
 6829   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 6830 
 6831   ins_encode(aarch64_enc_ldrw(dst, mem));
 6832 
 6833   ins_pipe(iload_reg_mem);
 6834 %}
 6835 
 6836 // Load Klass Pointer
 6837 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 6838 %{
 6839   match(Set dst (LoadKlass mem));
 6840   predicate(!needs_acquiring_load(n));
 6841 
 6842   ins_cost(4 * INSN_COST);
 6843   format %{ "ldr  $dst, $mem\t# class" %}
 6844 
 6845   ins_encode(aarch64_enc_ldr(dst, mem));
 6846 
 6847   ins_pipe(iload_reg_mem);
 6848 %}
 6849 
 6850 // Load Narrow Klass Pointer
 6851 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 6852 %{
 6853   match(Set dst (LoadNKlass mem));
 6854   predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
 6855 
 6856   ins_cost(4 * INSN_COST);
 6857   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 6858 
 6859   ins_encode(aarch64_enc_ldrw(dst, mem));
 6860 
 6861   ins_pipe(iload_reg_mem);
 6862 %}
 6863 
 6864 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
 6865 %{
 6866   match(Set dst (LoadNKlass mem));
 6867   predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
 6868 
 6869   ins_cost(4 * INSN_COST);
 6870   format %{
 6871     "ldrw  $dst, $mem\t# compressed class ptr, shifted\n\t"
 6872     "lsrw  $dst, $dst, markWord::klass_shift_at_offset"
 6873   %}
 6874   ins_encode %{
 6875     // inlined aarch64_enc_ldrw
 6876     loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
 6877               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 6878     __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
 6879   %}
 6880   ins_pipe(iload_reg_mem);
 6881 %}
 6882 
 6883 // Load Float
 6884 instruct loadF(vRegF dst, memory4 mem)
 6885 %{
 6886   match(Set dst (LoadF mem));
 6887   predicate(!needs_acquiring_load(n));
 6888 
 6889   ins_cost(4 * INSN_COST);
 6890   format %{ "ldrs  $dst, $mem\t# float" %}
 6891 
 6892   ins_encode( aarch64_enc_ldrs(dst, mem) );
 6893 
 6894   ins_pipe(pipe_class_memory);
 6895 %}
 6896 
 6897 // Load Double
 6898 instruct loadD(vRegD dst, memory8 mem)
 6899 %{
 6900   match(Set dst (LoadD mem));
 6901   predicate(!needs_acquiring_load(n));
 6902 
 6903   ins_cost(4 * INSN_COST);
 6904   format %{ "ldrd  $dst, $mem\t# double" %}
 6905 
 6906   ins_encode( aarch64_enc_ldrd(dst, mem) );
 6907 
 6908   ins_pipe(pipe_class_memory);
 6909 %}
 6910 
 6911 
 6912 // Load Int Constant
 6913 instruct loadConI(iRegINoSp dst, immI src)
 6914 %{
 6915   match(Set dst src);
 6916 
 6917   ins_cost(INSN_COST);
 6918   format %{ "mov $dst, $src\t# int" %}
 6919 
 6920   ins_encode( aarch64_enc_movw_imm(dst, src) );
 6921 
 6922   ins_pipe(ialu_imm);
 6923 %}
 6924 
 6925 // Load Long Constant
 6926 instruct loadConL(iRegLNoSp dst, immL src)
 6927 %{
 6928   match(Set dst src);
 6929 
 6930   ins_cost(INSN_COST);
 6931   format %{ "mov $dst, $src\t# long" %}
 6932 
 6933   ins_encode( aarch64_enc_mov_imm(dst, src) );
 6934 
 6935   ins_pipe(ialu_imm);
 6936 %}
 6937 
 6938 // Load Pointer Constant
 6939 
 6940 instruct loadConP(iRegPNoSp dst, immP con)
 6941 %{
 6942   match(Set dst con);
 6943 
 6944   ins_cost(INSN_COST * 4);
 6945   format %{
 6946     "mov  $dst, $con\t# ptr\n\t"
 6947   %}
 6948 
 6949   ins_encode(aarch64_enc_mov_p(dst, con));
 6950 
 6951   ins_pipe(ialu_imm);
 6952 %}
 6953 
 6954 // Load Null Pointer Constant
 6955 
 6956 instruct loadConP0(iRegPNoSp dst, immP0 con)
 6957 %{
 6958   match(Set dst con);
 6959 
 6960   ins_cost(INSN_COST);
 6961   format %{ "mov  $dst, $con\t# nullptr ptr" %}
 6962 
 6963   ins_encode(aarch64_enc_mov_p0(dst, con));
 6964 
 6965   ins_pipe(ialu_imm);
 6966 %}
 6967 
 6968 // Load Pointer Constant One
 6969 
 6970 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 6971 %{
 6972   match(Set dst con);
 6973 
 6974   ins_cost(INSN_COST);
 6975   format %{ "mov  $dst, $con\t# nullptr ptr" %}
 6976 
 6977   ins_encode(aarch64_enc_mov_p1(dst, con));
 6978 
 6979   ins_pipe(ialu_imm);
 6980 %}
 6981 
 6982 // Load Narrow Pointer Constant
 6983 
 6984 instruct loadConN(iRegNNoSp dst, immN con)
 6985 %{
 6986   match(Set dst con);
 6987 
 6988   ins_cost(INSN_COST * 4);
 6989   format %{ "mov  $dst, $con\t# compressed ptr" %}
 6990 
 6991   ins_encode(aarch64_enc_mov_n(dst, con));
 6992 
 6993   ins_pipe(ialu_imm);
 6994 %}
 6995 
 6996 // Load Narrow Null Pointer Constant
 6997 
 6998 instruct loadConN0(iRegNNoSp dst, immN0 con)
 6999 %{
 7000   match(Set dst con);
 7001 
 7002   ins_cost(INSN_COST);
 7003   format %{ "mov  $dst, $con\t# compressed nullptr ptr" %}
 7004 
 7005   ins_encode(aarch64_enc_mov_n0(dst, con));
 7006 
 7007   ins_pipe(ialu_imm);
 7008 %}
 7009 
 7010 // Load Narrow Klass Constant
 7011 
 7012 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7013 %{
 7014   match(Set dst con);
 7015 
 7016   ins_cost(INSN_COST);
 7017   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7018 
 7019   ins_encode(aarch64_enc_mov_nk(dst, con));
 7020 
 7021   ins_pipe(ialu_imm);
 7022 %}
 7023 
 7024 // Load Packed Float Constant
 7025 
 7026 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7027   match(Set dst con);
 7028   ins_cost(INSN_COST * 4);
 7029   format %{ "fmovs  $dst, $con"%}
 7030   ins_encode %{
 7031     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7032   %}
 7033 
 7034   ins_pipe(fp_imm_s);
 7035 %}
 7036 
 7037 // Load Float Constant
 7038 
 7039 instruct loadConF(vRegF dst, immF con) %{
 7040   match(Set dst con);
 7041 
 7042   ins_cost(INSN_COST * 4);
 7043 
 7044   format %{
 7045     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7046   %}
 7047 
 7048   ins_encode %{
 7049     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7050   %}
 7051 
 7052   ins_pipe(fp_load_constant_s);
 7053 %}
 7054 
 7055 // Load Packed Double Constant
 7056 
 7057 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7058   match(Set dst con);
 7059   ins_cost(INSN_COST);
 7060   format %{ "fmovd  $dst, $con"%}
 7061   ins_encode %{
 7062     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7063   %}
 7064 
 7065   ins_pipe(fp_imm_d);
 7066 %}
 7067 
 7068 // Load Double Constant
 7069 
 7070 instruct loadConD(vRegD dst, immD con) %{
 7071   match(Set dst con);
 7072 
 7073   ins_cost(INSN_COST * 5);
 7074   format %{
 7075     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7076   %}
 7077 
 7078   ins_encode %{
 7079     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7080   %}
 7081 
 7082   ins_pipe(fp_load_constant_d);
 7083 %}
 7084 
 7085 // Load Half Float Constant
 7086 instruct loadConH(vRegF dst, immH con) %{
 7087   match(Set dst con);
 7088   format %{ "mov    rscratch1, $con\n\t"
 7089             "fmov   $dst, rscratch1"
 7090          %}
 7091   ins_encode %{
 7092     __ movw(rscratch1, (uint32_t)$con$$constant);
 7093     __ fmovs($dst$$FloatRegister, rscratch1);
 7094   %}
 7095   ins_pipe(pipe_class_default);
 7096 %}
 7097 
 7098 // Store Instructions
 7099 
 7100 // Store Byte
 7101 instruct storeB(iRegIorL2I src, memory1 mem)
 7102 %{
 7103   match(Set mem (StoreB mem src));
 7104   predicate(!needs_releasing_store(n));
 7105 
 7106   ins_cost(INSN_COST);
 7107   format %{ "strb  $src, $mem\t# byte" %}
 7108 
 7109   ins_encode(aarch64_enc_strb(src, mem));
 7110 
 7111   ins_pipe(istore_reg_mem);
 7112 %}
 7113 
 7114 
 7115 instruct storeimmB0(immI0 zero, memory1 mem)
 7116 %{
 7117   match(Set mem (StoreB mem zero));
 7118   predicate(!needs_releasing_store(n));
 7119 
 7120   ins_cost(INSN_COST);
 7121   format %{ "strb rscractch2, $mem\t# byte" %}
 7122 
 7123   ins_encode(aarch64_enc_strb0(mem));
 7124 
 7125   ins_pipe(istore_mem);
 7126 %}
 7127 
 7128 // Store Char/Short
 7129 instruct storeC(iRegIorL2I src, memory2 mem)
 7130 %{
 7131   match(Set mem (StoreC mem src));
 7132   predicate(!needs_releasing_store(n));
 7133 
 7134   ins_cost(INSN_COST);
 7135   format %{ "strh  $src, $mem\t# short" %}
 7136 
 7137   ins_encode(aarch64_enc_strh(src, mem));
 7138 
 7139   ins_pipe(istore_reg_mem);
 7140 %}
 7141 
 7142 instruct storeimmC0(immI0 zero, memory2 mem)
 7143 %{
 7144   match(Set mem (StoreC mem zero));
 7145   predicate(!needs_releasing_store(n));
 7146 
 7147   ins_cost(INSN_COST);
 7148   format %{ "strh  zr, $mem\t# short" %}
 7149 
 7150   ins_encode(aarch64_enc_strh0(mem));
 7151 
 7152   ins_pipe(istore_mem);
 7153 %}
 7154 
 7155 // Store Integer
 7156 
 7157 instruct storeI(iRegIorL2I src, memory4 mem)
 7158 %{
 7159   match(Set mem(StoreI mem src));
 7160   predicate(!needs_releasing_store(n));
 7161 
 7162   ins_cost(INSN_COST);
 7163   format %{ "strw  $src, $mem\t# int" %}
 7164 
 7165   ins_encode(aarch64_enc_strw(src, mem));
 7166 
 7167   ins_pipe(istore_reg_mem);
 7168 %}
 7169 
 7170 instruct storeimmI0(immI0 zero, memory4 mem)
 7171 %{
 7172   match(Set mem(StoreI mem zero));
 7173   predicate(!needs_releasing_store(n));
 7174 
 7175   ins_cost(INSN_COST);
 7176   format %{ "strw  zr, $mem\t# int" %}
 7177 
 7178   ins_encode(aarch64_enc_strw0(mem));
 7179 
 7180   ins_pipe(istore_mem);
 7181 %}
 7182 
 7183 // Store Long (64 bit signed)
 7184 instruct storeL(iRegL src, memory8 mem)
 7185 %{
 7186   match(Set mem (StoreL mem src));
 7187   predicate(!needs_releasing_store(n));
 7188 
 7189   ins_cost(INSN_COST);
 7190   format %{ "str  $src, $mem\t# int" %}
 7191 
 7192   ins_encode(aarch64_enc_str(src, mem));
 7193 
 7194   ins_pipe(istore_reg_mem);
 7195 %}
 7196 
 7197 // Store Long (64 bit signed)
 7198 instruct storeimmL0(immL0 zero, memory8 mem)
 7199 %{
 7200   match(Set mem (StoreL mem zero));
 7201   predicate(!needs_releasing_store(n));
 7202 
 7203   ins_cost(INSN_COST);
 7204   format %{ "str  zr, $mem\t# int" %}
 7205 
 7206   ins_encode(aarch64_enc_str0(mem));
 7207 
 7208   ins_pipe(istore_mem);
 7209 %}
 7210 
 7211 // Store Pointer
 7212 instruct storeP(iRegP src, memory8 mem)
 7213 %{
 7214   match(Set mem (StoreP mem src));
 7215   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7216 
 7217   ins_cost(INSN_COST);
 7218   format %{ "str  $src, $mem\t# ptr" %}
 7219 
 7220   ins_encode(aarch64_enc_str(src, mem));
 7221 
 7222   ins_pipe(istore_reg_mem);
 7223 %}
 7224 
 7225 // Store Pointer
 7226 instruct storeimmP0(immP0 zero, memory8 mem)
 7227 %{
 7228   match(Set mem (StoreP mem zero));
 7229   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7230 
 7231   ins_cost(INSN_COST);
 7232   format %{ "str zr, $mem\t# ptr" %}
 7233 
 7234   ins_encode(aarch64_enc_str0(mem));
 7235 
 7236   ins_pipe(istore_mem);
 7237 %}
 7238 
 7239 // Store Compressed Pointer
 7240 instruct storeN(iRegN src, memory4 mem)
 7241 %{
 7242   match(Set mem (StoreN mem src));
 7243   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7244 
 7245   ins_cost(INSN_COST);
 7246   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7247 
 7248   ins_encode(aarch64_enc_strw(src, mem));
 7249 
 7250   ins_pipe(istore_reg_mem);
 7251 %}
 7252 
 7253 instruct storeImmN0(immN0 zero, memory4 mem)
 7254 %{
 7255   match(Set mem (StoreN mem zero));
 7256   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7257 
 7258   ins_cost(INSN_COST);
 7259   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7260 
 7261   ins_encode(aarch64_enc_strw0(mem));
 7262 
 7263   ins_pipe(istore_mem);
 7264 %}
 7265 
 7266 // Store Float
 7267 instruct storeF(vRegF src, memory4 mem)
 7268 %{
 7269   match(Set mem (StoreF mem src));
 7270   predicate(!needs_releasing_store(n));
 7271 
 7272   ins_cost(INSN_COST);
 7273   format %{ "strs  $src, $mem\t# float" %}
 7274 
 7275   ins_encode( aarch64_enc_strs(src, mem) );
 7276 
 7277   ins_pipe(pipe_class_memory);
 7278 %}
 7279 
 7280 // TODO
 7281 // implement storeImmF0 and storeFImmPacked
 7282 
 7283 // Store Double
 7284 instruct storeD(vRegD src, memory8 mem)
 7285 %{
 7286   match(Set mem (StoreD mem src));
 7287   predicate(!needs_releasing_store(n));
 7288 
 7289   ins_cost(INSN_COST);
 7290   format %{ "strd  $src, $mem\t# double" %}
 7291 
 7292   ins_encode( aarch64_enc_strd(src, mem) );
 7293 
 7294   ins_pipe(pipe_class_memory);
 7295 %}
 7296 
 7297 // Store Compressed Klass Pointer
 7298 instruct storeNKlass(iRegN src, memory4 mem)
 7299 %{
 7300   predicate(!needs_releasing_store(n));
 7301   match(Set mem (StoreNKlass mem src));
 7302 
 7303   ins_cost(INSN_COST);
 7304   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7305 
 7306   ins_encode(aarch64_enc_strw(src, mem));
 7307 
 7308   ins_pipe(istore_reg_mem);
 7309 %}
 7310 
 7311 // TODO
 7312 // implement storeImmD0 and storeDImmPacked
 7313 
 7314 // prefetch instructions
 7315 // Must be safe to execute with invalid address (cannot fault).
 7316 
 7317 instruct prefetchalloc( memory8 mem ) %{
 7318   match(PrefetchAllocation mem);
 7319 
 7320   ins_cost(INSN_COST);
 7321   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7322 
 7323   ins_encode( aarch64_enc_prefetchw(mem) );
 7324 
 7325   ins_pipe(iload_prefetch);
 7326 %}
 7327 
 7328 //  ---------------- volatile loads and stores ----------------
 7329 
 7330 // Load Byte (8 bit signed)
 7331 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7332 %{
 7333   match(Set dst (LoadB mem));
 7334 
 7335   ins_cost(VOLATILE_REF_COST);
 7336   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7337 
 7338   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7339 
 7340   ins_pipe(pipe_serial);
 7341 %}
 7342 
 7343 // Load Byte (8 bit signed) into long
 7344 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7345 %{
 7346   match(Set dst (ConvI2L (LoadB mem)));
 7347 
 7348   ins_cost(VOLATILE_REF_COST);
 7349   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7350 
 7351   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7352 
 7353   ins_pipe(pipe_serial);
 7354 %}
 7355 
 7356 // Load Byte (8 bit unsigned)
 7357 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7358 %{
 7359   match(Set dst (LoadUB mem));
 7360 
 7361   ins_cost(VOLATILE_REF_COST);
 7362   format %{ "ldarb  $dst, $mem\t# byte" %}
 7363 
 7364   ins_encode(aarch64_enc_ldarb(dst, mem));
 7365 
 7366   ins_pipe(pipe_serial);
 7367 %}
 7368 
 7369 // Load Byte (8 bit unsigned) into long
 7370 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7371 %{
 7372   match(Set dst (ConvI2L (LoadUB mem)));
 7373 
 7374   ins_cost(VOLATILE_REF_COST);
 7375   format %{ "ldarb  $dst, $mem\t# byte" %}
 7376 
 7377   ins_encode(aarch64_enc_ldarb(dst, mem));
 7378 
 7379   ins_pipe(pipe_serial);
 7380 %}
 7381 
 7382 // Load Short (16 bit signed)
 7383 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7384 %{
 7385   match(Set dst (LoadS mem));
 7386 
 7387   ins_cost(VOLATILE_REF_COST);
 7388   format %{ "ldarshw  $dst, $mem\t# short" %}
 7389 
 7390   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7391 
 7392   ins_pipe(pipe_serial);
 7393 %}
 7394 
 7395 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7396 %{
 7397   match(Set dst (LoadUS mem));
 7398 
 7399   ins_cost(VOLATILE_REF_COST);
 7400   format %{ "ldarhw  $dst, $mem\t# short" %}
 7401 
 7402   ins_encode(aarch64_enc_ldarhw(dst, mem));
 7403 
 7404   ins_pipe(pipe_serial);
 7405 %}
 7406 
 7407 // Load Short/Char (16 bit unsigned) into long
 7408 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7409 %{
 7410   match(Set dst (ConvI2L (LoadUS mem)));
 7411 
 7412   ins_cost(VOLATILE_REF_COST);
 7413   format %{ "ldarh  $dst, $mem\t# short" %}
 7414 
 7415   ins_encode(aarch64_enc_ldarh(dst, mem));
 7416 
 7417   ins_pipe(pipe_serial);
 7418 %}
 7419 
 7420 // Load Short/Char (16 bit signed) into long
 7421 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7422 %{
 7423   match(Set dst (ConvI2L (LoadS mem)));
 7424 
 7425   ins_cost(VOLATILE_REF_COST);
 7426   format %{ "ldarh  $dst, $mem\t# short" %}
 7427 
 7428   ins_encode(aarch64_enc_ldarsh(dst, mem));
 7429 
 7430   ins_pipe(pipe_serial);
 7431 %}
 7432 
 7433 // Load Integer (32 bit signed)
 7434 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7435 %{
 7436   match(Set dst (LoadI mem));
 7437 
 7438   ins_cost(VOLATILE_REF_COST);
 7439   format %{ "ldarw  $dst, $mem\t# int" %}
 7440 
 7441   ins_encode(aarch64_enc_ldarw(dst, mem));
 7442 
 7443   ins_pipe(pipe_serial);
 7444 %}
 7445 
 7446 // Load Integer (32 bit unsigned) into long
 7447 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 7448 %{
 7449   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7450 
 7451   ins_cost(VOLATILE_REF_COST);
 7452   format %{ "ldarw  $dst, $mem\t# int" %}
 7453 
 7454   ins_encode(aarch64_enc_ldarw(dst, mem));
 7455 
 7456   ins_pipe(pipe_serial);
 7457 %}
 7458 
 7459 // Load Long (64 bit signed)
 7460 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7461 %{
 7462   match(Set dst (LoadL mem));
 7463 
 7464   ins_cost(VOLATILE_REF_COST);
 7465   format %{ "ldar  $dst, $mem\t# int" %}
 7466 
 7467   ins_encode(aarch64_enc_ldar(dst, mem));
 7468 
 7469   ins_pipe(pipe_serial);
 7470 %}
 7471 
 7472 // Load Pointer
 7473 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 7474 %{
 7475   match(Set dst (LoadP mem));
 7476   predicate(n->as_Load()->barrier_data() == 0);
 7477 
 7478   ins_cost(VOLATILE_REF_COST);
 7479   format %{ "ldar  $dst, $mem\t# ptr" %}
 7480 
 7481   ins_encode(aarch64_enc_ldar(dst, mem));
 7482 
 7483   ins_pipe(pipe_serial);
 7484 %}
 7485 
 7486 // Load Compressed Pointer
 7487 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 7488 %{
 7489   match(Set dst (LoadN mem));
 7490   predicate(n->as_Load()->barrier_data() == 0);
 7491 
 7492   ins_cost(VOLATILE_REF_COST);
 7493   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 7494 
 7495   ins_encode(aarch64_enc_ldarw(dst, mem));
 7496 
 7497   ins_pipe(pipe_serial);
 7498 %}
 7499 
 7500 // Load Float
 7501 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 7502 %{
 7503   match(Set dst (LoadF mem));
 7504 
 7505   ins_cost(VOLATILE_REF_COST);
 7506   format %{ "ldars  $dst, $mem\t# float" %}
 7507 
 7508   ins_encode( aarch64_enc_fldars(dst, mem) );
 7509 
 7510   ins_pipe(pipe_serial);
 7511 %}
 7512 
 7513 // Load Double
 7514 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 7515 %{
 7516   match(Set dst (LoadD mem));
 7517 
 7518   ins_cost(VOLATILE_REF_COST);
 7519   format %{ "ldard  $dst, $mem\t# double" %}
 7520 
 7521   ins_encode( aarch64_enc_fldard(dst, mem) );
 7522 
 7523   ins_pipe(pipe_serial);
 7524 %}
 7525 
 7526 // Store Byte
 7527 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7528 %{
 7529   match(Set mem (StoreB mem src));
 7530 
 7531   ins_cost(VOLATILE_REF_COST);
 7532   format %{ "stlrb  $src, $mem\t# byte" %}
 7533 
 7534   ins_encode(aarch64_enc_stlrb(src, mem));
 7535 
 7536   ins_pipe(pipe_class_memory);
 7537 %}
 7538 
 7539 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7540 %{
 7541   match(Set mem (StoreB mem zero));
 7542 
 7543   ins_cost(VOLATILE_REF_COST);
 7544   format %{ "stlrb  zr, $mem\t# byte" %}
 7545 
 7546   ins_encode(aarch64_enc_stlrb0(mem));
 7547 
 7548   ins_pipe(pipe_class_memory);
 7549 %}
 7550 
 7551 // Store Char/Short
 7552 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7553 %{
 7554   match(Set mem (StoreC mem src));
 7555 
 7556   ins_cost(VOLATILE_REF_COST);
 7557   format %{ "stlrh  $src, $mem\t# short" %}
 7558 
 7559   ins_encode(aarch64_enc_stlrh(src, mem));
 7560 
 7561   ins_pipe(pipe_class_memory);
 7562 %}
 7563 
 7564 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7565 %{
 7566   match(Set mem (StoreC mem zero));
 7567 
 7568   ins_cost(VOLATILE_REF_COST);
 7569   format %{ "stlrh  zr, $mem\t# short" %}
 7570 
 7571   ins_encode(aarch64_enc_stlrh0(mem));
 7572 
 7573   ins_pipe(pipe_class_memory);
 7574 %}
 7575 
 7576 // Store Integer
 7577 
 7578 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7579 %{
 7580   match(Set mem(StoreI mem src));
 7581 
 7582   ins_cost(VOLATILE_REF_COST);
 7583   format %{ "stlrw  $src, $mem\t# int" %}
 7584 
 7585   ins_encode(aarch64_enc_stlrw(src, mem));
 7586 
 7587   ins_pipe(pipe_class_memory);
 7588 %}
 7589 
 7590 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7591 %{
 7592   match(Set mem(StoreI mem zero));
 7593 
 7594   ins_cost(VOLATILE_REF_COST);
 7595   format %{ "stlrw  zr, $mem\t# int" %}
 7596 
 7597   ins_encode(aarch64_enc_stlrw0(mem));
 7598 
 7599   ins_pipe(pipe_class_memory);
 7600 %}
 7601 
 7602 // Store Long (64 bit signed)
 7603 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 7604 %{
 7605   match(Set mem (StoreL mem src));
 7606 
 7607   ins_cost(VOLATILE_REF_COST);
 7608   format %{ "stlr  $src, $mem\t# int" %}
 7609 
 7610   ins_encode(aarch64_enc_stlr(src, mem));
 7611 
 7612   ins_pipe(pipe_class_memory);
 7613 %}
 7614 
 7615 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 7616 %{
 7617   match(Set mem (StoreL mem zero));
 7618 
 7619   ins_cost(VOLATILE_REF_COST);
 7620   format %{ "stlr  zr, $mem\t# int" %}
 7621 
 7622   ins_encode(aarch64_enc_stlr0(mem));
 7623 
 7624   ins_pipe(pipe_class_memory);
 7625 %}
 7626 
 7627 // Store Pointer
 7628 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 7629 %{
 7630   match(Set mem (StoreP mem src));
 7631   predicate(n->as_Store()->barrier_data() == 0);
 7632 
 7633   ins_cost(VOLATILE_REF_COST);
 7634   format %{ "stlr  $src, $mem\t# ptr" %}
 7635 
 7636   ins_encode(aarch64_enc_stlr(src, mem));
 7637 
 7638   ins_pipe(pipe_class_memory);
 7639 %}
 7640 
 7641 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 7642 %{
 7643   match(Set mem (StoreP mem zero));
 7644   predicate(n->as_Store()->barrier_data() == 0);
 7645 
 7646   ins_cost(VOLATILE_REF_COST);
 7647   format %{ "stlr  zr, $mem\t# ptr" %}
 7648 
 7649   ins_encode(aarch64_enc_stlr0(mem));
 7650 
 7651   ins_pipe(pipe_class_memory);
 7652 %}
 7653 
 7654 // Store Compressed Pointer
 7655 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 7656 %{
 7657   match(Set mem (StoreN mem src));
 7658   predicate(n->as_Store()->barrier_data() == 0);
 7659 
 7660   ins_cost(VOLATILE_REF_COST);
 7661   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 7662 
 7663   ins_encode(aarch64_enc_stlrw(src, mem));
 7664 
 7665   ins_pipe(pipe_class_memory);
 7666 %}
 7667 
 7668 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 7669 %{
 7670   match(Set mem (StoreN mem zero));
 7671   predicate(n->as_Store()->barrier_data() == 0);
 7672 
 7673   ins_cost(VOLATILE_REF_COST);
 7674   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 7675 
 7676   ins_encode(aarch64_enc_stlrw0(mem));
 7677 
 7678   ins_pipe(pipe_class_memory);
 7679 %}
 7680 
 7681 // Store Float
 7682 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 7683 %{
 7684   match(Set mem (StoreF mem src));
 7685 
 7686   ins_cost(VOLATILE_REF_COST);
 7687   format %{ "stlrs  $src, $mem\t# float" %}
 7688 
 7689   ins_encode( aarch64_enc_fstlrs(src, mem) );
 7690 
 7691   ins_pipe(pipe_class_memory);
 7692 %}
 7693 
 7694 // TODO
 7695 // implement storeImmF0 and storeFImmPacked
 7696 
 7697 // Store Double
 7698 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 7699 %{
 7700   match(Set mem (StoreD mem src));
 7701 
 7702   ins_cost(VOLATILE_REF_COST);
 7703   format %{ "stlrd  $src, $mem\t# double" %}
 7704 
 7705   ins_encode( aarch64_enc_fstlrd(src, mem) );
 7706 
 7707   ins_pipe(pipe_class_memory);
 7708 %}
 7709 
 7710 //  ---------------- end of volatile loads and stores ----------------
 7711 
 7712 instruct cacheWB(indirect addr)
 7713 %{
 7714   predicate(VM_Version::supports_data_cache_line_flush());
 7715   match(CacheWB addr);
 7716 
 7717   ins_cost(100);
 7718   format %{"cache wb $addr" %}
 7719   ins_encode %{
 7720     assert($addr->index_position() < 0, "should be");
 7721     assert($addr$$disp == 0, "should be");
 7722     __ cache_wb(Address($addr$$base$$Register, 0));
 7723   %}
 7724   ins_pipe(pipe_slow); // XXX
 7725 %}
 7726 
 7727 instruct cacheWBPreSync()
 7728 %{
 7729   predicate(VM_Version::supports_data_cache_line_flush());
 7730   match(CacheWBPreSync);
 7731 
 7732   ins_cost(100);
 7733   format %{"cache wb presync" %}
 7734   ins_encode %{
 7735     __ cache_wbsync(true);
 7736   %}
 7737   ins_pipe(pipe_slow); // XXX
 7738 %}
 7739 
 7740 instruct cacheWBPostSync()
 7741 %{
 7742   predicate(VM_Version::supports_data_cache_line_flush());
 7743   match(CacheWBPostSync);
 7744 
 7745   ins_cost(100);
 7746   format %{"cache wb postsync" %}
 7747   ins_encode %{
 7748     __ cache_wbsync(false);
 7749   %}
 7750   ins_pipe(pipe_slow); // XXX
 7751 %}
 7752 
 7753 // ============================================================================
 7754 // BSWAP Instructions
 7755 
 7756 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 7757   match(Set dst (ReverseBytesI src));
 7758 
 7759   ins_cost(INSN_COST);
 7760   format %{ "revw  $dst, $src" %}
 7761 
 7762   ins_encode %{
 7763     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 7764   %}
 7765 
 7766   ins_pipe(ialu_reg);
 7767 %}
 7768 
 7769 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 7770   match(Set dst (ReverseBytesL src));
 7771 
 7772   ins_cost(INSN_COST);
 7773   format %{ "rev  $dst, $src" %}
 7774 
 7775   ins_encode %{
 7776     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 7777   %}
 7778 
 7779   ins_pipe(ialu_reg);
 7780 %}
 7781 
 7782 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 7783   match(Set dst (ReverseBytesUS src));
 7784 
 7785   ins_cost(INSN_COST);
 7786   format %{ "rev16w  $dst, $src" %}
 7787 
 7788   ins_encode %{
 7789     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 7790   %}
 7791 
 7792   ins_pipe(ialu_reg);
 7793 %}
 7794 
 7795 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 7796   match(Set dst (ReverseBytesS src));
 7797 
 7798   ins_cost(INSN_COST);
 7799   format %{ "rev16w  $dst, $src\n\t"
 7800             "sbfmw $dst, $dst, #0, #15" %}
 7801 
 7802   ins_encode %{
 7803     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 7804     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 7805   %}
 7806 
 7807   ins_pipe(ialu_reg);
 7808 %}
 7809 
 7810 // ============================================================================
 7811 // Zero Count Instructions
 7812 
 7813 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 7814   match(Set dst (CountLeadingZerosI src));
 7815 
 7816   ins_cost(INSN_COST);
 7817   format %{ "clzw  $dst, $src" %}
 7818   ins_encode %{
 7819     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 7820   %}
 7821 
 7822   ins_pipe(ialu_reg);
 7823 %}
 7824 
 7825 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 7826   match(Set dst (CountLeadingZerosL src));
 7827 
 7828   ins_cost(INSN_COST);
 7829   format %{ "clz   $dst, $src" %}
 7830   ins_encode %{
 7831     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 7832   %}
 7833 
 7834   ins_pipe(ialu_reg);
 7835 %}
 7836 
 7837 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 7838   match(Set dst (CountTrailingZerosI src));
 7839 
 7840   ins_cost(INSN_COST * 2);
 7841   format %{ "rbitw  $dst, $src\n\t"
 7842             "clzw   $dst, $dst" %}
 7843   ins_encode %{
 7844     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 7845     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 7846   %}
 7847 
 7848   ins_pipe(ialu_reg);
 7849 %}
 7850 
 7851 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 7852   match(Set dst (CountTrailingZerosL src));
 7853 
 7854   ins_cost(INSN_COST * 2);
 7855   format %{ "rbit   $dst, $src\n\t"
 7856             "clz    $dst, $dst" %}
 7857   ins_encode %{
 7858     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 7859     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 7860   %}
 7861 
 7862   ins_pipe(ialu_reg);
 7863 %}
 7864 
 7865 //---------- Population Count Instructions -------------------------------------
 7866 //
 7867 
 7868 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 7869   match(Set dst (PopCountI src));
 7870   effect(TEMP tmp);
 7871   ins_cost(INSN_COST * 13);
 7872 
 7873   format %{ "fmovs  $tmp, $src\t# vector (1S)\n\t"
 7874             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7875             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7876             "mov    $dst, $tmp\t# vector (1D)" %}
 7877   ins_encode %{
 7878     __ fmovs($tmp$$FloatRegister, $src$$Register);
 7879     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7880     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7881     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7882   %}
 7883 
 7884   ins_pipe(pipe_class_default);
 7885 %}
 7886 
 7887 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 7888   match(Set dst (PopCountI (LoadI mem)));
 7889   effect(TEMP tmp);
 7890   ins_cost(INSN_COST * 13);
 7891 
 7892   format %{ "ldrs   $tmp, $mem\n\t"
 7893             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7894             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7895             "mov    $dst, $tmp\t# vector (1D)" %}
 7896   ins_encode %{
 7897     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 7898     loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 7899               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 7900     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7901     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7902     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7903   %}
 7904 
 7905   ins_pipe(pipe_class_default);
 7906 %}
 7907 
 7908 // Note: Long.bitCount(long) returns an int.
 7909 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 7910   match(Set dst (PopCountL src));
 7911   effect(TEMP tmp);
 7912   ins_cost(INSN_COST * 13);
 7913 
 7914   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 7915             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7916             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7917             "mov    $dst, $tmp\t# vector (1D)" %}
 7918   ins_encode %{
 7919     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 7920     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7921     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7922     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7923   %}
 7924 
 7925   ins_pipe(pipe_class_default);
 7926 %}
 7927 
 7928 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 7929   match(Set dst (PopCountL (LoadL mem)));
 7930   effect(TEMP tmp);
 7931   ins_cost(INSN_COST * 13);
 7932 
 7933   format %{ "ldrd   $tmp, $mem\n\t"
 7934             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7935             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7936             "mov    $dst, $tmp\t# vector (1D)" %}
 7937   ins_encode %{
 7938     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 7939     loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 7940               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 7941     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7942     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7943     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7944   %}
 7945 
 7946   ins_pipe(pipe_class_default);
 7947 %}
 7948 
 7949 // ============================================================================
 7950 // VerifyVectorAlignment Instruction
 7951 
 7952 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
 7953   match(Set addr (VerifyVectorAlignment addr mask));
 7954   effect(KILL cr);
 7955   format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
 7956   ins_encode %{
 7957     Label Lskip;
 7958     // check if masked bits of addr are zero
 7959     __ tst($addr$$Register, $mask$$constant);
 7960     __ br(Assembler::EQ, Lskip);
 7961     __ stop("verify_vector_alignment found a misaligned vector memory access");
 7962     __ bind(Lskip);
 7963   %}
 7964   ins_pipe(pipe_slow);
 7965 %}
 7966 
 7967 // ============================================================================
 7968 // MemBar Instruction
 7969 
 7970 instruct load_fence() %{
 7971   match(LoadFence);
 7972   ins_cost(VOLATILE_REF_COST);
 7973 
 7974   format %{ "load_fence" %}
 7975 
 7976   ins_encode %{
 7977     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 7978   %}
 7979   ins_pipe(pipe_serial);
 7980 %}
 7981 
 7982 instruct unnecessary_membar_acquire() %{
 7983   predicate(unnecessary_acquire(n));
 7984   match(MemBarAcquire);
 7985   ins_cost(0);
 7986 
 7987   format %{ "membar_acquire (elided)" %}
 7988 
 7989   ins_encode %{
 7990     __ block_comment("membar_acquire (elided)");
 7991   %}
 7992 
 7993   ins_pipe(pipe_class_empty);
 7994 %}
 7995 
 7996 instruct membar_acquire() %{
 7997   match(MemBarAcquire);
 7998   ins_cost(VOLATILE_REF_COST);
 7999 
 8000   format %{ "membar_acquire\n\t"
 8001             "dmb ishld" %}
 8002 
 8003   ins_encode %{
 8004     __ block_comment("membar_acquire");
 8005     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8006   %}
 8007 
 8008   ins_pipe(pipe_serial);
 8009 %}
 8010 
 8011 
 8012 instruct membar_acquire_lock() %{
 8013   match(MemBarAcquireLock);
 8014   ins_cost(VOLATILE_REF_COST);
 8015 
 8016   format %{ "membar_acquire_lock (elided)" %}
 8017 
 8018   ins_encode %{
 8019     __ block_comment("membar_acquire_lock (elided)");
 8020   %}
 8021 
 8022   ins_pipe(pipe_serial);
 8023 %}
 8024 
 8025 instruct store_fence() %{
 8026   match(StoreFence);
 8027   ins_cost(VOLATILE_REF_COST);
 8028 
 8029   format %{ "store_fence" %}
 8030 
 8031   ins_encode %{
 8032     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8033   %}
 8034   ins_pipe(pipe_serial);
 8035 %}
 8036 
 8037 instruct unnecessary_membar_release() %{
 8038   predicate(unnecessary_release(n));
 8039   match(MemBarRelease);
 8040   ins_cost(0);
 8041 
 8042   format %{ "membar_release (elided)" %}
 8043 
 8044   ins_encode %{
 8045     __ block_comment("membar_release (elided)");
 8046   %}
 8047   ins_pipe(pipe_serial);
 8048 %}
 8049 
 8050 instruct membar_release() %{
 8051   match(MemBarRelease);
 8052   ins_cost(VOLATILE_REF_COST);
 8053 
 8054   format %{ "membar_release\n\t"
 8055             "dmb ishst\n\tdmb ishld" %}
 8056 
 8057   ins_encode %{
 8058     __ block_comment("membar_release");
 8059     // These will be merged if AlwaysMergeDMB is enabled.
 8060     __ membar(Assembler::StoreStore);
 8061     __ membar(Assembler::LoadStore);
 8062   %}
 8063   ins_pipe(pipe_serial);
 8064 %}
 8065 
 8066 instruct membar_storestore() %{
 8067   match(MemBarStoreStore);
 8068   match(StoreStoreFence);
 8069   ins_cost(VOLATILE_REF_COST);
 8070 
 8071   format %{ "MEMBAR-store-store" %}
 8072 
 8073   ins_encode %{
 8074     __ membar(Assembler::StoreStore);
 8075   %}
 8076   ins_pipe(pipe_serial);
 8077 %}
 8078 
 8079 instruct membar_release_lock() %{
 8080   match(MemBarReleaseLock);
 8081   ins_cost(VOLATILE_REF_COST);
 8082 
 8083   format %{ "membar_release_lock (elided)" %}
 8084 
 8085   ins_encode %{
 8086     __ block_comment("membar_release_lock (elided)");
 8087   %}
 8088 
 8089   ins_pipe(pipe_serial);
 8090 %}
 8091 
 8092 instruct unnecessary_membar_volatile() %{
 8093   predicate(unnecessary_volatile(n));
 8094   match(MemBarVolatile);
 8095   ins_cost(0);
 8096 
 8097   format %{ "membar_volatile (elided)" %}
 8098 
 8099   ins_encode %{
 8100     __ block_comment("membar_volatile (elided)");
 8101   %}
 8102 
 8103   ins_pipe(pipe_serial);
 8104 %}
 8105 
 8106 instruct membar_volatile() %{
 8107   match(MemBarVolatile);
 8108   ins_cost(VOLATILE_REF_COST*100);
 8109 
 8110   format %{ "membar_volatile\n\t"
 8111              "dmb ish"%}
 8112 
 8113   ins_encode %{
 8114     __ block_comment("membar_volatile");
 8115     __ membar(Assembler::StoreLoad);
 8116   %}
 8117 
 8118   ins_pipe(pipe_serial);
 8119 %}
 8120 
 8121 // ============================================================================
 8122 // Cast/Convert Instructions
 8123 
 8124 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8125   match(Set dst (CastX2P src));
 8126 
 8127   ins_cost(INSN_COST);
 8128   format %{ "mov $dst, $src\t# long -> ptr" %}
 8129 
 8130   ins_encode %{
 8131     if ($dst$$reg != $src$$reg) {
 8132       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8133     }
 8134   %}
 8135 
 8136   ins_pipe(ialu_reg);
 8137 %}
 8138 
 8139 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8140   match(Set dst (CastP2X src));
 8141 
 8142   ins_cost(INSN_COST);
 8143   format %{ "mov $dst, $src\t# ptr -> long" %}
 8144 
 8145   ins_encode %{
 8146     if ($dst$$reg != $src$$reg) {
 8147       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8148     }
 8149   %}
 8150 
 8151   ins_pipe(ialu_reg);
 8152 %}
 8153 
 8154 // Convert oop into int for vectors alignment masking
 8155 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8156   match(Set dst (ConvL2I (CastP2X src)));
 8157 
 8158   ins_cost(INSN_COST);
 8159   format %{ "movw $dst, $src\t# ptr -> int" %}
 8160   ins_encode %{
 8161     __ movw($dst$$Register, $src$$Register);
 8162   %}
 8163 
 8164   ins_pipe(ialu_reg);
 8165 %}
 8166 
 8167 // Convert compressed oop into int for vectors alignment masking
 8168 // in case of 32bit oops (heap < 4Gb).
 8169 instruct convN2I(iRegINoSp dst, iRegN src)
 8170 %{
 8171   predicate(CompressedOops::shift() == 0);
 8172   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8173 
 8174   ins_cost(INSN_COST);
 8175   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8176   ins_encode %{
 8177     __ movw($dst$$Register, $src$$Register);
 8178   %}
 8179 
 8180   ins_pipe(ialu_reg);
 8181 %}
 8182 
 8183 
 8184 // Convert oop pointer into compressed form
 8185 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8186   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8187   match(Set dst (EncodeP src));
 8188   effect(KILL cr);
 8189   ins_cost(INSN_COST * 3);
 8190   format %{ "encode_heap_oop $dst, $src" %}
 8191   ins_encode %{
 8192     Register s = $src$$Register;
 8193     Register d = $dst$$Register;
 8194     __ encode_heap_oop(d, s);
 8195   %}
 8196   ins_pipe(ialu_reg);
 8197 %}
 8198 
 8199 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8200   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8201   match(Set dst (EncodeP src));
 8202   ins_cost(INSN_COST * 3);
 8203   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8204   ins_encode %{
 8205     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8206   %}
 8207   ins_pipe(ialu_reg);
 8208 %}
 8209 
 8210 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8211   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8212             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8213   match(Set dst (DecodeN src));
 8214   ins_cost(INSN_COST * 3);
 8215   format %{ "decode_heap_oop $dst, $src" %}
 8216   ins_encode %{
 8217     Register s = $src$$Register;
 8218     Register d = $dst$$Register;
 8219     __ decode_heap_oop(d, s);
 8220   %}
 8221   ins_pipe(ialu_reg);
 8222 %}
 8223 
 8224 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8225   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8226             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8227   match(Set dst (DecodeN src));
 8228   ins_cost(INSN_COST * 3);
 8229   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8230   ins_encode %{
 8231     Register s = $src$$Register;
 8232     Register d = $dst$$Register;
 8233     __ decode_heap_oop_not_null(d, s);
 8234   %}
 8235   ins_pipe(ialu_reg);
 8236 %}
 8237 
 8238 // n.b. AArch64 implementations of encode_klass_not_null and
 8239 // decode_klass_not_null do not modify the flags register so, unlike
 8240 // Intel, we don't kill CR as a side effect here
 8241 
 8242 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8243   match(Set dst (EncodePKlass src));
 8244 
 8245   ins_cost(INSN_COST * 3);
 8246   format %{ "encode_klass_not_null $dst,$src" %}
 8247 
 8248   ins_encode %{
 8249     Register src_reg = as_Register($src$$reg);
 8250     Register dst_reg = as_Register($dst$$reg);
 8251     __ encode_klass_not_null(dst_reg, src_reg);
 8252   %}
 8253 
 8254    ins_pipe(ialu_reg);
 8255 %}
 8256 
 8257 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8258   match(Set dst (DecodeNKlass src));
 8259 
 8260   ins_cost(INSN_COST * 3);
 8261   format %{ "decode_klass_not_null $dst,$src" %}
 8262 
 8263   ins_encode %{
 8264     Register src_reg = as_Register($src$$reg);
 8265     Register dst_reg = as_Register($dst$$reg);
 8266     if (dst_reg != src_reg) {
 8267       __ decode_klass_not_null(dst_reg, src_reg);
 8268     } else {
 8269       __ decode_klass_not_null(dst_reg);
 8270     }
 8271   %}
 8272 
 8273    ins_pipe(ialu_reg);
 8274 %}
 8275 
 8276 instruct checkCastPP(iRegPNoSp dst)
 8277 %{
 8278   match(Set dst (CheckCastPP dst));
 8279 
 8280   size(0);
 8281   format %{ "# checkcastPP of $dst" %}
 8282   ins_encode(/* empty encoding */);
 8283   ins_pipe(pipe_class_empty);
 8284 %}
 8285 
 8286 instruct castPP(iRegPNoSp dst)
 8287 %{
 8288   match(Set dst (CastPP dst));
 8289 
 8290   size(0);
 8291   format %{ "# castPP of $dst" %}
 8292   ins_encode(/* empty encoding */);
 8293   ins_pipe(pipe_class_empty);
 8294 %}
 8295 
 8296 instruct castII(iRegI dst)
 8297 %{
 8298   predicate(VerifyConstraintCasts == 0);
 8299   match(Set dst (CastII dst));
 8300 
 8301   size(0);
 8302   format %{ "# castII of $dst" %}
 8303   ins_encode(/* empty encoding */);
 8304   ins_cost(0);
 8305   ins_pipe(pipe_class_empty);
 8306 %}
 8307 
 8308 instruct castII_checked(iRegI dst, rFlagsReg cr)
 8309 %{
 8310   predicate(VerifyConstraintCasts > 0);
 8311   match(Set dst (CastII dst));
 8312   effect(KILL cr);
 8313 
 8314   format %{ "# castII_checked of $dst" %}
 8315   ins_encode %{
 8316     __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
 8317   %}
 8318   ins_pipe(pipe_slow);
 8319 %}
 8320 
 8321 instruct castLL(iRegL dst)
 8322 %{
 8323   predicate(VerifyConstraintCasts == 0);
 8324   match(Set dst (CastLL dst));
 8325 
 8326   size(0);
 8327   format %{ "# castLL of $dst" %}
 8328   ins_encode(/* empty encoding */);
 8329   ins_cost(0);
 8330   ins_pipe(pipe_class_empty);
 8331 %}
 8332 
 8333 instruct castLL_checked(iRegL dst, rFlagsReg cr)
 8334 %{
 8335   predicate(VerifyConstraintCasts > 0);
 8336   match(Set dst (CastLL dst));
 8337   effect(KILL cr);
 8338 
 8339   format %{ "# castLL_checked of $dst" %}
 8340   ins_encode %{
 8341     __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
 8342   %}
 8343   ins_pipe(pipe_slow);
 8344 %}
 8345 
 8346 instruct castHH(vRegF dst)
 8347 %{
 8348   match(Set dst (CastHH dst));
 8349   size(0);
 8350   format %{ "# castHH of $dst" %}
 8351   ins_encode(/* empty encoding */);
 8352   ins_cost(0);
 8353   ins_pipe(pipe_class_empty);
 8354 %}
 8355 
 8356 instruct castFF(vRegF dst)
 8357 %{
 8358   match(Set dst (CastFF dst));
 8359 
 8360   size(0);
 8361   format %{ "# castFF of $dst" %}
 8362   ins_encode(/* empty encoding */);
 8363   ins_cost(0);
 8364   ins_pipe(pipe_class_empty);
 8365 %}
 8366 
 8367 instruct castDD(vRegD dst)
 8368 %{
 8369   match(Set dst (CastDD dst));
 8370 
 8371   size(0);
 8372   format %{ "# castDD of $dst" %}
 8373   ins_encode(/* empty encoding */);
 8374   ins_cost(0);
 8375   ins_pipe(pipe_class_empty);
 8376 %}
 8377 
 8378 instruct castVV(vReg dst)
 8379 %{
 8380   match(Set dst (CastVV dst));
 8381 
 8382   size(0);
 8383   format %{ "# castVV of $dst" %}
 8384   ins_encode(/* empty encoding */);
 8385   ins_cost(0);
 8386   ins_pipe(pipe_class_empty);
 8387 %}
 8388 
 8389 instruct castVVMask(pRegGov dst)
 8390 %{
 8391   match(Set dst (CastVV dst));
 8392 
 8393   size(0);
 8394   format %{ "# castVV of $dst" %}
 8395   ins_encode(/* empty encoding */);
 8396   ins_cost(0);
 8397   ins_pipe(pipe_class_empty);
 8398 %}
 8399 
 8400 // ============================================================================
 8401 // Atomic operation instructions
 8402 //
 8403 
 8404 // standard CompareAndSwapX when we are using barriers
 8405 // these have higher priority than the rules selected by a predicate
 8406 
 8407 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 8408 // can't match them
 8409 
 8410 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8411 
 8412   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8413   ins_cost(2 * VOLATILE_REF_COST);
 8414 
 8415   effect(KILL cr);
 8416 
 8417   format %{
 8418     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8419     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8420   %}
 8421 
 8422   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 8423             aarch64_enc_cset_eq(res));
 8424 
 8425   ins_pipe(pipe_slow);
 8426 %}
 8427 
 8428 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8429 
 8430   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8431   ins_cost(2 * VOLATILE_REF_COST);
 8432 
 8433   effect(KILL cr);
 8434 
 8435   format %{
 8436     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8437     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8438   %}
 8439 
 8440   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 8441             aarch64_enc_cset_eq(res));
 8442 
 8443   ins_pipe(pipe_slow);
 8444 %}
 8445 
 8446 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8447 
 8448   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8449   ins_cost(2 * VOLATILE_REF_COST);
 8450 
 8451   effect(KILL cr);
 8452 
 8453  format %{
 8454     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8455     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8456  %}
 8457 
 8458  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8459             aarch64_enc_cset_eq(res));
 8460 
 8461   ins_pipe(pipe_slow);
 8462 %}
 8463 
 8464 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8465 
 8466   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8467   ins_cost(2 * VOLATILE_REF_COST);
 8468 
 8469   effect(KILL cr);
 8470 
 8471  format %{
 8472     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8473     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8474  %}
 8475 
 8476  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8477             aarch64_enc_cset_eq(res));
 8478 
 8479   ins_pipe(pipe_slow);
 8480 %}
 8481 
 8482 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8483 
 8484   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8485   predicate(n->as_LoadStore()->barrier_data() == 0);
 8486   ins_cost(2 * VOLATILE_REF_COST);
 8487 
 8488   effect(KILL cr);
 8489 
 8490  format %{
 8491     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8492     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8493  %}
 8494 
 8495  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8496             aarch64_enc_cset_eq(res));
 8497 
 8498   ins_pipe(pipe_slow);
 8499 %}
 8500 
 8501 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8502 
 8503   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8504   predicate(n->as_LoadStore()->barrier_data() == 0);
 8505   ins_cost(2 * VOLATILE_REF_COST);
 8506 
 8507   effect(KILL cr);
 8508 
 8509  format %{
 8510     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8511     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8512  %}
 8513 
 8514  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8515             aarch64_enc_cset_eq(res));
 8516 
 8517   ins_pipe(pipe_slow);
 8518 %}
 8519 
 8520 // alternative CompareAndSwapX when we are eliding barriers
 8521 
 8522 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8523 
 8524   predicate(needs_acquiring_load_exclusive(n));
 8525   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8526   ins_cost(VOLATILE_REF_COST);
 8527 
 8528   effect(KILL cr);
 8529 
 8530   format %{
 8531     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8532     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8533   %}
 8534 
 8535   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 8536             aarch64_enc_cset_eq(res));
 8537 
 8538   ins_pipe(pipe_slow);
 8539 %}
 8540 
 8541 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8542 
 8543   predicate(needs_acquiring_load_exclusive(n));
 8544   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8545   ins_cost(VOLATILE_REF_COST);
 8546 
 8547   effect(KILL cr);
 8548 
 8549   format %{
 8550     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8551     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8552   %}
 8553 
 8554   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 8555             aarch64_enc_cset_eq(res));
 8556 
 8557   ins_pipe(pipe_slow);
 8558 %}
 8559 
 8560 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8561 
 8562   predicate(needs_acquiring_load_exclusive(n));
 8563   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8564   ins_cost(VOLATILE_REF_COST);
 8565 
 8566   effect(KILL cr);
 8567 
 8568  format %{
 8569     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8570     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8571  %}
 8572 
 8573  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8574             aarch64_enc_cset_eq(res));
 8575 
 8576   ins_pipe(pipe_slow);
 8577 %}
 8578 
 8579 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8580 
 8581   predicate(needs_acquiring_load_exclusive(n));
 8582   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8583   ins_cost(VOLATILE_REF_COST);
 8584 
 8585   effect(KILL cr);
 8586 
 8587  format %{
 8588     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8589     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8590  %}
 8591 
 8592  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8593             aarch64_enc_cset_eq(res));
 8594 
 8595   ins_pipe(pipe_slow);
 8596 %}
 8597 
 8598 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8599 
 8600   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8601   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8602   ins_cost(VOLATILE_REF_COST);
 8603 
 8604   effect(KILL cr);
 8605 
 8606  format %{
 8607     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8608     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8609  %}
 8610 
 8611  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8612             aarch64_enc_cset_eq(res));
 8613 
 8614   ins_pipe(pipe_slow);
 8615 %}
 8616 
 8617 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8618 
 8619   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 8620   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8621   ins_cost(VOLATILE_REF_COST);
 8622 
 8623   effect(KILL cr);
 8624 
 8625  format %{
 8626     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8627     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8628  %}
 8629 
 8630  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8631             aarch64_enc_cset_eq(res));
 8632 
 8633   ins_pipe(pipe_slow);
 8634 %}
 8635 
 8636 
 8637 // ---------------------------------------------------------------------
 8638 
 8639 // BEGIN This section of the file is automatically generated. Do not edit --------------
 8640 
 8641 // Sundry CAS operations.  Note that release is always true,
 8642 // regardless of the memory ordering of the CAS.  This is because we
 8643 // need the volatile case to be sequentially consistent but there is
 8644 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 8645 // can't check the type of memory ordering here, so we always emit a
 8646 // STLXR.
 8647 
 8648 // This section is generated from cas.m4
 8649 
 8650 
 8651 // This pattern is generated automatically from cas.m4.
 8652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8653 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8654   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8655   ins_cost(2 * VOLATILE_REF_COST);
 8656   effect(TEMP_DEF res, KILL cr);
 8657   format %{
 8658     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8659   %}
 8660   ins_encode %{
 8661     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8662                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8663                /*weak*/ false, $res$$Register);
 8664     __ sxtbw($res$$Register, $res$$Register);
 8665   %}
 8666   ins_pipe(pipe_slow);
 8667 %}
 8668 
 8669 // This pattern is generated automatically from cas.m4.
 8670 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8671 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8672   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 8673   ins_cost(2 * VOLATILE_REF_COST);
 8674   effect(TEMP_DEF res, KILL cr);
 8675   format %{
 8676     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8677   %}
 8678   ins_encode %{
 8679     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8680                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 8681                /*weak*/ false, $res$$Register);
 8682     __ sxthw($res$$Register, $res$$Register);
 8683   %}
 8684   ins_pipe(pipe_slow);
 8685 %}
 8686 
 8687 // This pattern is generated automatically from cas.m4.
 8688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8689 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8690   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 8691   ins_cost(2 * VOLATILE_REF_COST);
 8692   effect(TEMP_DEF res, KILL cr);
 8693   format %{
 8694     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8695   %}
 8696   ins_encode %{
 8697     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8698                Assembler::word, /*acquire*/ false, /*release*/ true,
 8699                /*weak*/ false, $res$$Register);
 8700   %}
 8701   ins_pipe(pipe_slow);
 8702 %}
 8703 
 8704 // This pattern is generated automatically from cas.m4.
 8705 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8706 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8707   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 8708   ins_cost(2 * VOLATILE_REF_COST);
 8709   effect(TEMP_DEF res, KILL cr);
 8710   format %{
 8711     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8712   %}
 8713   ins_encode %{
 8714     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8715                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8716                /*weak*/ false, $res$$Register);
 8717   %}
 8718   ins_pipe(pipe_slow);
 8719 %}
 8720 
 8721 // This pattern is generated automatically from cas.m4.
 8722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8723 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8724   predicate(n->as_LoadStore()->barrier_data() == 0);
 8725   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 8726   ins_cost(2 * VOLATILE_REF_COST);
 8727   effect(TEMP_DEF res, KILL cr);
 8728   format %{
 8729     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8730   %}
 8731   ins_encode %{
 8732     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8733                Assembler::word, /*acquire*/ false, /*release*/ true,
 8734                /*weak*/ false, $res$$Register);
 8735   %}
 8736   ins_pipe(pipe_slow);
 8737 %}
 8738 
 8739 // This pattern is generated automatically from cas.m4.
 8740 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8741 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8742   predicate(n->as_LoadStore()->barrier_data() == 0);
 8743   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 8744   ins_cost(2 * VOLATILE_REF_COST);
 8745   effect(TEMP_DEF res, KILL cr);
 8746   format %{
 8747     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8748   %}
 8749   ins_encode %{
 8750     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8751                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8752                /*weak*/ false, $res$$Register);
 8753   %}
 8754   ins_pipe(pipe_slow);
 8755 %}
 8756 
 8757 // This pattern is generated automatically from cas.m4.
 8758 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8759 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8760   predicate(needs_acquiring_load_exclusive(n));
 8761   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8762   ins_cost(VOLATILE_REF_COST);
 8763   effect(TEMP_DEF res, KILL cr);
 8764   format %{
 8765     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8766   %}
 8767   ins_encode %{
 8768     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8769                Assembler::byte, /*acquire*/ true, /*release*/ true,
 8770                /*weak*/ false, $res$$Register);
 8771     __ sxtbw($res$$Register, $res$$Register);
 8772   %}
 8773   ins_pipe(pipe_slow);
 8774 %}
 8775 
 8776 // This pattern is generated automatically from cas.m4.
 8777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8778 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8779   predicate(needs_acquiring_load_exclusive(n));
 8780   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 8781   ins_cost(VOLATILE_REF_COST);
 8782   effect(TEMP_DEF res, KILL cr);
 8783   format %{
 8784     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8785   %}
 8786   ins_encode %{
 8787     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8788                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 8789                /*weak*/ false, $res$$Register);
 8790     __ sxthw($res$$Register, $res$$Register);
 8791   %}
 8792   ins_pipe(pipe_slow);
 8793 %}
 8794 
 8795 // This pattern is generated automatically from cas.m4.
 8796 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8797 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8798   predicate(needs_acquiring_load_exclusive(n));
 8799   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 8800   ins_cost(VOLATILE_REF_COST);
 8801   effect(TEMP_DEF res, KILL cr);
 8802   format %{
 8803     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8804   %}
 8805   ins_encode %{
 8806     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8807                Assembler::word, /*acquire*/ true, /*release*/ true,
 8808                /*weak*/ false, $res$$Register);
 8809   %}
 8810   ins_pipe(pipe_slow);
 8811 %}
 8812 
 8813 // This pattern is generated automatically from cas.m4.
 8814 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8815 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8816   predicate(needs_acquiring_load_exclusive(n));
 8817   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 8818   ins_cost(VOLATILE_REF_COST);
 8819   effect(TEMP_DEF res, KILL cr);
 8820   format %{
 8821     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8822   %}
 8823   ins_encode %{
 8824     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8825                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8826                /*weak*/ false, $res$$Register);
 8827   %}
 8828   ins_pipe(pipe_slow);
 8829 %}
 8830 
 8831 // This pattern is generated automatically from cas.m4.
 8832 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8833 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8834   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 8835   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 8836   ins_cost(VOLATILE_REF_COST);
 8837   effect(TEMP_DEF res, KILL cr);
 8838   format %{
 8839     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8840   %}
 8841   ins_encode %{
 8842     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8843                Assembler::word, /*acquire*/ true, /*release*/ true,
 8844                /*weak*/ false, $res$$Register);
 8845   %}
 8846   ins_pipe(pipe_slow);
 8847 %}
 8848 
 8849 // This pattern is generated automatically from cas.m4.
 8850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8851 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8852   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8853   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 8854   ins_cost(VOLATILE_REF_COST);
 8855   effect(TEMP_DEF res, KILL cr);
 8856   format %{
 8857     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8858   %}
 8859   ins_encode %{
 8860     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8861                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8862                /*weak*/ false, $res$$Register);
 8863   %}
 8864   ins_pipe(pipe_slow);
 8865 %}
 8866 
 8867 // This pattern is generated automatically from cas.m4.
 8868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8869 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8870   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 8871   ins_cost(2 * VOLATILE_REF_COST);
 8872   effect(KILL cr);
 8873   format %{
 8874     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8875     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8876   %}
 8877   ins_encode %{
 8878     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8879                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8880                /*weak*/ true, noreg);
 8881     __ csetw($res$$Register, Assembler::EQ);
 8882   %}
 8883   ins_pipe(pipe_slow);
 8884 %}
 8885 
 8886 // This pattern is generated automatically from cas.m4.
 8887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8888 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8889   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 8890   ins_cost(2 * VOLATILE_REF_COST);
 8891   effect(KILL cr);
 8892   format %{
 8893     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8894     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8895   %}
 8896   ins_encode %{
 8897     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8898                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 8899                /*weak*/ true, noreg);
 8900     __ csetw($res$$Register, Assembler::EQ);
 8901   %}
 8902   ins_pipe(pipe_slow);
 8903 %}
 8904 
 8905 // This pattern is generated automatically from cas.m4.
 8906 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8907 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8908   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 8909   ins_cost(2 * VOLATILE_REF_COST);
 8910   effect(KILL cr);
 8911   format %{
 8912     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8913     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8914   %}
 8915   ins_encode %{
 8916     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8917                Assembler::word, /*acquire*/ false, /*release*/ true,
 8918                /*weak*/ true, noreg);
 8919     __ csetw($res$$Register, Assembler::EQ);
 8920   %}
 8921   ins_pipe(pipe_slow);
 8922 %}
 8923 
 8924 // This pattern is generated automatically from cas.m4.
 8925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8926 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8927   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 8928   ins_cost(2 * VOLATILE_REF_COST);
 8929   effect(KILL cr);
 8930   format %{
 8931     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8932     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8933   %}
 8934   ins_encode %{
 8935     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8936                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8937                /*weak*/ true, noreg);
 8938     __ csetw($res$$Register, Assembler::EQ);
 8939   %}
 8940   ins_pipe(pipe_slow);
 8941 %}
 8942 
 8943 // This pattern is generated automatically from cas.m4.
 8944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8945 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8946   predicate(n->as_LoadStore()->barrier_data() == 0);
 8947   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 8948   ins_cost(2 * VOLATILE_REF_COST);
 8949   effect(KILL cr);
 8950   format %{
 8951     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8952     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8953   %}
 8954   ins_encode %{
 8955     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8956                Assembler::word, /*acquire*/ false, /*release*/ true,
 8957                /*weak*/ true, noreg);
 8958     __ csetw($res$$Register, Assembler::EQ);
 8959   %}
 8960   ins_pipe(pipe_slow);
 8961 %}
 8962 
 8963 // This pattern is generated automatically from cas.m4.
 8964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8965 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8966   predicate(n->as_LoadStore()->barrier_data() == 0);
 8967   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 8968   ins_cost(2 * VOLATILE_REF_COST);
 8969   effect(KILL cr);
 8970   format %{
 8971     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8972     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8973   %}
 8974   ins_encode %{
 8975     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8976                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8977                /*weak*/ true, noreg);
 8978     __ csetw($res$$Register, Assembler::EQ);
 8979   %}
 8980   ins_pipe(pipe_slow);
 8981 %}
 8982 
 8983 // This pattern is generated automatically from cas.m4.
 8984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8985 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8986   predicate(needs_acquiring_load_exclusive(n));
 8987   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 8988   ins_cost(VOLATILE_REF_COST);
 8989   effect(KILL cr);
 8990   format %{
 8991     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8992     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8993   %}
 8994   ins_encode %{
 8995     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8996                Assembler::byte, /*acquire*/ true, /*release*/ true,
 8997                /*weak*/ true, noreg);
 8998     __ csetw($res$$Register, Assembler::EQ);
 8999   %}
 9000   ins_pipe(pipe_slow);
 9001 %}
 9002 
 9003 // This pattern is generated automatically from cas.m4.
 9004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9005 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9006   predicate(needs_acquiring_load_exclusive(n));
 9007   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9008   ins_cost(VOLATILE_REF_COST);
 9009   effect(KILL cr);
 9010   format %{
 9011     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9012     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9013   %}
 9014   ins_encode %{
 9015     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9016                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9017                /*weak*/ true, noreg);
 9018     __ csetw($res$$Register, Assembler::EQ);
 9019   %}
 9020   ins_pipe(pipe_slow);
 9021 %}
 9022 
 9023 // This pattern is generated automatically from cas.m4.
 9024 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9025 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9026   predicate(needs_acquiring_load_exclusive(n));
 9027   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9028   ins_cost(VOLATILE_REF_COST);
 9029   effect(KILL cr);
 9030   format %{
 9031     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9032     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9033   %}
 9034   ins_encode %{
 9035     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9036                Assembler::word, /*acquire*/ true, /*release*/ true,
 9037                /*weak*/ true, noreg);
 9038     __ csetw($res$$Register, Assembler::EQ);
 9039   %}
 9040   ins_pipe(pipe_slow);
 9041 %}
 9042 
 9043 // This pattern is generated automatically from cas.m4.
 9044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9045 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9046   predicate(needs_acquiring_load_exclusive(n));
 9047   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9048   ins_cost(VOLATILE_REF_COST);
 9049   effect(KILL cr);
 9050   format %{
 9051     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9052     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9053   %}
 9054   ins_encode %{
 9055     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9056                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9057                /*weak*/ true, noreg);
 9058     __ csetw($res$$Register, Assembler::EQ);
 9059   %}
 9060   ins_pipe(pipe_slow);
 9061 %}
 9062 
 9063 // This pattern is generated automatically from cas.m4.
 9064 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9065 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9066   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 9067   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9068   ins_cost(VOLATILE_REF_COST);
 9069   effect(KILL cr);
 9070   format %{
 9071     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9072     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9073   %}
 9074   ins_encode %{
 9075     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9076                Assembler::word, /*acquire*/ true, /*release*/ true,
 9077                /*weak*/ true, noreg);
 9078     __ csetw($res$$Register, Assembler::EQ);
 9079   %}
 9080   ins_pipe(pipe_slow);
 9081 %}
 9082 
 9083 // This pattern is generated automatically from cas.m4.
 9084 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9085 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9086   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9087   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9088   ins_cost(VOLATILE_REF_COST);
 9089   effect(KILL cr);
 9090   format %{
 9091     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9092     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9093   %}
 9094   ins_encode %{
 9095     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9096                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9097                /*weak*/ true, noreg);
 9098     __ csetw($res$$Register, Assembler::EQ);
 9099   %}
 9100   ins_pipe(pipe_slow);
 9101 %}
 9102 
 9103 // END This section of the file is automatically generated. Do not edit --------------
 9104 // ---------------------------------------------------------------------
 9105 
 9106 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9107   match(Set prev (GetAndSetI mem newv));
 9108   ins_cost(2 * VOLATILE_REF_COST);
 9109   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9110   ins_encode %{
 9111     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9112   %}
 9113   ins_pipe(pipe_serial);
 9114 %}
 9115 
 9116 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9117   match(Set prev (GetAndSetL mem newv));
 9118   ins_cost(2 * VOLATILE_REF_COST);
 9119   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9120   ins_encode %{
 9121     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9122   %}
 9123   ins_pipe(pipe_serial);
 9124 %}
 9125 
 9126 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9127   predicate(n->as_LoadStore()->barrier_data() == 0);
 9128   match(Set prev (GetAndSetN mem newv));
 9129   ins_cost(2 * VOLATILE_REF_COST);
 9130   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9131   ins_encode %{
 9132     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9133   %}
 9134   ins_pipe(pipe_serial);
 9135 %}
 9136 
 9137 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9138   predicate(n->as_LoadStore()->barrier_data() == 0);
 9139   match(Set prev (GetAndSetP mem newv));
 9140   ins_cost(2 * VOLATILE_REF_COST);
 9141   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9142   ins_encode %{
 9143     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9144   %}
 9145   ins_pipe(pipe_serial);
 9146 %}
 9147 
 9148 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9149   predicate(needs_acquiring_load_exclusive(n));
 9150   match(Set prev (GetAndSetI mem newv));
 9151   ins_cost(VOLATILE_REF_COST);
 9152   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9153   ins_encode %{
 9154     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9155   %}
 9156   ins_pipe(pipe_serial);
 9157 %}
 9158 
 9159 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9160   predicate(needs_acquiring_load_exclusive(n));
 9161   match(Set prev (GetAndSetL mem newv));
 9162   ins_cost(VOLATILE_REF_COST);
 9163   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9164   ins_encode %{
 9165     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9166   %}
 9167   ins_pipe(pipe_serial);
 9168 %}
 9169 
 9170 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9171   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 9172   match(Set prev (GetAndSetN mem newv));
 9173   ins_cost(VOLATILE_REF_COST);
 9174   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9175   ins_encode %{
 9176     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9177   %}
 9178   ins_pipe(pipe_serial);
 9179 %}
 9180 
 9181 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9182   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9183   match(Set prev (GetAndSetP mem newv));
 9184   ins_cost(VOLATILE_REF_COST);
 9185   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9186   ins_encode %{
 9187     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9188   %}
 9189   ins_pipe(pipe_serial);
 9190 %}
 9191 
 9192 
 9193 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9194   match(Set newval (GetAndAddL mem incr));
 9195   ins_cost(2 * VOLATILE_REF_COST + 1);
 9196   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9197   ins_encode %{
 9198     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9199   %}
 9200   ins_pipe(pipe_serial);
 9201 %}
 9202 
 9203 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9204   predicate(n->as_LoadStore()->result_not_used());
 9205   match(Set dummy (GetAndAddL mem incr));
 9206   ins_cost(2 * VOLATILE_REF_COST);
 9207   format %{ "get_and_addL [$mem], $incr" %}
 9208   ins_encode %{
 9209     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9210   %}
 9211   ins_pipe(pipe_serial);
 9212 %}
 9213 
 9214 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9215   match(Set newval (GetAndAddL mem incr));
 9216   ins_cost(2 * VOLATILE_REF_COST + 1);
 9217   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9218   ins_encode %{
 9219     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9220   %}
 9221   ins_pipe(pipe_serial);
 9222 %}
 9223 
 9224 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9225   predicate(n->as_LoadStore()->result_not_used());
 9226   match(Set dummy (GetAndAddL mem incr));
 9227   ins_cost(2 * VOLATILE_REF_COST);
 9228   format %{ "get_and_addL [$mem], $incr" %}
 9229   ins_encode %{
 9230     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9231   %}
 9232   ins_pipe(pipe_serial);
 9233 %}
 9234 
 9235 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9236   match(Set newval (GetAndAddI mem incr));
 9237   ins_cost(2 * VOLATILE_REF_COST + 1);
 9238   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9239   ins_encode %{
 9240     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9241   %}
 9242   ins_pipe(pipe_serial);
 9243 %}
 9244 
 9245 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9246   predicate(n->as_LoadStore()->result_not_used());
 9247   match(Set dummy (GetAndAddI mem incr));
 9248   ins_cost(2 * VOLATILE_REF_COST);
 9249   format %{ "get_and_addI [$mem], $incr" %}
 9250   ins_encode %{
 9251     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9252   %}
 9253   ins_pipe(pipe_serial);
 9254 %}
 9255 
 9256 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9257   match(Set newval (GetAndAddI mem incr));
 9258   ins_cost(2 * VOLATILE_REF_COST + 1);
 9259   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9260   ins_encode %{
 9261     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9262   %}
 9263   ins_pipe(pipe_serial);
 9264 %}
 9265 
 9266 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9267   predicate(n->as_LoadStore()->result_not_used());
 9268   match(Set dummy (GetAndAddI mem incr));
 9269   ins_cost(2 * VOLATILE_REF_COST);
 9270   format %{ "get_and_addI [$mem], $incr" %}
 9271   ins_encode %{
 9272     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9273   %}
 9274   ins_pipe(pipe_serial);
 9275 %}
 9276 
 9277 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9278   predicate(needs_acquiring_load_exclusive(n));
 9279   match(Set newval (GetAndAddL mem incr));
 9280   ins_cost(VOLATILE_REF_COST + 1);
 9281   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9282   ins_encode %{
 9283     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9284   %}
 9285   ins_pipe(pipe_serial);
 9286 %}
 9287 
 9288 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9289   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9290   match(Set dummy (GetAndAddL mem incr));
 9291   ins_cost(VOLATILE_REF_COST);
 9292   format %{ "get_and_addL_acq [$mem], $incr" %}
 9293   ins_encode %{
 9294     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9295   %}
 9296   ins_pipe(pipe_serial);
 9297 %}
 9298 
 9299 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9300   predicate(needs_acquiring_load_exclusive(n));
 9301   match(Set newval (GetAndAddL mem incr));
 9302   ins_cost(VOLATILE_REF_COST + 1);
 9303   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9304   ins_encode %{
 9305     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9306   %}
 9307   ins_pipe(pipe_serial);
 9308 %}
 9309 
 9310 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9311   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9312   match(Set dummy (GetAndAddL mem incr));
 9313   ins_cost(VOLATILE_REF_COST);
 9314   format %{ "get_and_addL_acq [$mem], $incr" %}
 9315   ins_encode %{
 9316     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9317   %}
 9318   ins_pipe(pipe_serial);
 9319 %}
 9320 
 9321 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9322   predicate(needs_acquiring_load_exclusive(n));
 9323   match(Set newval (GetAndAddI mem incr));
 9324   ins_cost(VOLATILE_REF_COST + 1);
 9325   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9326   ins_encode %{
 9327     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9328   %}
 9329   ins_pipe(pipe_serial);
 9330 %}
 9331 
 9332 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9333   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9334   match(Set dummy (GetAndAddI mem incr));
 9335   ins_cost(VOLATILE_REF_COST);
 9336   format %{ "get_and_addI_acq [$mem], $incr" %}
 9337   ins_encode %{
 9338     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9339   %}
 9340   ins_pipe(pipe_serial);
 9341 %}
 9342 
 9343 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9344   predicate(needs_acquiring_load_exclusive(n));
 9345   match(Set newval (GetAndAddI mem incr));
 9346   ins_cost(VOLATILE_REF_COST + 1);
 9347   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9348   ins_encode %{
 9349     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9350   %}
 9351   ins_pipe(pipe_serial);
 9352 %}
 9353 
 9354 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9355   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9356   match(Set dummy (GetAndAddI mem incr));
 9357   ins_cost(VOLATILE_REF_COST);
 9358   format %{ "get_and_addI_acq [$mem], $incr" %}
 9359   ins_encode %{
 9360     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9361   %}
 9362   ins_pipe(pipe_serial);
 9363 %}
 9364 
 9365 // Manifest a CmpU result in an integer register.
 9366 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9367 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
 9368 %{
 9369   match(Set dst (CmpU3 src1 src2));
 9370   effect(KILL flags);
 9371 
 9372   ins_cost(INSN_COST * 3);
 9373   format %{
 9374       "cmpw $src1, $src2\n\t"
 9375       "csetw $dst, ne\n\t"
 9376       "cnegw $dst, lo\t# CmpU3(reg)"
 9377   %}
 9378   ins_encode %{
 9379     __ cmpw($src1$$Register, $src2$$Register);
 9380     __ csetw($dst$$Register, Assembler::NE);
 9381     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9382   %}
 9383 
 9384   ins_pipe(pipe_class_default);
 9385 %}
 9386 
 9387 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
 9388 %{
 9389   match(Set dst (CmpU3 src1 src2));
 9390   effect(KILL flags);
 9391 
 9392   ins_cost(INSN_COST * 3);
 9393   format %{
 9394       "subsw zr, $src1, $src2\n\t"
 9395       "csetw $dst, ne\n\t"
 9396       "cnegw $dst, lo\t# CmpU3(imm)"
 9397   %}
 9398   ins_encode %{
 9399     __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
 9400     __ csetw($dst$$Register, Assembler::NE);
 9401     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9402   %}
 9403 
 9404   ins_pipe(pipe_class_default);
 9405 %}
 9406 
 9407 // Manifest a CmpUL result in an integer register.
 9408 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9409 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9410 %{
 9411   match(Set dst (CmpUL3 src1 src2));
 9412   effect(KILL flags);
 9413 
 9414   ins_cost(INSN_COST * 3);
 9415   format %{
 9416       "cmp $src1, $src2\n\t"
 9417       "csetw $dst, ne\n\t"
 9418       "cnegw $dst, lo\t# CmpUL3(reg)"
 9419   %}
 9420   ins_encode %{
 9421     __ cmp($src1$$Register, $src2$$Register);
 9422     __ csetw($dst$$Register, Assembler::NE);
 9423     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9424   %}
 9425 
 9426   ins_pipe(pipe_class_default);
 9427 %}
 9428 
 9429 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9430 %{
 9431   match(Set dst (CmpUL3 src1 src2));
 9432   effect(KILL flags);
 9433 
 9434   ins_cost(INSN_COST * 3);
 9435   format %{
 9436       "subs zr, $src1, $src2\n\t"
 9437       "csetw $dst, ne\n\t"
 9438       "cnegw $dst, lo\t# CmpUL3(imm)"
 9439   %}
 9440   ins_encode %{
 9441     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9442     __ csetw($dst$$Register, Assembler::NE);
 9443     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9444   %}
 9445 
 9446   ins_pipe(pipe_class_default);
 9447 %}
 9448 
 9449 // Manifest a CmpL result in an integer register.
 9450 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9451 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9452 %{
 9453   match(Set dst (CmpL3 src1 src2));
 9454   effect(KILL flags);
 9455 
 9456   ins_cost(INSN_COST * 3);
 9457   format %{
 9458       "cmp $src1, $src2\n\t"
 9459       "csetw $dst, ne\n\t"
 9460       "cnegw $dst, lt\t# CmpL3(reg)"
 9461   %}
 9462   ins_encode %{
 9463     __ cmp($src1$$Register, $src2$$Register);
 9464     __ csetw($dst$$Register, Assembler::NE);
 9465     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9466   %}
 9467 
 9468   ins_pipe(pipe_class_default);
 9469 %}
 9470 
 9471 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9472 %{
 9473   match(Set dst (CmpL3 src1 src2));
 9474   effect(KILL flags);
 9475 
 9476   ins_cost(INSN_COST * 3);
 9477   format %{
 9478       "subs zr, $src1, $src2\n\t"
 9479       "csetw $dst, ne\n\t"
 9480       "cnegw $dst, lt\t# CmpL3(imm)"
 9481   %}
 9482   ins_encode %{
 9483     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9484     __ csetw($dst$$Register, Assembler::NE);
 9485     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9486   %}
 9487 
 9488   ins_pipe(pipe_class_default);
 9489 %}
 9490 
 9491 // ============================================================================
 9492 // Conditional Move Instructions
 9493 
 9494 // n.b. we have identical rules for both a signed compare op (cmpOp)
 9495 // and an unsigned compare op (cmpOpU). it would be nice if we could
 9496 // define an op class which merged both inputs and use it to type the
 9497 // argument to a single rule. unfortunatelyt his fails because the
 9498 // opclass does not live up to the COND_INTER interface of its
 9499 // component operands. When the generic code tries to negate the
 9500 // operand it ends up running the generci Machoper::negate method
 9501 // which throws a ShouldNotHappen. So, we have to provide two flavours
 9502 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
 9503 
 9504 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9505   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9506 
 9507   ins_cost(INSN_COST * 2);
 9508   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
 9509 
 9510   ins_encode %{
 9511     __ cselw(as_Register($dst$$reg),
 9512              as_Register($src2$$reg),
 9513              as_Register($src1$$reg),
 9514              (Assembler::Condition)$cmp$$cmpcode);
 9515   %}
 9516 
 9517   ins_pipe(icond_reg_reg);
 9518 %}
 9519 
 9520 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9521   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9522 
 9523   ins_cost(INSN_COST * 2);
 9524   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
 9525 
 9526   ins_encode %{
 9527     __ cselw(as_Register($dst$$reg),
 9528              as_Register($src2$$reg),
 9529              as_Register($src1$$reg),
 9530              (Assembler::Condition)$cmp$$cmpcode);
 9531   %}
 9532 
 9533   ins_pipe(icond_reg_reg);
 9534 %}
 9535 
 9536 // special cases where one arg is zero
 9537 
 9538 // n.b. this is selected in preference to the rule above because it
 9539 // avoids loading constant 0 into a source register
 9540 
 9541 // TODO
 9542 // we ought only to be able to cull one of these variants as the ideal
 9543 // transforms ought always to order the zero consistently (to left/right?)
 9544 
 9545 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9546   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9547 
 9548   ins_cost(INSN_COST * 2);
 9549   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
 9550 
 9551   ins_encode %{
 9552     __ cselw(as_Register($dst$$reg),
 9553              as_Register($src$$reg),
 9554              zr,
 9555              (Assembler::Condition)$cmp$$cmpcode);
 9556   %}
 9557 
 9558   ins_pipe(icond_reg);
 9559 %}
 9560 
 9561 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9562   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9563 
 9564   ins_cost(INSN_COST * 2);
 9565   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
 9566 
 9567   ins_encode %{
 9568     __ cselw(as_Register($dst$$reg),
 9569              as_Register($src$$reg),
 9570              zr,
 9571              (Assembler::Condition)$cmp$$cmpcode);
 9572   %}
 9573 
 9574   ins_pipe(icond_reg);
 9575 %}
 9576 
 9577 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9578   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9579 
 9580   ins_cost(INSN_COST * 2);
 9581   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
 9582 
 9583   ins_encode %{
 9584     __ cselw(as_Register($dst$$reg),
 9585              zr,
 9586              as_Register($src$$reg),
 9587              (Assembler::Condition)$cmp$$cmpcode);
 9588   %}
 9589 
 9590   ins_pipe(icond_reg);
 9591 %}
 9592 
 9593 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9594   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9595 
 9596   ins_cost(INSN_COST * 2);
 9597   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
 9598 
 9599   ins_encode %{
 9600     __ cselw(as_Register($dst$$reg),
 9601              zr,
 9602              as_Register($src$$reg),
 9603              (Assembler::Condition)$cmp$$cmpcode);
 9604   %}
 9605 
 9606   ins_pipe(icond_reg);
 9607 %}
 9608 
 9609 // special case for creating a boolean 0 or 1
 9610 
 9611 // n.b. this is selected in preference to the rule above because it
 9612 // avoids loading constants 0 and 1 into a source register
 9613 
 9614 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9615   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9616 
 9617   ins_cost(INSN_COST * 2);
 9618   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
 9619 
 9620   ins_encode %{
 9621     // equivalently
 9622     // cset(as_Register($dst$$reg),
 9623     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9624     __ csincw(as_Register($dst$$reg),
 9625              zr,
 9626              zr,
 9627              (Assembler::Condition)$cmp$$cmpcode);
 9628   %}
 9629 
 9630   ins_pipe(icond_none);
 9631 %}
 9632 
 9633 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9634   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9635 
 9636   ins_cost(INSN_COST * 2);
 9637   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
 9638 
 9639   ins_encode %{
 9640     // equivalently
 9641     // cset(as_Register($dst$$reg),
 9642     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9643     __ csincw(as_Register($dst$$reg),
 9644              zr,
 9645              zr,
 9646              (Assembler::Condition)$cmp$$cmpcode);
 9647   %}
 9648 
 9649   ins_pipe(icond_none);
 9650 %}
 9651 
 9652 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9653   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9654 
 9655   ins_cost(INSN_COST * 2);
 9656   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
 9657 
 9658   ins_encode %{
 9659     __ csel(as_Register($dst$$reg),
 9660             as_Register($src2$$reg),
 9661             as_Register($src1$$reg),
 9662             (Assembler::Condition)$cmp$$cmpcode);
 9663   %}
 9664 
 9665   ins_pipe(icond_reg_reg);
 9666 %}
 9667 
 9668 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9669   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9670 
 9671   ins_cost(INSN_COST * 2);
 9672   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
 9673 
 9674   ins_encode %{
 9675     __ csel(as_Register($dst$$reg),
 9676             as_Register($src2$$reg),
 9677             as_Register($src1$$reg),
 9678             (Assembler::Condition)$cmp$$cmpcode);
 9679   %}
 9680 
 9681   ins_pipe(icond_reg_reg);
 9682 %}
 9683 
 9684 // special cases where one arg is zero
 9685 
 9686 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9687   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9688 
 9689   ins_cost(INSN_COST * 2);
 9690   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
 9691 
 9692   ins_encode %{
 9693     __ csel(as_Register($dst$$reg),
 9694             zr,
 9695             as_Register($src$$reg),
 9696             (Assembler::Condition)$cmp$$cmpcode);
 9697   %}
 9698 
 9699   ins_pipe(icond_reg);
 9700 %}
 9701 
 9702 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9703   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9704 
 9705   ins_cost(INSN_COST * 2);
 9706   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
 9707 
 9708   ins_encode %{
 9709     __ csel(as_Register($dst$$reg),
 9710             zr,
 9711             as_Register($src$$reg),
 9712             (Assembler::Condition)$cmp$$cmpcode);
 9713   %}
 9714 
 9715   ins_pipe(icond_reg);
 9716 %}
 9717 
 9718 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9719   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9720 
 9721   ins_cost(INSN_COST * 2);
 9722   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
 9723 
 9724   ins_encode %{
 9725     __ csel(as_Register($dst$$reg),
 9726             as_Register($src$$reg),
 9727             zr,
 9728             (Assembler::Condition)$cmp$$cmpcode);
 9729   %}
 9730 
 9731   ins_pipe(icond_reg);
 9732 %}
 9733 
 9734 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9735   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9736 
 9737   ins_cost(INSN_COST * 2);
 9738   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
 9739 
 9740   ins_encode %{
 9741     __ csel(as_Register($dst$$reg),
 9742             as_Register($src$$reg),
 9743             zr,
 9744             (Assembler::Condition)$cmp$$cmpcode);
 9745   %}
 9746 
 9747   ins_pipe(icond_reg);
 9748 %}
 9749 
 9750 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
 9751   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
 9752 
 9753   ins_cost(INSN_COST * 2);
 9754   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
 9755 
 9756   ins_encode %{
 9757     __ csel(as_Register($dst$$reg),
 9758             as_Register($src2$$reg),
 9759             as_Register($src1$$reg),
 9760             (Assembler::Condition)$cmp$$cmpcode);
 9761   %}
 9762 
 9763   ins_pipe(icond_reg_reg);
 9764 %}
 9765 
 9766 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
 9767   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
 9768 
 9769   ins_cost(INSN_COST * 2);
 9770   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
 9771 
 9772   ins_encode %{
 9773     __ csel(as_Register($dst$$reg),
 9774             as_Register($src2$$reg),
 9775             as_Register($src1$$reg),
 9776             (Assembler::Condition)$cmp$$cmpcode);
 9777   %}
 9778 
 9779   ins_pipe(icond_reg_reg);
 9780 %}
 9781 
 9782 // special cases where one arg is zero
 9783 
 9784 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
 9785   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
 9786 
 9787   ins_cost(INSN_COST * 2);
 9788   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
 9789 
 9790   ins_encode %{
 9791     __ csel(as_Register($dst$$reg),
 9792             zr,
 9793             as_Register($src$$reg),
 9794             (Assembler::Condition)$cmp$$cmpcode);
 9795   %}
 9796 
 9797   ins_pipe(icond_reg);
 9798 %}
 9799 
 9800 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
 9801   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
 9802 
 9803   ins_cost(INSN_COST * 2);
 9804   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
 9805 
 9806   ins_encode %{
 9807     __ csel(as_Register($dst$$reg),
 9808             zr,
 9809             as_Register($src$$reg),
 9810             (Assembler::Condition)$cmp$$cmpcode);
 9811   %}
 9812 
 9813   ins_pipe(icond_reg);
 9814 %}
 9815 
 9816 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
 9817   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
 9818 
 9819   ins_cost(INSN_COST * 2);
 9820   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
 9821 
 9822   ins_encode %{
 9823     __ csel(as_Register($dst$$reg),
 9824             as_Register($src$$reg),
 9825             zr,
 9826             (Assembler::Condition)$cmp$$cmpcode);
 9827   %}
 9828 
 9829   ins_pipe(icond_reg);
 9830 %}
 9831 
 9832 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
 9833   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
 9834 
 9835   ins_cost(INSN_COST * 2);
 9836   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
 9837 
 9838   ins_encode %{
 9839     __ csel(as_Register($dst$$reg),
 9840             as_Register($src$$reg),
 9841             zr,
 9842             (Assembler::Condition)$cmp$$cmpcode);
 9843   %}
 9844 
 9845   ins_pipe(icond_reg);
 9846 %}
 9847 
 9848 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
 9849   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
 9850 
 9851   ins_cost(INSN_COST * 2);
 9852   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
 9853 
 9854   ins_encode %{
 9855     __ cselw(as_Register($dst$$reg),
 9856              as_Register($src2$$reg),
 9857              as_Register($src1$$reg),
 9858              (Assembler::Condition)$cmp$$cmpcode);
 9859   %}
 9860 
 9861   ins_pipe(icond_reg_reg);
 9862 %}
 9863 
 9864 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
 9865   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
 9866 
 9867   ins_cost(INSN_COST * 2);
 9868   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
 9869 
 9870   ins_encode %{
 9871     __ cselw(as_Register($dst$$reg),
 9872              as_Register($src2$$reg),
 9873              as_Register($src1$$reg),
 9874              (Assembler::Condition)$cmp$$cmpcode);
 9875   %}
 9876 
 9877   ins_pipe(icond_reg_reg);
 9878 %}
 9879 
 9880 // special cases where one arg is zero
 9881 
 9882 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
 9883   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
 9884 
 9885   ins_cost(INSN_COST * 2);
 9886   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
 9887 
 9888   ins_encode %{
 9889     __ cselw(as_Register($dst$$reg),
 9890              zr,
 9891              as_Register($src$$reg),
 9892              (Assembler::Condition)$cmp$$cmpcode);
 9893   %}
 9894 
 9895   ins_pipe(icond_reg);
 9896 %}
 9897 
 9898 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
 9899   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
 9900 
 9901   ins_cost(INSN_COST * 2);
 9902   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
 9903 
 9904   ins_encode %{
 9905     __ cselw(as_Register($dst$$reg),
 9906              zr,
 9907              as_Register($src$$reg),
 9908              (Assembler::Condition)$cmp$$cmpcode);
 9909   %}
 9910 
 9911   ins_pipe(icond_reg);
 9912 %}
 9913 
 9914 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
 9915   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
 9916 
 9917   ins_cost(INSN_COST * 2);
 9918   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
 9919 
 9920   ins_encode %{
 9921     __ cselw(as_Register($dst$$reg),
 9922              as_Register($src$$reg),
 9923              zr,
 9924              (Assembler::Condition)$cmp$$cmpcode);
 9925   %}
 9926 
 9927   ins_pipe(icond_reg);
 9928 %}
 9929 
 9930 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
 9931   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
 9932 
 9933   ins_cost(INSN_COST * 2);
 9934   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
 9935 
 9936   ins_encode %{
 9937     __ cselw(as_Register($dst$$reg),
 9938              as_Register($src$$reg),
 9939              zr,
 9940              (Assembler::Condition)$cmp$$cmpcode);
 9941   %}
 9942 
 9943   ins_pipe(icond_reg);
 9944 %}
 9945 
 9946 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
 9947 %{
 9948   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
 9949 
 9950   ins_cost(INSN_COST * 3);
 9951 
 9952   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
 9953   ins_encode %{
 9954     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9955     __ fcsels(as_FloatRegister($dst$$reg),
 9956               as_FloatRegister($src2$$reg),
 9957               as_FloatRegister($src1$$reg),
 9958               cond);
 9959   %}
 9960 
 9961   ins_pipe(fp_cond_reg_reg_s);
 9962 %}
 9963 
 9964 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
 9965 %{
 9966   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
 9967 
 9968   ins_cost(INSN_COST * 3);
 9969 
 9970   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
 9971   ins_encode %{
 9972     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9973     __ fcsels(as_FloatRegister($dst$$reg),
 9974               as_FloatRegister($src2$$reg),
 9975               as_FloatRegister($src1$$reg),
 9976               cond);
 9977   %}
 9978 
 9979   ins_pipe(fp_cond_reg_reg_s);
 9980 %}
 9981 
 9982 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
 9983 %{
 9984   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
 9985 
 9986   ins_cost(INSN_COST * 3);
 9987 
 9988   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
 9989   ins_encode %{
 9990     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9991     __ fcseld(as_FloatRegister($dst$$reg),
 9992               as_FloatRegister($src2$$reg),
 9993               as_FloatRegister($src1$$reg),
 9994               cond);
 9995   %}
 9996 
 9997   ins_pipe(fp_cond_reg_reg_d);
 9998 %}
 9999 
10000 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10001 %{
10002   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10003 
10004   ins_cost(INSN_COST * 3);
10005 
10006   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10007   ins_encode %{
10008     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10009     __ fcseld(as_FloatRegister($dst$$reg),
10010               as_FloatRegister($src2$$reg),
10011               as_FloatRegister($src1$$reg),
10012               cond);
10013   %}
10014 
10015   ins_pipe(fp_cond_reg_reg_d);
10016 %}
10017 
10018 // ============================================================================
10019 // Arithmetic Instructions
10020 //
10021 
10022 // Integer Addition
10023 
10024 // TODO
10025 // these currently employ operations which do not set CR and hence are
10026 // not flagged as killing CR but we would like to isolate the cases
10027 // where we want to set flags from those where we don't. need to work
10028 // out how to do that.
10029 
10030 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10031   match(Set dst (AddI src1 src2));
10032 
10033   ins_cost(INSN_COST);
10034   format %{ "addw  $dst, $src1, $src2" %}
10035 
10036   ins_encode %{
10037     __ addw(as_Register($dst$$reg),
10038             as_Register($src1$$reg),
10039             as_Register($src2$$reg));
10040   %}
10041 
10042   ins_pipe(ialu_reg_reg);
10043 %}
10044 
10045 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10046   match(Set dst (AddI src1 src2));
10047 
10048   ins_cost(INSN_COST);
10049   format %{ "addw $dst, $src1, $src2" %}
10050 
10051   // use opcode to indicate that this is an add not a sub
10052   opcode(0x0);
10053 
10054   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10055 
10056   ins_pipe(ialu_reg_imm);
10057 %}
10058 
10059 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10060   match(Set dst (AddI (ConvL2I src1) src2));
10061 
10062   ins_cost(INSN_COST);
10063   format %{ "addw $dst, $src1, $src2" %}
10064 
10065   // use opcode to indicate that this is an add not a sub
10066   opcode(0x0);
10067 
10068   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10069 
10070   ins_pipe(ialu_reg_imm);
10071 %}
10072 
10073 // Pointer Addition
10074 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
10075   match(Set dst (AddP src1 src2));
10076 
10077   ins_cost(INSN_COST);
10078   format %{ "add $dst, $src1, $src2\t# ptr" %}
10079 
10080   ins_encode %{
10081     __ add(as_Register($dst$$reg),
10082            as_Register($src1$$reg),
10083            as_Register($src2$$reg));
10084   %}
10085 
10086   ins_pipe(ialu_reg_reg);
10087 %}
10088 
10089 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
10090   match(Set dst (AddP src1 (ConvI2L src2)));
10091 
10092   ins_cost(1.9 * INSN_COST);
10093   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10094 
10095   ins_encode %{
10096     __ add(as_Register($dst$$reg),
10097            as_Register($src1$$reg),
10098            as_Register($src2$$reg), ext::sxtw);
10099   %}
10100 
10101   ins_pipe(ialu_reg_reg);
10102 %}
10103 
10104 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
10105   match(Set dst (AddP src1 (LShiftL src2 scale)));
10106 
10107   ins_cost(1.9 * INSN_COST);
10108   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10109 
10110   ins_encode %{
10111     __ lea(as_Register($dst$$reg),
10112            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10113                    Address::lsl($scale$$constant)));
10114   %}
10115 
10116   ins_pipe(ialu_reg_reg_shift);
10117 %}
10118 
10119 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
10120   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10121 
10122   ins_cost(1.9 * INSN_COST);
10123   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10124 
10125   ins_encode %{
10126     __ lea(as_Register($dst$$reg),
10127            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10128                    Address::sxtw($scale$$constant)));
10129   %}
10130 
10131   ins_pipe(ialu_reg_reg_shift);
10132 %}
10133 
10134 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10135   match(Set dst (LShiftL (ConvI2L src) scale));
10136 
10137   ins_cost(INSN_COST);
10138   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10139 
10140   ins_encode %{
10141     __ sbfiz(as_Register($dst$$reg),
10142           as_Register($src$$reg),
10143           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10144   %}
10145 
10146   ins_pipe(ialu_reg_shift);
10147 %}
10148 
10149 // Pointer Immediate Addition
10150 // n.b. this needs to be more expensive than using an indirect memory
10151 // operand
10152 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10153   match(Set dst (AddP src1 src2));
10154 
10155   ins_cost(INSN_COST);
10156   format %{ "add $dst, $src1, $src2\t# ptr" %}
10157 
10158   // use opcode to indicate that this is an add not a sub
10159   opcode(0x0);
10160 
10161   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10162 
10163   ins_pipe(ialu_reg_imm);
10164 %}
10165 
10166 // Long Addition
10167 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10168 
10169   match(Set dst (AddL src1 src2));
10170 
10171   ins_cost(INSN_COST);
10172   format %{ "add  $dst, $src1, $src2" %}
10173 
10174   ins_encode %{
10175     __ add(as_Register($dst$$reg),
10176            as_Register($src1$$reg),
10177            as_Register($src2$$reg));
10178   %}
10179 
10180   ins_pipe(ialu_reg_reg);
10181 %}
10182 
10183 // No constant pool entries requiredLong Immediate Addition.
10184 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10185   match(Set dst (AddL src1 src2));
10186 
10187   ins_cost(INSN_COST);
10188   format %{ "add $dst, $src1, $src2" %}
10189 
10190   // use opcode to indicate that this is an add not a sub
10191   opcode(0x0);
10192 
10193   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10194 
10195   ins_pipe(ialu_reg_imm);
10196 %}
10197 
10198 // Integer Subtraction
10199 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10200   match(Set dst (SubI src1 src2));
10201 
10202   ins_cost(INSN_COST);
10203   format %{ "subw  $dst, $src1, $src2" %}
10204 
10205   ins_encode %{
10206     __ subw(as_Register($dst$$reg),
10207             as_Register($src1$$reg),
10208             as_Register($src2$$reg));
10209   %}
10210 
10211   ins_pipe(ialu_reg_reg);
10212 %}
10213 
10214 // Immediate Subtraction
10215 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10216   match(Set dst (SubI src1 src2));
10217 
10218   ins_cost(INSN_COST);
10219   format %{ "subw $dst, $src1, $src2" %}
10220 
10221   // use opcode to indicate that this is a sub not an add
10222   opcode(0x1);
10223 
10224   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10225 
10226   ins_pipe(ialu_reg_imm);
10227 %}
10228 
10229 // Long Subtraction
10230 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10231 
10232   match(Set dst (SubL src1 src2));
10233 
10234   ins_cost(INSN_COST);
10235   format %{ "sub  $dst, $src1, $src2" %}
10236 
10237   ins_encode %{
10238     __ sub(as_Register($dst$$reg),
10239            as_Register($src1$$reg),
10240            as_Register($src2$$reg));
10241   %}
10242 
10243   ins_pipe(ialu_reg_reg);
10244 %}
10245 
10246 // No constant pool entries requiredLong Immediate Subtraction.
10247 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10248   match(Set dst (SubL src1 src2));
10249 
10250   ins_cost(INSN_COST);
10251   format %{ "sub$dst, $src1, $src2" %}
10252 
10253   // use opcode to indicate that this is a sub not an add
10254   opcode(0x1);
10255 
10256   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10257 
10258   ins_pipe(ialu_reg_imm);
10259 %}
10260 
10261 // Integer Negation (special case for sub)
10262 
10263 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10264   match(Set dst (SubI zero src));
10265 
10266   ins_cost(INSN_COST);
10267   format %{ "negw $dst, $src\t# int" %}
10268 
10269   ins_encode %{
10270     __ negw(as_Register($dst$$reg),
10271             as_Register($src$$reg));
10272   %}
10273 
10274   ins_pipe(ialu_reg);
10275 %}
10276 
10277 // Long Negation
10278 
10279 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10280   match(Set dst (SubL zero src));
10281 
10282   ins_cost(INSN_COST);
10283   format %{ "neg $dst, $src\t# long" %}
10284 
10285   ins_encode %{
10286     __ neg(as_Register($dst$$reg),
10287            as_Register($src$$reg));
10288   %}
10289 
10290   ins_pipe(ialu_reg);
10291 %}
10292 
10293 // Integer Multiply
10294 
10295 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10296   match(Set dst (MulI src1 src2));
10297 
10298   ins_cost(INSN_COST * 3);
10299   format %{ "mulw  $dst, $src1, $src2" %}
10300 
10301   ins_encode %{
10302     __ mulw(as_Register($dst$$reg),
10303             as_Register($src1$$reg),
10304             as_Register($src2$$reg));
10305   %}
10306 
10307   ins_pipe(imul_reg_reg);
10308 %}
10309 
10310 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10311   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10312 
10313   ins_cost(INSN_COST * 3);
10314   format %{ "smull  $dst, $src1, $src2" %}
10315 
10316   ins_encode %{
10317     __ smull(as_Register($dst$$reg),
10318              as_Register($src1$$reg),
10319              as_Register($src2$$reg));
10320   %}
10321 
10322   ins_pipe(imul_reg_reg);
10323 %}
10324 
10325 // Long Multiply
10326 
10327 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10328   match(Set dst (MulL src1 src2));
10329 
10330   ins_cost(INSN_COST * 5);
10331   format %{ "mul  $dst, $src1, $src2" %}
10332 
10333   ins_encode %{
10334     __ mul(as_Register($dst$$reg),
10335            as_Register($src1$$reg),
10336            as_Register($src2$$reg));
10337   %}
10338 
10339   ins_pipe(lmul_reg_reg);
10340 %}
10341 
10342 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10343 %{
10344   match(Set dst (MulHiL src1 src2));
10345 
10346   ins_cost(INSN_COST * 7);
10347   format %{ "smulh   $dst, $src1, $src2\t# mulhi" %}
10348 
10349   ins_encode %{
10350     __ smulh(as_Register($dst$$reg),
10351              as_Register($src1$$reg),
10352              as_Register($src2$$reg));
10353   %}
10354 
10355   ins_pipe(lmul_reg_reg);
10356 %}
10357 
10358 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10359 %{
10360   match(Set dst (UMulHiL src1 src2));
10361 
10362   ins_cost(INSN_COST * 7);
10363   format %{ "umulh   $dst, $src1, $src2\t# umulhi" %}
10364 
10365   ins_encode %{
10366     __ umulh(as_Register($dst$$reg),
10367              as_Register($src1$$reg),
10368              as_Register($src2$$reg));
10369   %}
10370 
10371   ins_pipe(lmul_reg_reg);
10372 %}
10373 
10374 // Combined Integer Multiply & Add/Sub
10375 
10376 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10377   match(Set dst (AddI src3 (MulI src1 src2)));
10378 
10379   ins_cost(INSN_COST * 3);
10380   format %{ "madd  $dst, $src1, $src2, $src3" %}
10381 
10382   ins_encode %{
10383     __ maddw(as_Register($dst$$reg),
10384              as_Register($src1$$reg),
10385              as_Register($src2$$reg),
10386              as_Register($src3$$reg));
10387   %}
10388 
10389   ins_pipe(imac_reg_reg);
10390 %}
10391 
10392 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10393   match(Set dst (SubI src3 (MulI src1 src2)));
10394 
10395   ins_cost(INSN_COST * 3);
10396   format %{ "msub  $dst, $src1, $src2, $src3" %}
10397 
10398   ins_encode %{
10399     __ msubw(as_Register($dst$$reg),
10400              as_Register($src1$$reg),
10401              as_Register($src2$$reg),
10402              as_Register($src3$$reg));
10403   %}
10404 
10405   ins_pipe(imac_reg_reg);
10406 %}
10407 
10408 // Combined Integer Multiply & Neg
10409 
10410 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10411   match(Set dst (MulI (SubI zero src1) src2));
10412 
10413   ins_cost(INSN_COST * 3);
10414   format %{ "mneg  $dst, $src1, $src2" %}
10415 
10416   ins_encode %{
10417     __ mnegw(as_Register($dst$$reg),
10418              as_Register($src1$$reg),
10419              as_Register($src2$$reg));
10420   %}
10421 
10422   ins_pipe(imac_reg_reg);
10423 %}
10424 
10425 // Combined Long Multiply & Add/Sub
10426 
10427 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10428   match(Set dst (AddL src3 (MulL src1 src2)));
10429 
10430   ins_cost(INSN_COST * 5);
10431   format %{ "madd  $dst, $src1, $src2, $src3" %}
10432 
10433   ins_encode %{
10434     __ madd(as_Register($dst$$reg),
10435             as_Register($src1$$reg),
10436             as_Register($src2$$reg),
10437             as_Register($src3$$reg));
10438   %}
10439 
10440   ins_pipe(lmac_reg_reg);
10441 %}
10442 
10443 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10444   match(Set dst (SubL src3 (MulL src1 src2)));
10445 
10446   ins_cost(INSN_COST * 5);
10447   format %{ "msub  $dst, $src1, $src2, $src3" %}
10448 
10449   ins_encode %{
10450     __ msub(as_Register($dst$$reg),
10451             as_Register($src1$$reg),
10452             as_Register($src2$$reg),
10453             as_Register($src3$$reg));
10454   %}
10455 
10456   ins_pipe(lmac_reg_reg);
10457 %}
10458 
10459 // Combined Long Multiply & Neg
10460 
10461 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10462   match(Set dst (MulL (SubL zero src1) src2));
10463 
10464   ins_cost(INSN_COST * 5);
10465   format %{ "mneg  $dst, $src1, $src2" %}
10466 
10467   ins_encode %{
10468     __ mneg(as_Register($dst$$reg),
10469             as_Register($src1$$reg),
10470             as_Register($src2$$reg));
10471   %}
10472 
10473   ins_pipe(lmac_reg_reg);
10474 %}
10475 
10476 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10477 
10478 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10479   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10480 
10481   ins_cost(INSN_COST * 3);
10482   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10483 
10484   ins_encode %{
10485     __ smaddl(as_Register($dst$$reg),
10486               as_Register($src1$$reg),
10487               as_Register($src2$$reg),
10488               as_Register($src3$$reg));
10489   %}
10490 
10491   ins_pipe(imac_reg_reg);
10492 %}
10493 
10494 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10495   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10496 
10497   ins_cost(INSN_COST * 3);
10498   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10499 
10500   ins_encode %{
10501     __ smsubl(as_Register($dst$$reg),
10502               as_Register($src1$$reg),
10503               as_Register($src2$$reg),
10504               as_Register($src3$$reg));
10505   %}
10506 
10507   ins_pipe(imac_reg_reg);
10508 %}
10509 
10510 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10511   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10512 
10513   ins_cost(INSN_COST * 3);
10514   format %{ "smnegl  $dst, $src1, $src2" %}
10515 
10516   ins_encode %{
10517     __ smnegl(as_Register($dst$$reg),
10518               as_Register($src1$$reg),
10519               as_Register($src2$$reg));
10520   %}
10521 
10522   ins_pipe(imac_reg_reg);
10523 %}
10524 
10525 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10526 
10527 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10528   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10529 
10530   ins_cost(INSN_COST * 5);
10531   format %{ "mulw  rscratch1, $src1, $src2\n\t"
10532             "maddw $dst, $src3, $src4, rscratch1" %}
10533 
10534   ins_encode %{
10535     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10536     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10537 
10538   ins_pipe(imac_reg_reg);
10539 %}
10540 
10541 // Integer Divide
10542 
10543 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10544   match(Set dst (DivI src1 src2));
10545 
10546   ins_cost(INSN_COST * 19);
10547   format %{ "sdivw  $dst, $src1, $src2" %}
10548 
10549   ins_encode(aarch64_enc_divw(dst, src1, src2));
10550   ins_pipe(idiv_reg_reg);
10551 %}
10552 
10553 // Long Divide
10554 
10555 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10556   match(Set dst (DivL src1 src2));
10557 
10558   ins_cost(INSN_COST * 35);
10559   format %{ "sdiv   $dst, $src1, $src2" %}
10560 
10561   ins_encode(aarch64_enc_div(dst, src1, src2));
10562   ins_pipe(ldiv_reg_reg);
10563 %}
10564 
10565 // Integer Remainder
10566 
10567 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10568   match(Set dst (ModI src1 src2));
10569 
10570   ins_cost(INSN_COST * 22);
10571   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
10572             "msubw  $dst, rscratch1, $src2, $src1" %}
10573 
10574   ins_encode(aarch64_enc_modw(dst, src1, src2));
10575   ins_pipe(idiv_reg_reg);
10576 %}
10577 
10578 // Long Remainder
10579 
10580 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10581   match(Set dst (ModL src1 src2));
10582 
10583   ins_cost(INSN_COST * 38);
10584   format %{ "sdiv   rscratch1, $src1, $src2\n"
10585             "msub   $dst, rscratch1, $src2, $src1" %}
10586 
10587   ins_encode(aarch64_enc_mod(dst, src1, src2));
10588   ins_pipe(ldiv_reg_reg);
10589 %}
10590 
10591 // Unsigned Integer Divide
10592 
10593 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10594   match(Set dst (UDivI src1 src2));
10595 
10596   ins_cost(INSN_COST * 19);
10597   format %{ "udivw  $dst, $src1, $src2" %}
10598 
10599   ins_encode %{
10600     __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10601   %}
10602 
10603   ins_pipe(idiv_reg_reg);
10604 %}
10605 
10606 //  Unsigned Long Divide
10607 
10608 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10609   match(Set dst (UDivL src1 src2));
10610 
10611   ins_cost(INSN_COST * 35);
10612   format %{ "udiv   $dst, $src1, $src2" %}
10613 
10614   ins_encode %{
10615     __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10616   %}
10617 
10618   ins_pipe(ldiv_reg_reg);
10619 %}
10620 
10621 // Unsigned Integer Remainder
10622 
10623 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10624   match(Set dst (UModI src1 src2));
10625 
10626   ins_cost(INSN_COST * 22);
10627   format %{ "udivw  rscratch1, $src1, $src2\n\t"
10628             "msubw  $dst, rscratch1, $src2, $src1" %}
10629 
10630   ins_encode %{
10631     __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10632     __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10633   %}
10634 
10635   ins_pipe(idiv_reg_reg);
10636 %}
10637 
10638 // Unsigned Long Remainder
10639 
10640 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10641   match(Set dst (UModL src1 src2));
10642 
10643   ins_cost(INSN_COST * 38);
10644   format %{ "udiv   rscratch1, $src1, $src2\n"
10645             "msub   $dst, rscratch1, $src2, $src1" %}
10646 
10647   ins_encode %{
10648     __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10649     __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10650   %}
10651 
10652   ins_pipe(ldiv_reg_reg);
10653 %}
10654 
10655 // Integer Shifts
10656 
10657 // Shift Left Register
10658 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10659   match(Set dst (LShiftI src1 src2));
10660 
10661   ins_cost(INSN_COST * 2);
10662   format %{ "lslvw  $dst, $src1, $src2" %}
10663 
10664   ins_encode %{
10665     __ lslvw(as_Register($dst$$reg),
10666              as_Register($src1$$reg),
10667              as_Register($src2$$reg));
10668   %}
10669 
10670   ins_pipe(ialu_reg_reg_vshift);
10671 %}
10672 
10673 // Shift Left Immediate
10674 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10675   match(Set dst (LShiftI src1 src2));
10676 
10677   ins_cost(INSN_COST);
10678   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10679 
10680   ins_encode %{
10681     __ lslw(as_Register($dst$$reg),
10682             as_Register($src1$$reg),
10683             $src2$$constant & 0x1f);
10684   %}
10685 
10686   ins_pipe(ialu_reg_shift);
10687 %}
10688 
10689 // Shift Right Logical Register
10690 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10691   match(Set dst (URShiftI src1 src2));
10692 
10693   ins_cost(INSN_COST * 2);
10694   format %{ "lsrvw  $dst, $src1, $src2" %}
10695 
10696   ins_encode %{
10697     __ lsrvw(as_Register($dst$$reg),
10698              as_Register($src1$$reg),
10699              as_Register($src2$$reg));
10700   %}
10701 
10702   ins_pipe(ialu_reg_reg_vshift);
10703 %}
10704 
10705 // Shift Right Logical Immediate
10706 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10707   match(Set dst (URShiftI src1 src2));
10708 
10709   ins_cost(INSN_COST);
10710   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10711 
10712   ins_encode %{
10713     __ lsrw(as_Register($dst$$reg),
10714             as_Register($src1$$reg),
10715             $src2$$constant & 0x1f);
10716   %}
10717 
10718   ins_pipe(ialu_reg_shift);
10719 %}
10720 
10721 // Shift Right Arithmetic Register
10722 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10723   match(Set dst (RShiftI src1 src2));
10724 
10725   ins_cost(INSN_COST * 2);
10726   format %{ "asrvw  $dst, $src1, $src2" %}
10727 
10728   ins_encode %{
10729     __ asrvw(as_Register($dst$$reg),
10730              as_Register($src1$$reg),
10731              as_Register($src2$$reg));
10732   %}
10733 
10734   ins_pipe(ialu_reg_reg_vshift);
10735 %}
10736 
10737 // Shift Right Arithmetic Immediate
10738 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10739   match(Set dst (RShiftI src1 src2));
10740 
10741   ins_cost(INSN_COST);
10742   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10743 
10744   ins_encode %{
10745     __ asrw(as_Register($dst$$reg),
10746             as_Register($src1$$reg),
10747             $src2$$constant & 0x1f);
10748   %}
10749 
10750   ins_pipe(ialu_reg_shift);
10751 %}
10752 
10753 // Combined Int Mask and Right Shift (using UBFM)
10754 // TODO
10755 
10756 // Long Shifts
10757 
10758 // Shift Left Register
10759 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10760   match(Set dst (LShiftL src1 src2));
10761 
10762   ins_cost(INSN_COST * 2);
10763   format %{ "lslv  $dst, $src1, $src2" %}
10764 
10765   ins_encode %{
10766     __ lslv(as_Register($dst$$reg),
10767             as_Register($src1$$reg),
10768             as_Register($src2$$reg));
10769   %}
10770 
10771   ins_pipe(ialu_reg_reg_vshift);
10772 %}
10773 
10774 // Shift Left Immediate
10775 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10776   match(Set dst (LShiftL src1 src2));
10777 
10778   ins_cost(INSN_COST);
10779   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10780 
10781   ins_encode %{
10782     __ lsl(as_Register($dst$$reg),
10783             as_Register($src1$$reg),
10784             $src2$$constant & 0x3f);
10785   %}
10786 
10787   ins_pipe(ialu_reg_shift);
10788 %}
10789 
10790 // Shift Right Logical Register
10791 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10792   match(Set dst (URShiftL src1 src2));
10793 
10794   ins_cost(INSN_COST * 2);
10795   format %{ "lsrv  $dst, $src1, $src2" %}
10796 
10797   ins_encode %{
10798     __ lsrv(as_Register($dst$$reg),
10799             as_Register($src1$$reg),
10800             as_Register($src2$$reg));
10801   %}
10802 
10803   ins_pipe(ialu_reg_reg_vshift);
10804 %}
10805 
10806 // Shift Right Logical Immediate
10807 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10808   match(Set dst (URShiftL src1 src2));
10809 
10810   ins_cost(INSN_COST);
10811   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10812 
10813   ins_encode %{
10814     __ lsr(as_Register($dst$$reg),
10815            as_Register($src1$$reg),
10816            $src2$$constant & 0x3f);
10817   %}
10818 
10819   ins_pipe(ialu_reg_shift);
10820 %}
10821 
10822 // A special-case pattern for card table stores.
10823 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10824   match(Set dst (URShiftL (CastP2X src1) src2));
10825 
10826   ins_cost(INSN_COST);
10827   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10828 
10829   ins_encode %{
10830     __ lsr(as_Register($dst$$reg),
10831            as_Register($src1$$reg),
10832            $src2$$constant & 0x3f);
10833   %}
10834 
10835   ins_pipe(ialu_reg_shift);
10836 %}
10837 
10838 // Shift Right Arithmetic Register
10839 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10840   match(Set dst (RShiftL src1 src2));
10841 
10842   ins_cost(INSN_COST * 2);
10843   format %{ "asrv  $dst, $src1, $src2" %}
10844 
10845   ins_encode %{
10846     __ asrv(as_Register($dst$$reg),
10847             as_Register($src1$$reg),
10848             as_Register($src2$$reg));
10849   %}
10850 
10851   ins_pipe(ialu_reg_reg_vshift);
10852 %}
10853 
10854 // Shift Right Arithmetic Immediate
10855 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10856   match(Set dst (RShiftL src1 src2));
10857 
10858   ins_cost(INSN_COST);
10859   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10860 
10861   ins_encode %{
10862     __ asr(as_Register($dst$$reg),
10863            as_Register($src1$$reg),
10864            $src2$$constant & 0x3f);
10865   %}
10866 
10867   ins_pipe(ialu_reg_shift);
10868 %}
10869 
10870 // BEGIN This section of the file is automatically generated. Do not edit --------------
10871 // This section is generated from aarch64_ad.m4
10872 
10873 // This pattern is automatically generated from aarch64_ad.m4
10874 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10875 instruct regL_not_reg(iRegLNoSp dst,
10876                          iRegL src1, immL_M1 m1,
10877                          rFlagsReg cr) %{
10878   match(Set dst (XorL src1 m1));
10879   ins_cost(INSN_COST);
10880   format %{ "eon  $dst, $src1, zr" %}
10881 
10882   ins_encode %{
10883     __ eon(as_Register($dst$$reg),
10884               as_Register($src1$$reg),
10885               zr,
10886               Assembler::LSL, 0);
10887   %}
10888 
10889   ins_pipe(ialu_reg);
10890 %}
10891 
10892 // This pattern is automatically generated from aarch64_ad.m4
10893 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10894 instruct regI_not_reg(iRegINoSp dst,
10895                          iRegIorL2I src1, immI_M1 m1,
10896                          rFlagsReg cr) %{
10897   match(Set dst (XorI src1 m1));
10898   ins_cost(INSN_COST);
10899   format %{ "eonw  $dst, $src1, zr" %}
10900 
10901   ins_encode %{
10902     __ eonw(as_Register($dst$$reg),
10903               as_Register($src1$$reg),
10904               zr,
10905               Assembler::LSL, 0);
10906   %}
10907 
10908   ins_pipe(ialu_reg);
10909 %}
10910 
10911 // This pattern is automatically generated from aarch64_ad.m4
10912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10913 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10914                               immI0 zero, iRegIorL2I src1, immI src2) %{
10915   match(Set dst (SubI zero (URShiftI src1 src2)));
10916 
10917   ins_cost(1.9 * INSN_COST);
10918   format %{ "negw  $dst, $src1, LSR $src2" %}
10919 
10920   ins_encode %{
10921     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10922             Assembler::LSR, $src2$$constant & 0x1f);
10923   %}
10924 
10925   ins_pipe(ialu_reg_shift);
10926 %}
10927 
10928 // This pattern is automatically generated from aarch64_ad.m4
10929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10930 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10931                               immI0 zero, iRegIorL2I src1, immI src2) %{
10932   match(Set dst (SubI zero (RShiftI src1 src2)));
10933 
10934   ins_cost(1.9 * INSN_COST);
10935   format %{ "negw  $dst, $src1, ASR $src2" %}
10936 
10937   ins_encode %{
10938     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10939             Assembler::ASR, $src2$$constant & 0x1f);
10940   %}
10941 
10942   ins_pipe(ialu_reg_shift);
10943 %}
10944 
10945 // This pattern is automatically generated from aarch64_ad.m4
10946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10947 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10948                               immI0 zero, iRegIorL2I src1, immI src2) %{
10949   match(Set dst (SubI zero (LShiftI src1 src2)));
10950 
10951   ins_cost(1.9 * INSN_COST);
10952   format %{ "negw  $dst, $src1, LSL $src2" %}
10953 
10954   ins_encode %{
10955     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10956             Assembler::LSL, $src2$$constant & 0x1f);
10957   %}
10958 
10959   ins_pipe(ialu_reg_shift);
10960 %}
10961 
10962 // This pattern is automatically generated from aarch64_ad.m4
10963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10964 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10965                               immL0 zero, iRegL src1, immI src2) %{
10966   match(Set dst (SubL zero (URShiftL src1 src2)));
10967 
10968   ins_cost(1.9 * INSN_COST);
10969   format %{ "neg  $dst, $src1, LSR $src2" %}
10970 
10971   ins_encode %{
10972     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10973             Assembler::LSR, $src2$$constant & 0x3f);
10974   %}
10975 
10976   ins_pipe(ialu_reg_shift);
10977 %}
10978 
10979 // This pattern is automatically generated from aarch64_ad.m4
10980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10981 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10982                               immL0 zero, iRegL src1, immI src2) %{
10983   match(Set dst (SubL zero (RShiftL src1 src2)));
10984 
10985   ins_cost(1.9 * INSN_COST);
10986   format %{ "neg  $dst, $src1, ASR $src2" %}
10987 
10988   ins_encode %{
10989     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10990             Assembler::ASR, $src2$$constant & 0x3f);
10991   %}
10992 
10993   ins_pipe(ialu_reg_shift);
10994 %}
10995 
10996 // This pattern is automatically generated from aarch64_ad.m4
10997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10998 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10999                               immL0 zero, iRegL src1, immI src2) %{
11000   match(Set dst (SubL zero (LShiftL src1 src2)));
11001 
11002   ins_cost(1.9 * INSN_COST);
11003   format %{ "neg  $dst, $src1, LSL $src2" %}
11004 
11005   ins_encode %{
11006     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11007             Assembler::LSL, $src2$$constant & 0x3f);
11008   %}
11009 
11010   ins_pipe(ialu_reg_shift);
11011 %}
11012 
11013 // This pattern is automatically generated from aarch64_ad.m4
11014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11015 instruct AndI_reg_not_reg(iRegINoSp dst,
11016                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11017   match(Set dst (AndI src1 (XorI src2 m1)));
11018   ins_cost(INSN_COST);
11019   format %{ "bicw  $dst, $src1, $src2" %}
11020 
11021   ins_encode %{
11022     __ bicw(as_Register($dst$$reg),
11023               as_Register($src1$$reg),
11024               as_Register($src2$$reg),
11025               Assembler::LSL, 0);
11026   %}
11027 
11028   ins_pipe(ialu_reg_reg);
11029 %}
11030 
11031 // This pattern is automatically generated from aarch64_ad.m4
11032 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11033 instruct AndL_reg_not_reg(iRegLNoSp dst,
11034                          iRegL src1, iRegL src2, immL_M1 m1) %{
11035   match(Set dst (AndL src1 (XorL src2 m1)));
11036   ins_cost(INSN_COST);
11037   format %{ "bic  $dst, $src1, $src2" %}
11038 
11039   ins_encode %{
11040     __ bic(as_Register($dst$$reg),
11041               as_Register($src1$$reg),
11042               as_Register($src2$$reg),
11043               Assembler::LSL, 0);
11044   %}
11045 
11046   ins_pipe(ialu_reg_reg);
11047 %}
11048 
11049 // This pattern is automatically generated from aarch64_ad.m4
11050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11051 instruct OrI_reg_not_reg(iRegINoSp dst,
11052                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11053   match(Set dst (OrI src1 (XorI src2 m1)));
11054   ins_cost(INSN_COST);
11055   format %{ "ornw  $dst, $src1, $src2" %}
11056 
11057   ins_encode %{
11058     __ ornw(as_Register($dst$$reg),
11059               as_Register($src1$$reg),
11060               as_Register($src2$$reg),
11061               Assembler::LSL, 0);
11062   %}
11063 
11064   ins_pipe(ialu_reg_reg);
11065 %}
11066 
11067 // This pattern is automatically generated from aarch64_ad.m4
11068 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11069 instruct OrL_reg_not_reg(iRegLNoSp dst,
11070                          iRegL src1, iRegL src2, immL_M1 m1) %{
11071   match(Set dst (OrL src1 (XorL src2 m1)));
11072   ins_cost(INSN_COST);
11073   format %{ "orn  $dst, $src1, $src2" %}
11074 
11075   ins_encode %{
11076     __ orn(as_Register($dst$$reg),
11077               as_Register($src1$$reg),
11078               as_Register($src2$$reg),
11079               Assembler::LSL, 0);
11080   %}
11081 
11082   ins_pipe(ialu_reg_reg);
11083 %}
11084 
11085 // This pattern is automatically generated from aarch64_ad.m4
11086 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11087 instruct XorI_reg_not_reg(iRegINoSp dst,
11088                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11089   match(Set dst (XorI m1 (XorI src2 src1)));
11090   ins_cost(INSN_COST);
11091   format %{ "eonw  $dst, $src1, $src2" %}
11092 
11093   ins_encode %{
11094     __ eonw(as_Register($dst$$reg),
11095               as_Register($src1$$reg),
11096               as_Register($src2$$reg),
11097               Assembler::LSL, 0);
11098   %}
11099 
11100   ins_pipe(ialu_reg_reg);
11101 %}
11102 
11103 // This pattern is automatically generated from aarch64_ad.m4
11104 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11105 instruct XorL_reg_not_reg(iRegLNoSp dst,
11106                          iRegL src1, iRegL src2, immL_M1 m1) %{
11107   match(Set dst (XorL m1 (XorL src2 src1)));
11108   ins_cost(INSN_COST);
11109   format %{ "eon  $dst, $src1, $src2" %}
11110 
11111   ins_encode %{
11112     __ eon(as_Register($dst$$reg),
11113               as_Register($src1$$reg),
11114               as_Register($src2$$reg),
11115               Assembler::LSL, 0);
11116   %}
11117 
11118   ins_pipe(ialu_reg_reg);
11119 %}
11120 
11121 // This pattern is automatically generated from aarch64_ad.m4
11122 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11123 // val & (-1 ^ (val >>> shift)) ==> bicw
11124 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11125                          iRegIorL2I src1, iRegIorL2I src2,
11126                          immI src3, immI_M1 src4) %{
11127   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11128   ins_cost(1.9 * INSN_COST);
11129   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11130 
11131   ins_encode %{
11132     __ bicw(as_Register($dst$$reg),
11133               as_Register($src1$$reg),
11134               as_Register($src2$$reg),
11135               Assembler::LSR,
11136               $src3$$constant & 0x1f);
11137   %}
11138 
11139   ins_pipe(ialu_reg_reg_shift);
11140 %}
11141 
11142 // This pattern is automatically generated from aarch64_ad.m4
11143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11144 // val & (-1 ^ (val >>> shift)) ==> bic
11145 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11146                          iRegL src1, iRegL src2,
11147                          immI src3, immL_M1 src4) %{
11148   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11149   ins_cost(1.9 * INSN_COST);
11150   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11151 
11152   ins_encode %{
11153     __ bic(as_Register($dst$$reg),
11154               as_Register($src1$$reg),
11155               as_Register($src2$$reg),
11156               Assembler::LSR,
11157               $src3$$constant & 0x3f);
11158   %}
11159 
11160   ins_pipe(ialu_reg_reg_shift);
11161 %}
11162 
11163 // This pattern is automatically generated from aarch64_ad.m4
11164 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11165 // val & (-1 ^ (val >> shift)) ==> bicw
11166 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11167                          iRegIorL2I src1, iRegIorL2I src2,
11168                          immI src3, immI_M1 src4) %{
11169   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11170   ins_cost(1.9 * INSN_COST);
11171   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11172 
11173   ins_encode %{
11174     __ bicw(as_Register($dst$$reg),
11175               as_Register($src1$$reg),
11176               as_Register($src2$$reg),
11177               Assembler::ASR,
11178               $src3$$constant & 0x1f);
11179   %}
11180 
11181   ins_pipe(ialu_reg_reg_shift);
11182 %}
11183 
11184 // This pattern is automatically generated from aarch64_ad.m4
11185 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11186 // val & (-1 ^ (val >> shift)) ==> bic
11187 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11188                          iRegL src1, iRegL src2,
11189                          immI src3, immL_M1 src4) %{
11190   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11191   ins_cost(1.9 * INSN_COST);
11192   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11193 
11194   ins_encode %{
11195     __ bic(as_Register($dst$$reg),
11196               as_Register($src1$$reg),
11197               as_Register($src2$$reg),
11198               Assembler::ASR,
11199               $src3$$constant & 0x3f);
11200   %}
11201 
11202   ins_pipe(ialu_reg_reg_shift);
11203 %}
11204 
11205 // This pattern is automatically generated from aarch64_ad.m4
11206 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11207 // val & (-1 ^ (val ror shift)) ==> bicw
11208 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11209                          iRegIorL2I src1, iRegIorL2I src2,
11210                          immI src3, immI_M1 src4) %{
11211   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11212   ins_cost(1.9 * INSN_COST);
11213   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11214 
11215   ins_encode %{
11216     __ bicw(as_Register($dst$$reg),
11217               as_Register($src1$$reg),
11218               as_Register($src2$$reg),
11219               Assembler::ROR,
11220               $src3$$constant & 0x1f);
11221   %}
11222 
11223   ins_pipe(ialu_reg_reg_shift);
11224 %}
11225 
11226 // This pattern is automatically generated from aarch64_ad.m4
11227 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11228 // val & (-1 ^ (val ror shift)) ==> bic
11229 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11230                          iRegL src1, iRegL src2,
11231                          immI src3, immL_M1 src4) %{
11232   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11233   ins_cost(1.9 * INSN_COST);
11234   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11235 
11236   ins_encode %{
11237     __ bic(as_Register($dst$$reg),
11238               as_Register($src1$$reg),
11239               as_Register($src2$$reg),
11240               Assembler::ROR,
11241               $src3$$constant & 0x3f);
11242   %}
11243 
11244   ins_pipe(ialu_reg_reg_shift);
11245 %}
11246 
11247 // This pattern is automatically generated from aarch64_ad.m4
11248 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11249 // val & (-1 ^ (val << shift)) ==> bicw
11250 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11251                          iRegIorL2I src1, iRegIorL2I src2,
11252                          immI src3, immI_M1 src4) %{
11253   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11254   ins_cost(1.9 * INSN_COST);
11255   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11256 
11257   ins_encode %{
11258     __ bicw(as_Register($dst$$reg),
11259               as_Register($src1$$reg),
11260               as_Register($src2$$reg),
11261               Assembler::LSL,
11262               $src3$$constant & 0x1f);
11263   %}
11264 
11265   ins_pipe(ialu_reg_reg_shift);
11266 %}
11267 
11268 // This pattern is automatically generated from aarch64_ad.m4
11269 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11270 // val & (-1 ^ (val << shift)) ==> bic
11271 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11272                          iRegL src1, iRegL src2,
11273                          immI src3, immL_M1 src4) %{
11274   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11275   ins_cost(1.9 * INSN_COST);
11276   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11277 
11278   ins_encode %{
11279     __ bic(as_Register($dst$$reg),
11280               as_Register($src1$$reg),
11281               as_Register($src2$$reg),
11282               Assembler::LSL,
11283               $src3$$constant & 0x3f);
11284   %}
11285 
11286   ins_pipe(ialu_reg_reg_shift);
11287 %}
11288 
11289 // This pattern is automatically generated from aarch64_ad.m4
11290 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11291 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11292 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11293                          iRegIorL2I src1, iRegIorL2I src2,
11294                          immI src3, immI_M1 src4) %{
11295   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11296   ins_cost(1.9 * INSN_COST);
11297   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11298 
11299   ins_encode %{
11300     __ eonw(as_Register($dst$$reg),
11301               as_Register($src1$$reg),
11302               as_Register($src2$$reg),
11303               Assembler::LSR,
11304               $src3$$constant & 0x1f);
11305   %}
11306 
11307   ins_pipe(ialu_reg_reg_shift);
11308 %}
11309 
11310 // This pattern is automatically generated from aarch64_ad.m4
11311 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11312 // val ^ (-1 ^ (val >>> shift)) ==> eon
11313 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11314                          iRegL src1, iRegL src2,
11315                          immI src3, immL_M1 src4) %{
11316   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11317   ins_cost(1.9 * INSN_COST);
11318   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11319 
11320   ins_encode %{
11321     __ eon(as_Register($dst$$reg),
11322               as_Register($src1$$reg),
11323               as_Register($src2$$reg),
11324               Assembler::LSR,
11325               $src3$$constant & 0x3f);
11326   %}
11327 
11328   ins_pipe(ialu_reg_reg_shift);
11329 %}
11330 
11331 // This pattern is automatically generated from aarch64_ad.m4
11332 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11333 // val ^ (-1 ^ (val >> shift)) ==> eonw
11334 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11335                          iRegIorL2I src1, iRegIorL2I src2,
11336                          immI src3, immI_M1 src4) %{
11337   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11338   ins_cost(1.9 * INSN_COST);
11339   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11340 
11341   ins_encode %{
11342     __ eonw(as_Register($dst$$reg),
11343               as_Register($src1$$reg),
11344               as_Register($src2$$reg),
11345               Assembler::ASR,
11346               $src3$$constant & 0x1f);
11347   %}
11348 
11349   ins_pipe(ialu_reg_reg_shift);
11350 %}
11351 
11352 // This pattern is automatically generated from aarch64_ad.m4
11353 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11354 // val ^ (-1 ^ (val >> shift)) ==> eon
11355 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11356                          iRegL src1, iRegL src2,
11357                          immI src3, immL_M1 src4) %{
11358   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11359   ins_cost(1.9 * INSN_COST);
11360   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11361 
11362   ins_encode %{
11363     __ eon(as_Register($dst$$reg),
11364               as_Register($src1$$reg),
11365               as_Register($src2$$reg),
11366               Assembler::ASR,
11367               $src3$$constant & 0x3f);
11368   %}
11369 
11370   ins_pipe(ialu_reg_reg_shift);
11371 %}
11372 
11373 // This pattern is automatically generated from aarch64_ad.m4
11374 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11375 // val ^ (-1 ^ (val ror shift)) ==> eonw
11376 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11377                          iRegIorL2I src1, iRegIorL2I src2,
11378                          immI src3, immI_M1 src4) %{
11379   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11380   ins_cost(1.9 * INSN_COST);
11381   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11382 
11383   ins_encode %{
11384     __ eonw(as_Register($dst$$reg),
11385               as_Register($src1$$reg),
11386               as_Register($src2$$reg),
11387               Assembler::ROR,
11388               $src3$$constant & 0x1f);
11389   %}
11390 
11391   ins_pipe(ialu_reg_reg_shift);
11392 %}
11393 
11394 // This pattern is automatically generated from aarch64_ad.m4
11395 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11396 // val ^ (-1 ^ (val ror shift)) ==> eon
11397 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11398                          iRegL src1, iRegL src2,
11399                          immI src3, immL_M1 src4) %{
11400   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11401   ins_cost(1.9 * INSN_COST);
11402   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11403 
11404   ins_encode %{
11405     __ eon(as_Register($dst$$reg),
11406               as_Register($src1$$reg),
11407               as_Register($src2$$reg),
11408               Assembler::ROR,
11409               $src3$$constant & 0x3f);
11410   %}
11411 
11412   ins_pipe(ialu_reg_reg_shift);
11413 %}
11414 
11415 // This pattern is automatically generated from aarch64_ad.m4
11416 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11417 // val ^ (-1 ^ (val << shift)) ==> eonw
11418 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11419                          iRegIorL2I src1, iRegIorL2I src2,
11420                          immI src3, immI_M1 src4) %{
11421   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11422   ins_cost(1.9 * INSN_COST);
11423   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11424 
11425   ins_encode %{
11426     __ eonw(as_Register($dst$$reg),
11427               as_Register($src1$$reg),
11428               as_Register($src2$$reg),
11429               Assembler::LSL,
11430               $src3$$constant & 0x1f);
11431   %}
11432 
11433   ins_pipe(ialu_reg_reg_shift);
11434 %}
11435 
11436 // This pattern is automatically generated from aarch64_ad.m4
11437 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11438 // val ^ (-1 ^ (val << shift)) ==> eon
11439 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11440                          iRegL src1, iRegL src2,
11441                          immI src3, immL_M1 src4) %{
11442   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11443   ins_cost(1.9 * INSN_COST);
11444   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11445 
11446   ins_encode %{
11447     __ eon(as_Register($dst$$reg),
11448               as_Register($src1$$reg),
11449               as_Register($src2$$reg),
11450               Assembler::LSL,
11451               $src3$$constant & 0x3f);
11452   %}
11453 
11454   ins_pipe(ialu_reg_reg_shift);
11455 %}
11456 
11457 // This pattern is automatically generated from aarch64_ad.m4
11458 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11459 // val | (-1 ^ (val >>> shift)) ==> ornw
11460 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11461                          iRegIorL2I src1, iRegIorL2I src2,
11462                          immI src3, immI_M1 src4) %{
11463   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11464   ins_cost(1.9 * INSN_COST);
11465   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11466 
11467   ins_encode %{
11468     __ ornw(as_Register($dst$$reg),
11469               as_Register($src1$$reg),
11470               as_Register($src2$$reg),
11471               Assembler::LSR,
11472               $src3$$constant & 0x1f);
11473   %}
11474 
11475   ins_pipe(ialu_reg_reg_shift);
11476 %}
11477 
11478 // This pattern is automatically generated from aarch64_ad.m4
11479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11480 // val | (-1 ^ (val >>> shift)) ==> orn
11481 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11482                          iRegL src1, iRegL src2,
11483                          immI src3, immL_M1 src4) %{
11484   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11485   ins_cost(1.9 * INSN_COST);
11486   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11487 
11488   ins_encode %{
11489     __ orn(as_Register($dst$$reg),
11490               as_Register($src1$$reg),
11491               as_Register($src2$$reg),
11492               Assembler::LSR,
11493               $src3$$constant & 0x3f);
11494   %}
11495 
11496   ins_pipe(ialu_reg_reg_shift);
11497 %}
11498 
11499 // This pattern is automatically generated from aarch64_ad.m4
11500 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11501 // val | (-1 ^ (val >> shift)) ==> ornw
11502 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11503                          iRegIorL2I src1, iRegIorL2I src2,
11504                          immI src3, immI_M1 src4) %{
11505   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11506   ins_cost(1.9 * INSN_COST);
11507   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11508 
11509   ins_encode %{
11510     __ ornw(as_Register($dst$$reg),
11511               as_Register($src1$$reg),
11512               as_Register($src2$$reg),
11513               Assembler::ASR,
11514               $src3$$constant & 0x1f);
11515   %}
11516 
11517   ins_pipe(ialu_reg_reg_shift);
11518 %}
11519 
11520 // This pattern is automatically generated from aarch64_ad.m4
11521 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11522 // val | (-1 ^ (val >> shift)) ==> orn
11523 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11524                          iRegL src1, iRegL src2,
11525                          immI src3, immL_M1 src4) %{
11526   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11527   ins_cost(1.9 * INSN_COST);
11528   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11529 
11530   ins_encode %{
11531     __ orn(as_Register($dst$$reg),
11532               as_Register($src1$$reg),
11533               as_Register($src2$$reg),
11534               Assembler::ASR,
11535               $src3$$constant & 0x3f);
11536   %}
11537 
11538   ins_pipe(ialu_reg_reg_shift);
11539 %}
11540 
11541 // This pattern is automatically generated from aarch64_ad.m4
11542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11543 // val | (-1 ^ (val ror shift)) ==> ornw
11544 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11545                          iRegIorL2I src1, iRegIorL2I src2,
11546                          immI src3, immI_M1 src4) %{
11547   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11548   ins_cost(1.9 * INSN_COST);
11549   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11550 
11551   ins_encode %{
11552     __ ornw(as_Register($dst$$reg),
11553               as_Register($src1$$reg),
11554               as_Register($src2$$reg),
11555               Assembler::ROR,
11556               $src3$$constant & 0x1f);
11557   %}
11558 
11559   ins_pipe(ialu_reg_reg_shift);
11560 %}
11561 
11562 // This pattern is automatically generated from aarch64_ad.m4
11563 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11564 // val | (-1 ^ (val ror shift)) ==> orn
11565 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11566                          iRegL src1, iRegL src2,
11567                          immI src3, immL_M1 src4) %{
11568   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11569   ins_cost(1.9 * INSN_COST);
11570   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11571 
11572   ins_encode %{
11573     __ orn(as_Register($dst$$reg),
11574               as_Register($src1$$reg),
11575               as_Register($src2$$reg),
11576               Assembler::ROR,
11577               $src3$$constant & 0x3f);
11578   %}
11579 
11580   ins_pipe(ialu_reg_reg_shift);
11581 %}
11582 
11583 // This pattern is automatically generated from aarch64_ad.m4
11584 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11585 // val | (-1 ^ (val << shift)) ==> ornw
11586 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11587                          iRegIorL2I src1, iRegIorL2I src2,
11588                          immI src3, immI_M1 src4) %{
11589   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11590   ins_cost(1.9 * INSN_COST);
11591   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11592 
11593   ins_encode %{
11594     __ ornw(as_Register($dst$$reg),
11595               as_Register($src1$$reg),
11596               as_Register($src2$$reg),
11597               Assembler::LSL,
11598               $src3$$constant & 0x1f);
11599   %}
11600 
11601   ins_pipe(ialu_reg_reg_shift);
11602 %}
11603 
11604 // This pattern is automatically generated from aarch64_ad.m4
11605 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11606 // val | (-1 ^ (val << shift)) ==> orn
11607 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11608                          iRegL src1, iRegL src2,
11609                          immI src3, immL_M1 src4) %{
11610   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11611   ins_cost(1.9 * INSN_COST);
11612   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11613 
11614   ins_encode %{
11615     __ orn(as_Register($dst$$reg),
11616               as_Register($src1$$reg),
11617               as_Register($src2$$reg),
11618               Assembler::LSL,
11619               $src3$$constant & 0x3f);
11620   %}
11621 
11622   ins_pipe(ialu_reg_reg_shift);
11623 %}
11624 
11625 // This pattern is automatically generated from aarch64_ad.m4
11626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11627 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11628                          iRegIorL2I src1, iRegIorL2I src2,
11629                          immI src3) %{
11630   match(Set dst (AndI src1 (URShiftI src2 src3)));
11631 
11632   ins_cost(1.9 * INSN_COST);
11633   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
11634 
11635   ins_encode %{
11636     __ andw(as_Register($dst$$reg),
11637               as_Register($src1$$reg),
11638               as_Register($src2$$reg),
11639               Assembler::LSR,
11640               $src3$$constant & 0x1f);
11641   %}
11642 
11643   ins_pipe(ialu_reg_reg_shift);
11644 %}
11645 
11646 // This pattern is automatically generated from aarch64_ad.m4
11647 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11648 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11649                          iRegL src1, iRegL src2,
11650                          immI src3) %{
11651   match(Set dst (AndL src1 (URShiftL src2 src3)));
11652 
11653   ins_cost(1.9 * INSN_COST);
11654   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
11655 
11656   ins_encode %{
11657     __ andr(as_Register($dst$$reg),
11658               as_Register($src1$$reg),
11659               as_Register($src2$$reg),
11660               Assembler::LSR,
11661               $src3$$constant & 0x3f);
11662   %}
11663 
11664   ins_pipe(ialu_reg_reg_shift);
11665 %}
11666 
11667 // This pattern is automatically generated from aarch64_ad.m4
11668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11669 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11670                          iRegIorL2I src1, iRegIorL2I src2,
11671                          immI src3) %{
11672   match(Set dst (AndI src1 (RShiftI src2 src3)));
11673 
11674   ins_cost(1.9 * INSN_COST);
11675   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
11676 
11677   ins_encode %{
11678     __ andw(as_Register($dst$$reg),
11679               as_Register($src1$$reg),
11680               as_Register($src2$$reg),
11681               Assembler::ASR,
11682               $src3$$constant & 0x1f);
11683   %}
11684 
11685   ins_pipe(ialu_reg_reg_shift);
11686 %}
11687 
11688 // This pattern is automatically generated from aarch64_ad.m4
11689 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11690 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11691                          iRegL src1, iRegL src2,
11692                          immI src3) %{
11693   match(Set dst (AndL src1 (RShiftL src2 src3)));
11694 
11695   ins_cost(1.9 * INSN_COST);
11696   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
11697 
11698   ins_encode %{
11699     __ andr(as_Register($dst$$reg),
11700               as_Register($src1$$reg),
11701               as_Register($src2$$reg),
11702               Assembler::ASR,
11703               $src3$$constant & 0x3f);
11704   %}
11705 
11706   ins_pipe(ialu_reg_reg_shift);
11707 %}
11708 
11709 // This pattern is automatically generated from aarch64_ad.m4
11710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11711 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11712                          iRegIorL2I src1, iRegIorL2I src2,
11713                          immI src3) %{
11714   match(Set dst (AndI src1 (LShiftI src2 src3)));
11715 
11716   ins_cost(1.9 * INSN_COST);
11717   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
11718 
11719   ins_encode %{
11720     __ andw(as_Register($dst$$reg),
11721               as_Register($src1$$reg),
11722               as_Register($src2$$reg),
11723               Assembler::LSL,
11724               $src3$$constant & 0x1f);
11725   %}
11726 
11727   ins_pipe(ialu_reg_reg_shift);
11728 %}
11729 
11730 // This pattern is automatically generated from aarch64_ad.m4
11731 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11732 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11733                          iRegL src1, iRegL src2,
11734                          immI src3) %{
11735   match(Set dst (AndL src1 (LShiftL src2 src3)));
11736 
11737   ins_cost(1.9 * INSN_COST);
11738   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
11739 
11740   ins_encode %{
11741     __ andr(as_Register($dst$$reg),
11742               as_Register($src1$$reg),
11743               as_Register($src2$$reg),
11744               Assembler::LSL,
11745               $src3$$constant & 0x3f);
11746   %}
11747 
11748   ins_pipe(ialu_reg_reg_shift);
11749 %}
11750 
11751 // This pattern is automatically generated from aarch64_ad.m4
11752 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11753 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11754                          iRegIorL2I src1, iRegIorL2I src2,
11755                          immI src3) %{
11756   match(Set dst (AndI src1 (RotateRight src2 src3)));
11757 
11758   ins_cost(1.9 * INSN_COST);
11759   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
11760 
11761   ins_encode %{
11762     __ andw(as_Register($dst$$reg),
11763               as_Register($src1$$reg),
11764               as_Register($src2$$reg),
11765               Assembler::ROR,
11766               $src3$$constant & 0x1f);
11767   %}
11768 
11769   ins_pipe(ialu_reg_reg_shift);
11770 %}
11771 
11772 // This pattern is automatically generated from aarch64_ad.m4
11773 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11774 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11775                          iRegL src1, iRegL src2,
11776                          immI src3) %{
11777   match(Set dst (AndL src1 (RotateRight src2 src3)));
11778 
11779   ins_cost(1.9 * INSN_COST);
11780   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
11781 
11782   ins_encode %{
11783     __ andr(as_Register($dst$$reg),
11784               as_Register($src1$$reg),
11785               as_Register($src2$$reg),
11786               Assembler::ROR,
11787               $src3$$constant & 0x3f);
11788   %}
11789 
11790   ins_pipe(ialu_reg_reg_shift);
11791 %}
11792 
11793 // This pattern is automatically generated from aarch64_ad.m4
11794 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11795 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11796                          iRegIorL2I src1, iRegIorL2I src2,
11797                          immI src3) %{
11798   match(Set dst (XorI src1 (URShiftI src2 src3)));
11799 
11800   ins_cost(1.9 * INSN_COST);
11801   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
11802 
11803   ins_encode %{
11804     __ eorw(as_Register($dst$$reg),
11805               as_Register($src1$$reg),
11806               as_Register($src2$$reg),
11807               Assembler::LSR,
11808               $src3$$constant & 0x1f);
11809   %}
11810 
11811   ins_pipe(ialu_reg_reg_shift);
11812 %}
11813 
11814 // This pattern is automatically generated from aarch64_ad.m4
11815 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11816 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11817                          iRegL src1, iRegL src2,
11818                          immI src3) %{
11819   match(Set dst (XorL src1 (URShiftL src2 src3)));
11820 
11821   ins_cost(1.9 * INSN_COST);
11822   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
11823 
11824   ins_encode %{
11825     __ eor(as_Register($dst$$reg),
11826               as_Register($src1$$reg),
11827               as_Register($src2$$reg),
11828               Assembler::LSR,
11829               $src3$$constant & 0x3f);
11830   %}
11831 
11832   ins_pipe(ialu_reg_reg_shift);
11833 %}
11834 
11835 // This pattern is automatically generated from aarch64_ad.m4
11836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11837 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11838                          iRegIorL2I src1, iRegIorL2I src2,
11839                          immI src3) %{
11840   match(Set dst (XorI src1 (RShiftI src2 src3)));
11841 
11842   ins_cost(1.9 * INSN_COST);
11843   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
11844 
11845   ins_encode %{
11846     __ eorw(as_Register($dst$$reg),
11847               as_Register($src1$$reg),
11848               as_Register($src2$$reg),
11849               Assembler::ASR,
11850               $src3$$constant & 0x1f);
11851   %}
11852 
11853   ins_pipe(ialu_reg_reg_shift);
11854 %}
11855 
11856 // This pattern is automatically generated from aarch64_ad.m4
11857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11858 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11859                          iRegL src1, iRegL src2,
11860                          immI src3) %{
11861   match(Set dst (XorL src1 (RShiftL src2 src3)));
11862 
11863   ins_cost(1.9 * INSN_COST);
11864   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
11865 
11866   ins_encode %{
11867     __ eor(as_Register($dst$$reg),
11868               as_Register($src1$$reg),
11869               as_Register($src2$$reg),
11870               Assembler::ASR,
11871               $src3$$constant & 0x3f);
11872   %}
11873 
11874   ins_pipe(ialu_reg_reg_shift);
11875 %}
11876 
11877 // This pattern is automatically generated from aarch64_ad.m4
11878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11879 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11880                          iRegIorL2I src1, iRegIorL2I src2,
11881                          immI src3) %{
11882   match(Set dst (XorI src1 (LShiftI src2 src3)));
11883 
11884   ins_cost(1.9 * INSN_COST);
11885   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
11886 
11887   ins_encode %{
11888     __ eorw(as_Register($dst$$reg),
11889               as_Register($src1$$reg),
11890               as_Register($src2$$reg),
11891               Assembler::LSL,
11892               $src3$$constant & 0x1f);
11893   %}
11894 
11895   ins_pipe(ialu_reg_reg_shift);
11896 %}
11897 
11898 // This pattern is automatically generated from aarch64_ad.m4
11899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11900 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11901                          iRegL src1, iRegL src2,
11902                          immI src3) %{
11903   match(Set dst (XorL src1 (LShiftL src2 src3)));
11904 
11905   ins_cost(1.9 * INSN_COST);
11906   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
11907 
11908   ins_encode %{
11909     __ eor(as_Register($dst$$reg),
11910               as_Register($src1$$reg),
11911               as_Register($src2$$reg),
11912               Assembler::LSL,
11913               $src3$$constant & 0x3f);
11914   %}
11915 
11916   ins_pipe(ialu_reg_reg_shift);
11917 %}
11918 
11919 // This pattern is automatically generated from aarch64_ad.m4
11920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11921 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11922                          iRegIorL2I src1, iRegIorL2I src2,
11923                          immI src3) %{
11924   match(Set dst (XorI src1 (RotateRight src2 src3)));
11925 
11926   ins_cost(1.9 * INSN_COST);
11927   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
11928 
11929   ins_encode %{
11930     __ eorw(as_Register($dst$$reg),
11931               as_Register($src1$$reg),
11932               as_Register($src2$$reg),
11933               Assembler::ROR,
11934               $src3$$constant & 0x1f);
11935   %}
11936 
11937   ins_pipe(ialu_reg_reg_shift);
11938 %}
11939 
11940 // This pattern is automatically generated from aarch64_ad.m4
11941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11942 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11943                          iRegL src1, iRegL src2,
11944                          immI src3) %{
11945   match(Set dst (XorL src1 (RotateRight src2 src3)));
11946 
11947   ins_cost(1.9 * INSN_COST);
11948   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
11949 
11950   ins_encode %{
11951     __ eor(as_Register($dst$$reg),
11952               as_Register($src1$$reg),
11953               as_Register($src2$$reg),
11954               Assembler::ROR,
11955               $src3$$constant & 0x3f);
11956   %}
11957 
11958   ins_pipe(ialu_reg_reg_shift);
11959 %}
11960 
11961 // This pattern is automatically generated from aarch64_ad.m4
11962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11963 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11964                          iRegIorL2I src1, iRegIorL2I src2,
11965                          immI src3) %{
11966   match(Set dst (OrI src1 (URShiftI src2 src3)));
11967 
11968   ins_cost(1.9 * INSN_COST);
11969   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
11970 
11971   ins_encode %{
11972     __ orrw(as_Register($dst$$reg),
11973               as_Register($src1$$reg),
11974               as_Register($src2$$reg),
11975               Assembler::LSR,
11976               $src3$$constant & 0x1f);
11977   %}
11978 
11979   ins_pipe(ialu_reg_reg_shift);
11980 %}
11981 
11982 // This pattern is automatically generated from aarch64_ad.m4
11983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11984 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11985                          iRegL src1, iRegL src2,
11986                          immI src3) %{
11987   match(Set dst (OrL src1 (URShiftL src2 src3)));
11988 
11989   ins_cost(1.9 * INSN_COST);
11990   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
11991 
11992   ins_encode %{
11993     __ orr(as_Register($dst$$reg),
11994               as_Register($src1$$reg),
11995               as_Register($src2$$reg),
11996               Assembler::LSR,
11997               $src3$$constant & 0x3f);
11998   %}
11999 
12000   ins_pipe(ialu_reg_reg_shift);
12001 %}
12002 
12003 // This pattern is automatically generated from aarch64_ad.m4
12004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12005 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12006                          iRegIorL2I src1, iRegIorL2I src2,
12007                          immI src3) %{
12008   match(Set dst (OrI src1 (RShiftI src2 src3)));
12009 
12010   ins_cost(1.9 * INSN_COST);
12011   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12012 
12013   ins_encode %{
12014     __ orrw(as_Register($dst$$reg),
12015               as_Register($src1$$reg),
12016               as_Register($src2$$reg),
12017               Assembler::ASR,
12018               $src3$$constant & 0x1f);
12019   %}
12020 
12021   ins_pipe(ialu_reg_reg_shift);
12022 %}
12023 
12024 // This pattern is automatically generated from aarch64_ad.m4
12025 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12026 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12027                          iRegL src1, iRegL src2,
12028                          immI src3) %{
12029   match(Set dst (OrL src1 (RShiftL src2 src3)));
12030 
12031   ins_cost(1.9 * INSN_COST);
12032   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12033 
12034   ins_encode %{
12035     __ orr(as_Register($dst$$reg),
12036               as_Register($src1$$reg),
12037               as_Register($src2$$reg),
12038               Assembler::ASR,
12039               $src3$$constant & 0x3f);
12040   %}
12041 
12042   ins_pipe(ialu_reg_reg_shift);
12043 %}
12044 
12045 // This pattern is automatically generated from aarch64_ad.m4
12046 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12047 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12048                          iRegIorL2I src1, iRegIorL2I src2,
12049                          immI src3) %{
12050   match(Set dst (OrI src1 (LShiftI src2 src3)));
12051 
12052   ins_cost(1.9 * INSN_COST);
12053   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12054 
12055   ins_encode %{
12056     __ orrw(as_Register($dst$$reg),
12057               as_Register($src1$$reg),
12058               as_Register($src2$$reg),
12059               Assembler::LSL,
12060               $src3$$constant & 0x1f);
12061   %}
12062 
12063   ins_pipe(ialu_reg_reg_shift);
12064 %}
12065 
12066 // This pattern is automatically generated from aarch64_ad.m4
12067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12068 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12069                          iRegL src1, iRegL src2,
12070                          immI src3) %{
12071   match(Set dst (OrL src1 (LShiftL src2 src3)));
12072 
12073   ins_cost(1.9 * INSN_COST);
12074   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12075 
12076   ins_encode %{
12077     __ orr(as_Register($dst$$reg),
12078               as_Register($src1$$reg),
12079               as_Register($src2$$reg),
12080               Assembler::LSL,
12081               $src3$$constant & 0x3f);
12082   %}
12083 
12084   ins_pipe(ialu_reg_reg_shift);
12085 %}
12086 
12087 // This pattern is automatically generated from aarch64_ad.m4
12088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12089 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12090                          iRegIorL2I src1, iRegIorL2I src2,
12091                          immI src3) %{
12092   match(Set dst (OrI src1 (RotateRight src2 src3)));
12093 
12094   ins_cost(1.9 * INSN_COST);
12095   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12096 
12097   ins_encode %{
12098     __ orrw(as_Register($dst$$reg),
12099               as_Register($src1$$reg),
12100               as_Register($src2$$reg),
12101               Assembler::ROR,
12102               $src3$$constant & 0x1f);
12103   %}
12104 
12105   ins_pipe(ialu_reg_reg_shift);
12106 %}
12107 
12108 // This pattern is automatically generated from aarch64_ad.m4
12109 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12110 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12111                          iRegL src1, iRegL src2,
12112                          immI src3) %{
12113   match(Set dst (OrL src1 (RotateRight src2 src3)));
12114 
12115   ins_cost(1.9 * INSN_COST);
12116   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12117 
12118   ins_encode %{
12119     __ orr(as_Register($dst$$reg),
12120               as_Register($src1$$reg),
12121               as_Register($src2$$reg),
12122               Assembler::ROR,
12123               $src3$$constant & 0x3f);
12124   %}
12125 
12126   ins_pipe(ialu_reg_reg_shift);
12127 %}
12128 
12129 // This pattern is automatically generated from aarch64_ad.m4
12130 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12131 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12132                          iRegIorL2I src1, iRegIorL2I src2,
12133                          immI src3) %{
12134   match(Set dst (AddI src1 (URShiftI src2 src3)));
12135 
12136   ins_cost(1.9 * INSN_COST);
12137   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12138 
12139   ins_encode %{
12140     __ addw(as_Register($dst$$reg),
12141               as_Register($src1$$reg),
12142               as_Register($src2$$reg),
12143               Assembler::LSR,
12144               $src3$$constant & 0x1f);
12145   %}
12146 
12147   ins_pipe(ialu_reg_reg_shift);
12148 %}
12149 
12150 // This pattern is automatically generated from aarch64_ad.m4
12151 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12152 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12153                          iRegL src1, iRegL src2,
12154                          immI src3) %{
12155   match(Set dst (AddL src1 (URShiftL src2 src3)));
12156 
12157   ins_cost(1.9 * INSN_COST);
12158   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12159 
12160   ins_encode %{
12161     __ add(as_Register($dst$$reg),
12162               as_Register($src1$$reg),
12163               as_Register($src2$$reg),
12164               Assembler::LSR,
12165               $src3$$constant & 0x3f);
12166   %}
12167 
12168   ins_pipe(ialu_reg_reg_shift);
12169 %}
12170 
12171 // This pattern is automatically generated from aarch64_ad.m4
12172 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12173 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12174                          iRegIorL2I src1, iRegIorL2I src2,
12175                          immI src3) %{
12176   match(Set dst (AddI src1 (RShiftI src2 src3)));
12177 
12178   ins_cost(1.9 * INSN_COST);
12179   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12180 
12181   ins_encode %{
12182     __ addw(as_Register($dst$$reg),
12183               as_Register($src1$$reg),
12184               as_Register($src2$$reg),
12185               Assembler::ASR,
12186               $src3$$constant & 0x1f);
12187   %}
12188 
12189   ins_pipe(ialu_reg_reg_shift);
12190 %}
12191 
12192 // This pattern is automatically generated from aarch64_ad.m4
12193 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12194 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12195                          iRegL src1, iRegL src2,
12196                          immI src3) %{
12197   match(Set dst (AddL src1 (RShiftL src2 src3)));
12198 
12199   ins_cost(1.9 * INSN_COST);
12200   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12201 
12202   ins_encode %{
12203     __ add(as_Register($dst$$reg),
12204               as_Register($src1$$reg),
12205               as_Register($src2$$reg),
12206               Assembler::ASR,
12207               $src3$$constant & 0x3f);
12208   %}
12209 
12210   ins_pipe(ialu_reg_reg_shift);
12211 %}
12212 
12213 // This pattern is automatically generated from aarch64_ad.m4
12214 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12215 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12216                          iRegIorL2I src1, iRegIorL2I src2,
12217                          immI src3) %{
12218   match(Set dst (AddI src1 (LShiftI src2 src3)));
12219 
12220   ins_cost(1.9 * INSN_COST);
12221   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12222 
12223   ins_encode %{
12224     __ addw(as_Register($dst$$reg),
12225               as_Register($src1$$reg),
12226               as_Register($src2$$reg),
12227               Assembler::LSL,
12228               $src3$$constant & 0x1f);
12229   %}
12230 
12231   ins_pipe(ialu_reg_reg_shift);
12232 %}
12233 
12234 // This pattern is automatically generated from aarch64_ad.m4
12235 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12236 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12237                          iRegL src1, iRegL src2,
12238                          immI src3) %{
12239   match(Set dst (AddL src1 (LShiftL src2 src3)));
12240 
12241   ins_cost(1.9 * INSN_COST);
12242   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12243 
12244   ins_encode %{
12245     __ add(as_Register($dst$$reg),
12246               as_Register($src1$$reg),
12247               as_Register($src2$$reg),
12248               Assembler::LSL,
12249               $src3$$constant & 0x3f);
12250   %}
12251 
12252   ins_pipe(ialu_reg_reg_shift);
12253 %}
12254 
12255 // This pattern is automatically generated from aarch64_ad.m4
12256 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12257 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12258                          iRegIorL2I src1, iRegIorL2I src2,
12259                          immI src3) %{
12260   match(Set dst (SubI src1 (URShiftI src2 src3)));
12261 
12262   ins_cost(1.9 * INSN_COST);
12263   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12264 
12265   ins_encode %{
12266     __ subw(as_Register($dst$$reg),
12267               as_Register($src1$$reg),
12268               as_Register($src2$$reg),
12269               Assembler::LSR,
12270               $src3$$constant & 0x1f);
12271   %}
12272 
12273   ins_pipe(ialu_reg_reg_shift);
12274 %}
12275 
12276 // This pattern is automatically generated from aarch64_ad.m4
12277 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12278 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12279                          iRegL src1, iRegL src2,
12280                          immI src3) %{
12281   match(Set dst (SubL src1 (URShiftL src2 src3)));
12282 
12283   ins_cost(1.9 * INSN_COST);
12284   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12285 
12286   ins_encode %{
12287     __ sub(as_Register($dst$$reg),
12288               as_Register($src1$$reg),
12289               as_Register($src2$$reg),
12290               Assembler::LSR,
12291               $src3$$constant & 0x3f);
12292   %}
12293 
12294   ins_pipe(ialu_reg_reg_shift);
12295 %}
12296 
12297 // This pattern is automatically generated from aarch64_ad.m4
12298 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12299 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12300                          iRegIorL2I src1, iRegIorL2I src2,
12301                          immI src3) %{
12302   match(Set dst (SubI src1 (RShiftI src2 src3)));
12303 
12304   ins_cost(1.9 * INSN_COST);
12305   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12306 
12307   ins_encode %{
12308     __ subw(as_Register($dst$$reg),
12309               as_Register($src1$$reg),
12310               as_Register($src2$$reg),
12311               Assembler::ASR,
12312               $src3$$constant & 0x1f);
12313   %}
12314 
12315   ins_pipe(ialu_reg_reg_shift);
12316 %}
12317 
12318 // This pattern is automatically generated from aarch64_ad.m4
12319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12320 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12321                          iRegL src1, iRegL src2,
12322                          immI src3) %{
12323   match(Set dst (SubL src1 (RShiftL src2 src3)));
12324 
12325   ins_cost(1.9 * INSN_COST);
12326   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12327 
12328   ins_encode %{
12329     __ sub(as_Register($dst$$reg),
12330               as_Register($src1$$reg),
12331               as_Register($src2$$reg),
12332               Assembler::ASR,
12333               $src3$$constant & 0x3f);
12334   %}
12335 
12336   ins_pipe(ialu_reg_reg_shift);
12337 %}
12338 
12339 // This pattern is automatically generated from aarch64_ad.m4
12340 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12341 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12342                          iRegIorL2I src1, iRegIorL2I src2,
12343                          immI src3) %{
12344   match(Set dst (SubI src1 (LShiftI src2 src3)));
12345 
12346   ins_cost(1.9 * INSN_COST);
12347   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12348 
12349   ins_encode %{
12350     __ subw(as_Register($dst$$reg),
12351               as_Register($src1$$reg),
12352               as_Register($src2$$reg),
12353               Assembler::LSL,
12354               $src3$$constant & 0x1f);
12355   %}
12356 
12357   ins_pipe(ialu_reg_reg_shift);
12358 %}
12359 
12360 // This pattern is automatically generated from aarch64_ad.m4
12361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12362 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12363                          iRegL src1, iRegL src2,
12364                          immI src3) %{
12365   match(Set dst (SubL src1 (LShiftL src2 src3)));
12366 
12367   ins_cost(1.9 * INSN_COST);
12368   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12369 
12370   ins_encode %{
12371     __ sub(as_Register($dst$$reg),
12372               as_Register($src1$$reg),
12373               as_Register($src2$$reg),
12374               Assembler::LSL,
12375               $src3$$constant & 0x3f);
12376   %}
12377 
12378   ins_pipe(ialu_reg_reg_shift);
12379 %}
12380 
12381 // This pattern is automatically generated from aarch64_ad.m4
12382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12383 
12384 // Shift Left followed by Shift Right.
12385 // This idiom is used by the compiler for the i2b bytecode etc.
12386 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12387 %{
12388   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12389   ins_cost(INSN_COST * 2);
12390   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12391   ins_encode %{
12392     int lshift = $lshift_count$$constant & 63;
12393     int rshift = $rshift_count$$constant & 63;
12394     int s = 63 - lshift;
12395     int r = (rshift - lshift) & 63;
12396     __ sbfm(as_Register($dst$$reg),
12397             as_Register($src$$reg),
12398             r, s);
12399   %}
12400 
12401   ins_pipe(ialu_reg_shift);
12402 %}
12403 
12404 // This pattern is automatically generated from aarch64_ad.m4
12405 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12406 
12407 // Shift Left followed by Shift Right.
12408 // This idiom is used by the compiler for the i2b bytecode etc.
12409 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12410 %{
12411   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12412   ins_cost(INSN_COST * 2);
12413   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12414   ins_encode %{
12415     int lshift = $lshift_count$$constant & 31;
12416     int rshift = $rshift_count$$constant & 31;
12417     int s = 31 - lshift;
12418     int r = (rshift - lshift) & 31;
12419     __ sbfmw(as_Register($dst$$reg),
12420             as_Register($src$$reg),
12421             r, s);
12422   %}
12423 
12424   ins_pipe(ialu_reg_shift);
12425 %}
12426 
12427 // This pattern is automatically generated from aarch64_ad.m4
12428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12429 
12430 // Shift Left followed by Shift Right.
12431 // This idiom is used by the compiler for the i2b bytecode etc.
12432 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12433 %{
12434   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12435   ins_cost(INSN_COST * 2);
12436   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12437   ins_encode %{
12438     int lshift = $lshift_count$$constant & 63;
12439     int rshift = $rshift_count$$constant & 63;
12440     int s = 63 - lshift;
12441     int r = (rshift - lshift) & 63;
12442     __ ubfm(as_Register($dst$$reg),
12443             as_Register($src$$reg),
12444             r, s);
12445   %}
12446 
12447   ins_pipe(ialu_reg_shift);
12448 %}
12449 
12450 // This pattern is automatically generated from aarch64_ad.m4
12451 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12452 
12453 // Shift Left followed by Shift Right.
12454 // This idiom is used by the compiler for the i2b bytecode etc.
12455 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12456 %{
12457   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12458   ins_cost(INSN_COST * 2);
12459   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12460   ins_encode %{
12461     int lshift = $lshift_count$$constant & 31;
12462     int rshift = $rshift_count$$constant & 31;
12463     int s = 31 - lshift;
12464     int r = (rshift - lshift) & 31;
12465     __ ubfmw(as_Register($dst$$reg),
12466             as_Register($src$$reg),
12467             r, s);
12468   %}
12469 
12470   ins_pipe(ialu_reg_shift);
12471 %}
12472 
12473 // Bitfield extract with shift & mask
12474 
12475 // This pattern is automatically generated from aarch64_ad.m4
12476 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12477 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12478 %{
12479   match(Set dst (AndI (URShiftI src rshift) mask));
12480   // Make sure we are not going to exceed what ubfxw can do.
12481   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12482 
12483   ins_cost(INSN_COST);
12484   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12485   ins_encode %{
12486     int rshift = $rshift$$constant & 31;
12487     intptr_t mask = $mask$$constant;
12488     int width = exact_log2(mask+1);
12489     __ ubfxw(as_Register($dst$$reg),
12490             as_Register($src$$reg), rshift, width);
12491   %}
12492   ins_pipe(ialu_reg_shift);
12493 %}
12494 
12495 // This pattern is automatically generated from aarch64_ad.m4
12496 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12497 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12498 %{
12499   match(Set dst (AndL (URShiftL src rshift) mask));
12500   // Make sure we are not going to exceed what ubfx can do.
12501   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12502 
12503   ins_cost(INSN_COST);
12504   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12505   ins_encode %{
12506     int rshift = $rshift$$constant & 63;
12507     intptr_t mask = $mask$$constant;
12508     int width = exact_log2_long(mask+1);
12509     __ ubfx(as_Register($dst$$reg),
12510             as_Register($src$$reg), rshift, width);
12511   %}
12512   ins_pipe(ialu_reg_shift);
12513 %}
12514 
12515 
12516 // This pattern is automatically generated from aarch64_ad.m4
12517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12518 
12519 // We can use ubfx when extending an And with a mask when we know mask
12520 // is positive.  We know that because immI_bitmask guarantees it.
12521 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12522 %{
12523   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12524   // Make sure we are not going to exceed what ubfxw can do.
12525   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12526 
12527   ins_cost(INSN_COST * 2);
12528   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12529   ins_encode %{
12530     int rshift = $rshift$$constant & 31;
12531     intptr_t mask = $mask$$constant;
12532     int width = exact_log2(mask+1);
12533     __ ubfx(as_Register($dst$$reg),
12534             as_Register($src$$reg), rshift, width);
12535   %}
12536   ins_pipe(ialu_reg_shift);
12537 %}
12538 
12539 
12540 // This pattern is automatically generated from aarch64_ad.m4
12541 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12542 
12543 // We can use ubfiz when masking by a positive number and then left shifting the result.
12544 // We know that the mask is positive because immI_bitmask guarantees it.
12545 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12546 %{
12547   match(Set dst (LShiftI (AndI src mask) lshift));
12548   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12549 
12550   ins_cost(INSN_COST);
12551   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12552   ins_encode %{
12553     int lshift = $lshift$$constant & 31;
12554     intptr_t mask = $mask$$constant;
12555     int width = exact_log2(mask+1);
12556     __ ubfizw(as_Register($dst$$reg),
12557           as_Register($src$$reg), lshift, width);
12558   %}
12559   ins_pipe(ialu_reg_shift);
12560 %}
12561 
12562 // This pattern is automatically generated from aarch64_ad.m4
12563 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12564 
12565 // We can use ubfiz when masking by a positive number and then left shifting the result.
12566 // We know that the mask is positive because immL_bitmask guarantees it.
12567 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12568 %{
12569   match(Set dst (LShiftL (AndL src mask) lshift));
12570   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12571 
12572   ins_cost(INSN_COST);
12573   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12574   ins_encode %{
12575     int lshift = $lshift$$constant & 63;
12576     intptr_t mask = $mask$$constant;
12577     int width = exact_log2_long(mask+1);
12578     __ ubfiz(as_Register($dst$$reg),
12579           as_Register($src$$reg), lshift, width);
12580   %}
12581   ins_pipe(ialu_reg_shift);
12582 %}
12583 
12584 // This pattern is automatically generated from aarch64_ad.m4
12585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12586 
12587 // We can use ubfiz when masking by a positive number and then left shifting the result.
12588 // We know that the mask is positive because immI_bitmask guarantees it.
12589 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12590 %{
12591   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12592   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12593 
12594   ins_cost(INSN_COST);
12595   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12596   ins_encode %{
12597     int lshift = $lshift$$constant & 31;
12598     intptr_t mask = $mask$$constant;
12599     int width = exact_log2(mask+1);
12600     __ ubfizw(as_Register($dst$$reg),
12601           as_Register($src$$reg), lshift, width);
12602   %}
12603   ins_pipe(ialu_reg_shift);
12604 %}
12605 
12606 // This pattern is automatically generated from aarch64_ad.m4
12607 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12608 
12609 // We can use ubfiz when masking by a positive number and then left shifting the result.
12610 // We know that the mask is positive because immL_bitmask guarantees it.
12611 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12612 %{
12613   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12614   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12615 
12616   ins_cost(INSN_COST);
12617   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12618   ins_encode %{
12619     int lshift = $lshift$$constant & 63;
12620     intptr_t mask = $mask$$constant;
12621     int width = exact_log2_long(mask+1);
12622     __ ubfiz(as_Register($dst$$reg),
12623           as_Register($src$$reg), lshift, width);
12624   %}
12625   ins_pipe(ialu_reg_shift);
12626 %}
12627 
12628 
12629 // This pattern is automatically generated from aarch64_ad.m4
12630 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12631 
12632 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12633 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12634 %{
12635   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12636   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12637 
12638   ins_cost(INSN_COST);
12639   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12640   ins_encode %{
12641     int lshift = $lshift$$constant & 63;
12642     intptr_t mask = $mask$$constant;
12643     int width = exact_log2(mask+1);
12644     __ ubfiz(as_Register($dst$$reg),
12645              as_Register($src$$reg), lshift, width);
12646   %}
12647   ins_pipe(ialu_reg_shift);
12648 %}
12649 
12650 // This pattern is automatically generated from aarch64_ad.m4
12651 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12652 
12653 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12654 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12655 %{
12656   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12657   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12658 
12659   ins_cost(INSN_COST);
12660   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12661   ins_encode %{
12662     int lshift = $lshift$$constant & 31;
12663     intptr_t mask = $mask$$constant;
12664     int width = exact_log2(mask+1);
12665     __ ubfiz(as_Register($dst$$reg),
12666              as_Register($src$$reg), lshift, width);
12667   %}
12668   ins_pipe(ialu_reg_shift);
12669 %}
12670 
12671 // This pattern is automatically generated from aarch64_ad.m4
12672 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12673 
12674 // Can skip int2long conversions after AND with small bitmask
12675 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12676 %{
12677   match(Set dst (ConvI2L (AndI src msk)));
12678   ins_cost(INSN_COST);
12679   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12680   ins_encode %{
12681     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12682   %}
12683   ins_pipe(ialu_reg_shift);
12684 %}
12685 
12686 
12687 // Rotations
12688 
12689 // This pattern is automatically generated from aarch64_ad.m4
12690 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12691 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12692 %{
12693   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12694   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12695 
12696   ins_cost(INSN_COST);
12697   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12698 
12699   ins_encode %{
12700     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12701             $rshift$$constant & 63);
12702   %}
12703   ins_pipe(ialu_reg_reg_extr);
12704 %}
12705 
12706 
12707 // This pattern is automatically generated from aarch64_ad.m4
12708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12709 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12710 %{
12711   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12712   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12713 
12714   ins_cost(INSN_COST);
12715   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12716 
12717   ins_encode %{
12718     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12719             $rshift$$constant & 31);
12720   %}
12721   ins_pipe(ialu_reg_reg_extr);
12722 %}
12723 
12724 
12725 // This pattern is automatically generated from aarch64_ad.m4
12726 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12727 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12728 %{
12729   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12730   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12731 
12732   ins_cost(INSN_COST);
12733   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12734 
12735   ins_encode %{
12736     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12737             $rshift$$constant & 63);
12738   %}
12739   ins_pipe(ialu_reg_reg_extr);
12740 %}
12741 
12742 
12743 // This pattern is automatically generated from aarch64_ad.m4
12744 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12745 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12746 %{
12747   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12748   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12749 
12750   ins_cost(INSN_COST);
12751   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12752 
12753   ins_encode %{
12754     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12755             $rshift$$constant & 31);
12756   %}
12757   ins_pipe(ialu_reg_reg_extr);
12758 %}
12759 
12760 // This pattern is automatically generated from aarch64_ad.m4
12761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12762 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12763 %{
12764   match(Set dst (RotateRight src shift));
12765 
12766   ins_cost(INSN_COST);
12767   format %{ "ror    $dst, $src, $shift" %}
12768 
12769   ins_encode %{
12770      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12771                $shift$$constant & 0x1f);
12772   %}
12773   ins_pipe(ialu_reg_reg_vshift);
12774 %}
12775 
12776 // This pattern is automatically generated from aarch64_ad.m4
12777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12778 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12779 %{
12780   match(Set dst (RotateRight src shift));
12781 
12782   ins_cost(INSN_COST);
12783   format %{ "ror    $dst, $src, $shift" %}
12784 
12785   ins_encode %{
12786      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12787                $shift$$constant & 0x3f);
12788   %}
12789   ins_pipe(ialu_reg_reg_vshift);
12790 %}
12791 
12792 // This pattern is automatically generated from aarch64_ad.m4
12793 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12794 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12795 %{
12796   match(Set dst (RotateRight src shift));
12797 
12798   ins_cost(INSN_COST);
12799   format %{ "ror    $dst, $src, $shift" %}
12800 
12801   ins_encode %{
12802      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12803   %}
12804   ins_pipe(ialu_reg_reg_vshift);
12805 %}
12806 
12807 // This pattern is automatically generated from aarch64_ad.m4
12808 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12809 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12810 %{
12811   match(Set dst (RotateRight src shift));
12812 
12813   ins_cost(INSN_COST);
12814   format %{ "ror    $dst, $src, $shift" %}
12815 
12816   ins_encode %{
12817      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12818   %}
12819   ins_pipe(ialu_reg_reg_vshift);
12820 %}
12821 
12822 // This pattern is automatically generated from aarch64_ad.m4
12823 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12824 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12825 %{
12826   match(Set dst (RotateLeft src shift));
12827 
12828   ins_cost(INSN_COST);
12829   format %{ "rol    $dst, $src, $shift" %}
12830 
12831   ins_encode %{
12832      __ subw(rscratch1, zr, as_Register($shift$$reg));
12833      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12834   %}
12835   ins_pipe(ialu_reg_reg_vshift);
12836 %}
12837 
12838 // This pattern is automatically generated from aarch64_ad.m4
12839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12840 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12841 %{
12842   match(Set dst (RotateLeft src shift));
12843 
12844   ins_cost(INSN_COST);
12845   format %{ "rol    $dst, $src, $shift" %}
12846 
12847   ins_encode %{
12848      __ subw(rscratch1, zr, as_Register($shift$$reg));
12849      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12850   %}
12851   ins_pipe(ialu_reg_reg_vshift);
12852 %}
12853 
12854 
12855 // Add/subtract (extended)
12856 
12857 // This pattern is automatically generated from aarch64_ad.m4
12858 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12859 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12860 %{
12861   match(Set dst (AddL src1 (ConvI2L src2)));
12862   ins_cost(INSN_COST);
12863   format %{ "add  $dst, $src1, $src2, sxtw" %}
12864 
12865    ins_encode %{
12866      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12867             as_Register($src2$$reg), ext::sxtw);
12868    %}
12869   ins_pipe(ialu_reg_reg);
12870 %}
12871 
12872 // This pattern is automatically generated from aarch64_ad.m4
12873 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12874 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12875 %{
12876   match(Set dst (SubL src1 (ConvI2L src2)));
12877   ins_cost(INSN_COST);
12878   format %{ "sub  $dst, $src1, $src2, sxtw" %}
12879 
12880    ins_encode %{
12881      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12882             as_Register($src2$$reg), ext::sxtw);
12883    %}
12884   ins_pipe(ialu_reg_reg);
12885 %}
12886 
12887 // This pattern is automatically generated from aarch64_ad.m4
12888 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12889 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12890 %{
12891   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12892   ins_cost(INSN_COST);
12893   format %{ "add  $dst, $src1, $src2, sxth" %}
12894 
12895    ins_encode %{
12896      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12897             as_Register($src2$$reg), ext::sxth);
12898    %}
12899   ins_pipe(ialu_reg_reg);
12900 %}
12901 
12902 // This pattern is automatically generated from aarch64_ad.m4
12903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12904 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12905 %{
12906   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12907   ins_cost(INSN_COST);
12908   format %{ "add  $dst, $src1, $src2, sxtb" %}
12909 
12910    ins_encode %{
12911      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12912             as_Register($src2$$reg), ext::sxtb);
12913    %}
12914   ins_pipe(ialu_reg_reg);
12915 %}
12916 
12917 // This pattern is automatically generated from aarch64_ad.m4
12918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12919 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12920 %{
12921   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12922   ins_cost(INSN_COST);
12923   format %{ "add  $dst, $src1, $src2, uxtb" %}
12924 
12925    ins_encode %{
12926      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12927             as_Register($src2$$reg), ext::uxtb);
12928    %}
12929   ins_pipe(ialu_reg_reg);
12930 %}
12931 
12932 // This pattern is automatically generated from aarch64_ad.m4
12933 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12934 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12935 %{
12936   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12937   ins_cost(INSN_COST);
12938   format %{ "add  $dst, $src1, $src2, sxth" %}
12939 
12940    ins_encode %{
12941      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12942             as_Register($src2$$reg), ext::sxth);
12943    %}
12944   ins_pipe(ialu_reg_reg);
12945 %}
12946 
12947 // This pattern is automatically generated from aarch64_ad.m4
12948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12949 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12950 %{
12951   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12952   ins_cost(INSN_COST);
12953   format %{ "add  $dst, $src1, $src2, sxtw" %}
12954 
12955    ins_encode %{
12956      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12957             as_Register($src2$$reg), ext::sxtw);
12958    %}
12959   ins_pipe(ialu_reg_reg);
12960 %}
12961 
12962 // This pattern is automatically generated from aarch64_ad.m4
12963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12964 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12965 %{
12966   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12967   ins_cost(INSN_COST);
12968   format %{ "add  $dst, $src1, $src2, sxtb" %}
12969 
12970    ins_encode %{
12971      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12972             as_Register($src2$$reg), ext::sxtb);
12973    %}
12974   ins_pipe(ialu_reg_reg);
12975 %}
12976 
12977 // This pattern is automatically generated from aarch64_ad.m4
12978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12979 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12980 %{
12981   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12982   ins_cost(INSN_COST);
12983   format %{ "add  $dst, $src1, $src2, uxtb" %}
12984 
12985    ins_encode %{
12986      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12987             as_Register($src2$$reg), ext::uxtb);
12988    %}
12989   ins_pipe(ialu_reg_reg);
12990 %}
12991 
12992 // This pattern is automatically generated from aarch64_ad.m4
12993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12994 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12995 %{
12996   match(Set dst (AddI src1 (AndI src2 mask)));
12997   ins_cost(INSN_COST);
12998   format %{ "addw  $dst, $src1, $src2, uxtb" %}
12999 
13000    ins_encode %{
13001      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13002             as_Register($src2$$reg), ext::uxtb);
13003    %}
13004   ins_pipe(ialu_reg_reg);
13005 %}
13006 
13007 // This pattern is automatically generated from aarch64_ad.m4
13008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13009 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13010 %{
13011   match(Set dst (AddI src1 (AndI src2 mask)));
13012   ins_cost(INSN_COST);
13013   format %{ "addw  $dst, $src1, $src2, uxth" %}
13014 
13015    ins_encode %{
13016      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13017             as_Register($src2$$reg), ext::uxth);
13018    %}
13019   ins_pipe(ialu_reg_reg);
13020 %}
13021 
13022 // This pattern is automatically generated from aarch64_ad.m4
13023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13024 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13025 %{
13026   match(Set dst (AddL src1 (AndL src2 mask)));
13027   ins_cost(INSN_COST);
13028   format %{ "add  $dst, $src1, $src2, uxtb" %}
13029 
13030    ins_encode %{
13031      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13032             as_Register($src2$$reg), ext::uxtb);
13033    %}
13034   ins_pipe(ialu_reg_reg);
13035 %}
13036 
13037 // This pattern is automatically generated from aarch64_ad.m4
13038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13039 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13040 %{
13041   match(Set dst (AddL src1 (AndL src2 mask)));
13042   ins_cost(INSN_COST);
13043   format %{ "add  $dst, $src1, $src2, uxth" %}
13044 
13045    ins_encode %{
13046      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13047             as_Register($src2$$reg), ext::uxth);
13048    %}
13049   ins_pipe(ialu_reg_reg);
13050 %}
13051 
13052 // This pattern is automatically generated from aarch64_ad.m4
13053 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13054 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13055 %{
13056   match(Set dst (AddL src1 (AndL src2 mask)));
13057   ins_cost(INSN_COST);
13058   format %{ "add  $dst, $src1, $src2, uxtw" %}
13059 
13060    ins_encode %{
13061      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13062             as_Register($src2$$reg), ext::uxtw);
13063    %}
13064   ins_pipe(ialu_reg_reg);
13065 %}
13066 
13067 // This pattern is automatically generated from aarch64_ad.m4
13068 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13069 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13070 %{
13071   match(Set dst (SubI src1 (AndI src2 mask)));
13072   ins_cost(INSN_COST);
13073   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13074 
13075    ins_encode %{
13076      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13077             as_Register($src2$$reg), ext::uxtb);
13078    %}
13079   ins_pipe(ialu_reg_reg);
13080 %}
13081 
13082 // This pattern is automatically generated from aarch64_ad.m4
13083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13084 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13085 %{
13086   match(Set dst (SubI src1 (AndI src2 mask)));
13087   ins_cost(INSN_COST);
13088   format %{ "subw  $dst, $src1, $src2, uxth" %}
13089 
13090    ins_encode %{
13091      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13092             as_Register($src2$$reg), ext::uxth);
13093    %}
13094   ins_pipe(ialu_reg_reg);
13095 %}
13096 
13097 // This pattern is automatically generated from aarch64_ad.m4
13098 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13099 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13100 %{
13101   match(Set dst (SubL src1 (AndL src2 mask)));
13102   ins_cost(INSN_COST);
13103   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13104 
13105    ins_encode %{
13106      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13107             as_Register($src2$$reg), ext::uxtb);
13108    %}
13109   ins_pipe(ialu_reg_reg);
13110 %}
13111 
13112 // This pattern is automatically generated from aarch64_ad.m4
13113 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13114 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13115 %{
13116   match(Set dst (SubL src1 (AndL src2 mask)));
13117   ins_cost(INSN_COST);
13118   format %{ "sub  $dst, $src1, $src2, uxth" %}
13119 
13120    ins_encode %{
13121      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13122             as_Register($src2$$reg), ext::uxth);
13123    %}
13124   ins_pipe(ialu_reg_reg);
13125 %}
13126 
13127 // This pattern is automatically generated from aarch64_ad.m4
13128 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13129 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13130 %{
13131   match(Set dst (SubL src1 (AndL src2 mask)));
13132   ins_cost(INSN_COST);
13133   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13134 
13135    ins_encode %{
13136      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13137             as_Register($src2$$reg), ext::uxtw);
13138    %}
13139   ins_pipe(ialu_reg_reg);
13140 %}
13141 
13142 
13143 // This pattern is automatically generated from aarch64_ad.m4
13144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13145 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13146 %{
13147   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13148   ins_cost(1.9 * INSN_COST);
13149   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13150 
13151    ins_encode %{
13152      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13153             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13154    %}
13155   ins_pipe(ialu_reg_reg_shift);
13156 %}
13157 
13158 // This pattern is automatically generated from aarch64_ad.m4
13159 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13160 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13161 %{
13162   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13163   ins_cost(1.9 * INSN_COST);
13164   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13165 
13166    ins_encode %{
13167      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13168             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13169    %}
13170   ins_pipe(ialu_reg_reg_shift);
13171 %}
13172 
13173 // This pattern is automatically generated from aarch64_ad.m4
13174 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13175 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13176 %{
13177   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13178   ins_cost(1.9 * INSN_COST);
13179   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13180 
13181    ins_encode %{
13182      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13183             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13184    %}
13185   ins_pipe(ialu_reg_reg_shift);
13186 %}
13187 
13188 // This pattern is automatically generated from aarch64_ad.m4
13189 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13190 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13191 %{
13192   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13193   ins_cost(1.9 * INSN_COST);
13194   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13195 
13196    ins_encode %{
13197      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13198             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13199    %}
13200   ins_pipe(ialu_reg_reg_shift);
13201 %}
13202 
13203 // This pattern is automatically generated from aarch64_ad.m4
13204 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13205 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13206 %{
13207   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13208   ins_cost(1.9 * INSN_COST);
13209   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13210 
13211    ins_encode %{
13212      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13213             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13214    %}
13215   ins_pipe(ialu_reg_reg_shift);
13216 %}
13217 
13218 // This pattern is automatically generated from aarch64_ad.m4
13219 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13220 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13221 %{
13222   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13223   ins_cost(1.9 * INSN_COST);
13224   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13225 
13226    ins_encode %{
13227      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13228             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13229    %}
13230   ins_pipe(ialu_reg_reg_shift);
13231 %}
13232 
13233 // This pattern is automatically generated from aarch64_ad.m4
13234 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13235 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13236 %{
13237   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13238   ins_cost(1.9 * INSN_COST);
13239   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13240 
13241    ins_encode %{
13242      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13243             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13244    %}
13245   ins_pipe(ialu_reg_reg_shift);
13246 %}
13247 
13248 // This pattern is automatically generated from aarch64_ad.m4
13249 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13250 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13251 %{
13252   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13253   ins_cost(1.9 * INSN_COST);
13254   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13255 
13256    ins_encode %{
13257      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13258             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13259    %}
13260   ins_pipe(ialu_reg_reg_shift);
13261 %}
13262 
13263 // This pattern is automatically generated from aarch64_ad.m4
13264 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13265 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13266 %{
13267   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13268   ins_cost(1.9 * INSN_COST);
13269   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13270 
13271    ins_encode %{
13272      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13273             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13274    %}
13275   ins_pipe(ialu_reg_reg_shift);
13276 %}
13277 
13278 // This pattern is automatically generated from aarch64_ad.m4
13279 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13280 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13281 %{
13282   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13283   ins_cost(1.9 * INSN_COST);
13284   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13285 
13286    ins_encode %{
13287      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13288             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13289    %}
13290   ins_pipe(ialu_reg_reg_shift);
13291 %}
13292 
13293 // This pattern is automatically generated from aarch64_ad.m4
13294 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13295 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13296 %{
13297   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13298   ins_cost(1.9 * INSN_COST);
13299   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13300 
13301    ins_encode %{
13302      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13303             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13304    %}
13305   ins_pipe(ialu_reg_reg_shift);
13306 %}
13307 
13308 // This pattern is automatically generated from aarch64_ad.m4
13309 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13310 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13311 %{
13312   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13313   ins_cost(1.9 * INSN_COST);
13314   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13315 
13316    ins_encode %{
13317      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13318             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13319    %}
13320   ins_pipe(ialu_reg_reg_shift);
13321 %}
13322 
13323 // This pattern is automatically generated from aarch64_ad.m4
13324 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13325 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13326 %{
13327   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13328   ins_cost(1.9 * INSN_COST);
13329   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13330 
13331    ins_encode %{
13332      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13333             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13334    %}
13335   ins_pipe(ialu_reg_reg_shift);
13336 %}
13337 
13338 // This pattern is automatically generated from aarch64_ad.m4
13339 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13340 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13341 %{
13342   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13343   ins_cost(1.9 * INSN_COST);
13344   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13345 
13346    ins_encode %{
13347      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13348             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13349    %}
13350   ins_pipe(ialu_reg_reg_shift);
13351 %}
13352 
13353 // This pattern is automatically generated from aarch64_ad.m4
13354 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13355 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13356 %{
13357   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13358   ins_cost(1.9 * INSN_COST);
13359   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13360 
13361    ins_encode %{
13362      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13363             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13364    %}
13365   ins_pipe(ialu_reg_reg_shift);
13366 %}
13367 
13368 // This pattern is automatically generated from aarch64_ad.m4
13369 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13370 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13371 %{
13372   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13373   ins_cost(1.9 * INSN_COST);
13374   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13375 
13376    ins_encode %{
13377      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13378             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13379    %}
13380   ins_pipe(ialu_reg_reg_shift);
13381 %}
13382 
13383 // This pattern is automatically generated from aarch64_ad.m4
13384 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13385 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13386 %{
13387   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13388   ins_cost(1.9 * INSN_COST);
13389   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13390 
13391    ins_encode %{
13392      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13393             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13394    %}
13395   ins_pipe(ialu_reg_reg_shift);
13396 %}
13397 
13398 // This pattern is automatically generated from aarch64_ad.m4
13399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13400 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13401 %{
13402   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13403   ins_cost(1.9 * INSN_COST);
13404   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13405 
13406    ins_encode %{
13407      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13408             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13409    %}
13410   ins_pipe(ialu_reg_reg_shift);
13411 %}
13412 
13413 // This pattern is automatically generated from aarch64_ad.m4
13414 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13415 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13416 %{
13417   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13418   ins_cost(1.9 * INSN_COST);
13419   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13420 
13421    ins_encode %{
13422      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13423             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13424    %}
13425   ins_pipe(ialu_reg_reg_shift);
13426 %}
13427 
13428 // This pattern is automatically generated from aarch64_ad.m4
13429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13430 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13431 %{
13432   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13433   ins_cost(1.9 * INSN_COST);
13434   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13435 
13436    ins_encode %{
13437      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13438             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13439    %}
13440   ins_pipe(ialu_reg_reg_shift);
13441 %}
13442 
13443 // This pattern is automatically generated from aarch64_ad.m4
13444 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13445 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13446 %{
13447   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13448   ins_cost(1.9 * INSN_COST);
13449   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13450 
13451    ins_encode %{
13452      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13453             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13454    %}
13455   ins_pipe(ialu_reg_reg_shift);
13456 %}
13457 
13458 // This pattern is automatically generated from aarch64_ad.m4
13459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13460 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13461 %{
13462   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13463   ins_cost(1.9 * INSN_COST);
13464   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13465 
13466    ins_encode %{
13467      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13468             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13469    %}
13470   ins_pipe(ialu_reg_reg_shift);
13471 %}
13472 
13473 // This pattern is automatically generated from aarch64_ad.m4
13474 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13475 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13476 %{
13477   effect(DEF dst, USE src1, USE src2, USE cr);
13478   ins_cost(INSN_COST * 2);
13479   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
13480 
13481   ins_encode %{
13482     __ cselw($dst$$Register,
13483              $src1$$Register,
13484              $src2$$Register,
13485              Assembler::LT);
13486   %}
13487   ins_pipe(icond_reg_reg);
13488 %}
13489 
13490 // This pattern is automatically generated from aarch64_ad.m4
13491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13492 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13493 %{
13494   effect(DEF dst, USE src1, USE src2, USE cr);
13495   ins_cost(INSN_COST * 2);
13496   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
13497 
13498   ins_encode %{
13499     __ cselw($dst$$Register,
13500              $src1$$Register,
13501              $src2$$Register,
13502              Assembler::GT);
13503   %}
13504   ins_pipe(icond_reg_reg);
13505 %}
13506 
13507 // This pattern is automatically generated from aarch64_ad.m4
13508 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13509 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13510 %{
13511   effect(DEF dst, USE src1, USE cr);
13512   ins_cost(INSN_COST * 2);
13513   format %{ "cselw $dst, $src1, zr lt\t"  %}
13514 
13515   ins_encode %{
13516     __ cselw($dst$$Register,
13517              $src1$$Register,
13518              zr,
13519              Assembler::LT);
13520   %}
13521   ins_pipe(icond_reg);
13522 %}
13523 
13524 // This pattern is automatically generated from aarch64_ad.m4
13525 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13526 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13527 %{
13528   effect(DEF dst, USE src1, USE cr);
13529   ins_cost(INSN_COST * 2);
13530   format %{ "cselw $dst, $src1, zr gt\t"  %}
13531 
13532   ins_encode %{
13533     __ cselw($dst$$Register,
13534              $src1$$Register,
13535              zr,
13536              Assembler::GT);
13537   %}
13538   ins_pipe(icond_reg);
13539 %}
13540 
13541 // This pattern is automatically generated from aarch64_ad.m4
13542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13543 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13544 %{
13545   effect(DEF dst, USE src1, USE cr);
13546   ins_cost(INSN_COST * 2);
13547   format %{ "csincw $dst, $src1, zr le\t"  %}
13548 
13549   ins_encode %{
13550     __ csincw($dst$$Register,
13551              $src1$$Register,
13552              zr,
13553              Assembler::LE);
13554   %}
13555   ins_pipe(icond_reg);
13556 %}
13557 
13558 // This pattern is automatically generated from aarch64_ad.m4
13559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13560 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13561 %{
13562   effect(DEF dst, USE src1, USE cr);
13563   ins_cost(INSN_COST * 2);
13564   format %{ "csincw $dst, $src1, zr gt\t"  %}
13565 
13566   ins_encode %{
13567     __ csincw($dst$$Register,
13568              $src1$$Register,
13569              zr,
13570              Assembler::GT);
13571   %}
13572   ins_pipe(icond_reg);
13573 %}
13574 
13575 // This pattern is automatically generated from aarch64_ad.m4
13576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13577 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13578 %{
13579   effect(DEF dst, USE src1, USE cr);
13580   ins_cost(INSN_COST * 2);
13581   format %{ "csinvw $dst, $src1, zr lt\t"  %}
13582 
13583   ins_encode %{
13584     __ csinvw($dst$$Register,
13585              $src1$$Register,
13586              zr,
13587              Assembler::LT);
13588   %}
13589   ins_pipe(icond_reg);
13590 %}
13591 
13592 // This pattern is automatically generated from aarch64_ad.m4
13593 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13594 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13595 %{
13596   effect(DEF dst, USE src1, USE cr);
13597   ins_cost(INSN_COST * 2);
13598   format %{ "csinvw $dst, $src1, zr ge\t"  %}
13599 
13600   ins_encode %{
13601     __ csinvw($dst$$Register,
13602              $src1$$Register,
13603              zr,
13604              Assembler::GE);
13605   %}
13606   ins_pipe(icond_reg);
13607 %}
13608 
13609 // This pattern is automatically generated from aarch64_ad.m4
13610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13611 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13612 %{
13613   match(Set dst (MinI src imm));
13614   ins_cost(INSN_COST * 3);
13615   expand %{
13616     rFlagsReg cr;
13617     compI_reg_imm0(cr, src);
13618     cmovI_reg_imm0_lt(dst, src, cr);
13619   %}
13620 %}
13621 
13622 // This pattern is automatically generated from aarch64_ad.m4
13623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13624 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13625 %{
13626   match(Set dst (MinI imm src));
13627   ins_cost(INSN_COST * 3);
13628   expand %{
13629     rFlagsReg cr;
13630     compI_reg_imm0(cr, src);
13631     cmovI_reg_imm0_lt(dst, src, cr);
13632   %}
13633 %}
13634 
13635 // This pattern is automatically generated from aarch64_ad.m4
13636 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13637 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13638 %{
13639   match(Set dst (MinI src imm));
13640   ins_cost(INSN_COST * 3);
13641   expand %{
13642     rFlagsReg cr;
13643     compI_reg_imm0(cr, src);
13644     cmovI_reg_imm1_le(dst, src, cr);
13645   %}
13646 %}
13647 
13648 // This pattern is automatically generated from aarch64_ad.m4
13649 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13650 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13651 %{
13652   match(Set dst (MinI imm src));
13653   ins_cost(INSN_COST * 3);
13654   expand %{
13655     rFlagsReg cr;
13656     compI_reg_imm0(cr, src);
13657     cmovI_reg_imm1_le(dst, src, cr);
13658   %}
13659 %}
13660 
13661 // This pattern is automatically generated from aarch64_ad.m4
13662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13663 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13664 %{
13665   match(Set dst (MinI src imm));
13666   ins_cost(INSN_COST * 3);
13667   expand %{
13668     rFlagsReg cr;
13669     compI_reg_imm0(cr, src);
13670     cmovI_reg_immM1_lt(dst, src, cr);
13671   %}
13672 %}
13673 
13674 // This pattern is automatically generated from aarch64_ad.m4
13675 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13676 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13677 %{
13678   match(Set dst (MinI imm src));
13679   ins_cost(INSN_COST * 3);
13680   expand %{
13681     rFlagsReg cr;
13682     compI_reg_imm0(cr, src);
13683     cmovI_reg_immM1_lt(dst, src, cr);
13684   %}
13685 %}
13686 
13687 // This pattern is automatically generated from aarch64_ad.m4
13688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13689 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13690 %{
13691   match(Set dst (MaxI src imm));
13692   ins_cost(INSN_COST * 3);
13693   expand %{
13694     rFlagsReg cr;
13695     compI_reg_imm0(cr, src);
13696     cmovI_reg_imm0_gt(dst, src, cr);
13697   %}
13698 %}
13699 
13700 // This pattern is automatically generated from aarch64_ad.m4
13701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13702 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13703 %{
13704   match(Set dst (MaxI imm src));
13705   ins_cost(INSN_COST * 3);
13706   expand %{
13707     rFlagsReg cr;
13708     compI_reg_imm0(cr, src);
13709     cmovI_reg_imm0_gt(dst, src, cr);
13710   %}
13711 %}
13712 
13713 // This pattern is automatically generated from aarch64_ad.m4
13714 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13715 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13716 %{
13717   match(Set dst (MaxI src imm));
13718   ins_cost(INSN_COST * 3);
13719   expand %{
13720     rFlagsReg cr;
13721     compI_reg_imm0(cr, src);
13722     cmovI_reg_imm1_gt(dst, src, cr);
13723   %}
13724 %}
13725 
13726 // This pattern is automatically generated from aarch64_ad.m4
13727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13728 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13729 %{
13730   match(Set dst (MaxI imm src));
13731   ins_cost(INSN_COST * 3);
13732   expand %{
13733     rFlagsReg cr;
13734     compI_reg_imm0(cr, src);
13735     cmovI_reg_imm1_gt(dst, src, cr);
13736   %}
13737 %}
13738 
13739 // This pattern is automatically generated from aarch64_ad.m4
13740 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13741 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13742 %{
13743   match(Set dst (MaxI src imm));
13744   ins_cost(INSN_COST * 3);
13745   expand %{
13746     rFlagsReg cr;
13747     compI_reg_imm0(cr, src);
13748     cmovI_reg_immM1_ge(dst, src, cr);
13749   %}
13750 %}
13751 
13752 // This pattern is automatically generated from aarch64_ad.m4
13753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13754 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13755 %{
13756   match(Set dst (MaxI imm src));
13757   ins_cost(INSN_COST * 3);
13758   expand %{
13759     rFlagsReg cr;
13760     compI_reg_imm0(cr, src);
13761     cmovI_reg_immM1_ge(dst, src, cr);
13762   %}
13763 %}
13764 
13765 // This pattern is automatically generated from aarch64_ad.m4
13766 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13767 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13768 %{
13769   match(Set dst (ReverseI src));
13770   ins_cost(INSN_COST);
13771   format %{ "rbitw  $dst, $src" %}
13772   ins_encode %{
13773     __ rbitw($dst$$Register, $src$$Register);
13774   %}
13775   ins_pipe(ialu_reg);
13776 %}
13777 
13778 // This pattern is automatically generated from aarch64_ad.m4
13779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13780 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13781 %{
13782   match(Set dst (ReverseL src));
13783   ins_cost(INSN_COST);
13784   format %{ "rbit  $dst, $src" %}
13785   ins_encode %{
13786     __ rbit($dst$$Register, $src$$Register);
13787   %}
13788   ins_pipe(ialu_reg);
13789 %}
13790 
13791 
13792 // END This section of the file is automatically generated. Do not edit --------------
13793 
13794 
13795 // ============================================================================
13796 // Floating Point Arithmetic Instructions
13797 
13798 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13799   match(Set dst (AddHF src1 src2));
13800   format %{ "faddh $dst, $src1, $src2" %}
13801   ins_encode %{
13802     __ faddh($dst$$FloatRegister,
13803              $src1$$FloatRegister,
13804              $src2$$FloatRegister);
13805   %}
13806   ins_pipe(fp_dop_reg_reg_s);
13807 %}
13808 
13809 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13810   match(Set dst (AddF src1 src2));
13811 
13812   ins_cost(INSN_COST * 5);
13813   format %{ "fadds   $dst, $src1, $src2" %}
13814 
13815   ins_encode %{
13816     __ fadds(as_FloatRegister($dst$$reg),
13817              as_FloatRegister($src1$$reg),
13818              as_FloatRegister($src2$$reg));
13819   %}
13820 
13821   ins_pipe(fp_dop_reg_reg_s);
13822 %}
13823 
13824 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13825   match(Set dst (AddD src1 src2));
13826 
13827   ins_cost(INSN_COST * 5);
13828   format %{ "faddd   $dst, $src1, $src2" %}
13829 
13830   ins_encode %{
13831     __ faddd(as_FloatRegister($dst$$reg),
13832              as_FloatRegister($src1$$reg),
13833              as_FloatRegister($src2$$reg));
13834   %}
13835 
13836   ins_pipe(fp_dop_reg_reg_d);
13837 %}
13838 
13839 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13840   match(Set dst (SubHF src1 src2));
13841   format %{ "fsubh $dst, $src1, $src2" %}
13842   ins_encode %{
13843     __ fsubh($dst$$FloatRegister,
13844              $src1$$FloatRegister,
13845              $src2$$FloatRegister);
13846   %}
13847   ins_pipe(fp_dop_reg_reg_s);
13848 %}
13849 
13850 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13851   match(Set dst (SubF src1 src2));
13852 
13853   ins_cost(INSN_COST * 5);
13854   format %{ "fsubs   $dst, $src1, $src2" %}
13855 
13856   ins_encode %{
13857     __ fsubs(as_FloatRegister($dst$$reg),
13858              as_FloatRegister($src1$$reg),
13859              as_FloatRegister($src2$$reg));
13860   %}
13861 
13862   ins_pipe(fp_dop_reg_reg_s);
13863 %}
13864 
13865 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13866   match(Set dst (SubD src1 src2));
13867 
13868   ins_cost(INSN_COST * 5);
13869   format %{ "fsubd   $dst, $src1, $src2" %}
13870 
13871   ins_encode %{
13872     __ fsubd(as_FloatRegister($dst$$reg),
13873              as_FloatRegister($src1$$reg),
13874              as_FloatRegister($src2$$reg));
13875   %}
13876 
13877   ins_pipe(fp_dop_reg_reg_d);
13878 %}
13879 
13880 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13881   match(Set dst (MulHF src1 src2));
13882   format %{ "fmulh $dst, $src1, $src2" %}
13883   ins_encode %{
13884     __ fmulh($dst$$FloatRegister,
13885              $src1$$FloatRegister,
13886              $src2$$FloatRegister);
13887   %}
13888   ins_pipe(fp_dop_reg_reg_s);
13889 %}
13890 
13891 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13892   match(Set dst (MulF src1 src2));
13893 
13894   ins_cost(INSN_COST * 6);
13895   format %{ "fmuls   $dst, $src1, $src2" %}
13896 
13897   ins_encode %{
13898     __ fmuls(as_FloatRegister($dst$$reg),
13899              as_FloatRegister($src1$$reg),
13900              as_FloatRegister($src2$$reg));
13901   %}
13902 
13903   ins_pipe(fp_dop_reg_reg_s);
13904 %}
13905 
13906 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13907   match(Set dst (MulD src1 src2));
13908 
13909   ins_cost(INSN_COST * 6);
13910   format %{ "fmuld   $dst, $src1, $src2" %}
13911 
13912   ins_encode %{
13913     __ fmuld(as_FloatRegister($dst$$reg),
13914              as_FloatRegister($src1$$reg),
13915              as_FloatRegister($src2$$reg));
13916   %}
13917 
13918   ins_pipe(fp_dop_reg_reg_d);
13919 %}
13920 
13921 // src1 * src2 + src3 (half-precision float)
13922 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13923   match(Set dst (FmaHF src3 (Binary src1 src2)));
13924   format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13925   ins_encode %{
13926     assert(UseFMA, "Needs FMA instructions support.");
13927     __ fmaddh($dst$$FloatRegister,
13928               $src1$$FloatRegister,
13929               $src2$$FloatRegister,
13930               $src3$$FloatRegister);
13931   %}
13932   ins_pipe(pipe_class_default);
13933 %}
13934 
13935 // src1 * src2 + src3
13936 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13937   match(Set dst (FmaF src3 (Binary src1 src2)));
13938 
13939   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
13940 
13941   ins_encode %{
13942     assert(UseFMA, "Needs FMA instructions support.");
13943     __ fmadds(as_FloatRegister($dst$$reg),
13944              as_FloatRegister($src1$$reg),
13945              as_FloatRegister($src2$$reg),
13946              as_FloatRegister($src3$$reg));
13947   %}
13948 
13949   ins_pipe(pipe_class_default);
13950 %}
13951 
13952 // src1 * src2 + src3
13953 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13954   match(Set dst (FmaD src3 (Binary src1 src2)));
13955 
13956   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
13957 
13958   ins_encode %{
13959     assert(UseFMA, "Needs FMA instructions support.");
13960     __ fmaddd(as_FloatRegister($dst$$reg),
13961              as_FloatRegister($src1$$reg),
13962              as_FloatRegister($src2$$reg),
13963              as_FloatRegister($src3$$reg));
13964   %}
13965 
13966   ins_pipe(pipe_class_default);
13967 %}
13968 
13969 // src1 * (-src2) + src3
13970 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13971 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13972   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13973 
13974   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
13975 
13976   ins_encode %{
13977     assert(UseFMA, "Needs FMA instructions support.");
13978     __ fmsubs(as_FloatRegister($dst$$reg),
13979               as_FloatRegister($src1$$reg),
13980               as_FloatRegister($src2$$reg),
13981               as_FloatRegister($src3$$reg));
13982   %}
13983 
13984   ins_pipe(pipe_class_default);
13985 %}
13986 
13987 // src1 * (-src2) + src3
13988 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13989 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13990   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13991 
13992   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
13993 
13994   ins_encode %{
13995     assert(UseFMA, "Needs FMA instructions support.");
13996     __ fmsubd(as_FloatRegister($dst$$reg),
13997               as_FloatRegister($src1$$reg),
13998               as_FloatRegister($src2$$reg),
13999               as_FloatRegister($src3$$reg));
14000   %}
14001 
14002   ins_pipe(pipe_class_default);
14003 %}
14004 
14005 // src1 * (-src2) - src3
14006 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14007 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14008   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14009 
14010   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14011 
14012   ins_encode %{
14013     assert(UseFMA, "Needs FMA instructions support.");
14014     __ fnmadds(as_FloatRegister($dst$$reg),
14015                as_FloatRegister($src1$$reg),
14016                as_FloatRegister($src2$$reg),
14017                as_FloatRegister($src3$$reg));
14018   %}
14019 
14020   ins_pipe(pipe_class_default);
14021 %}
14022 
14023 // src1 * (-src2) - src3
14024 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14025 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14026   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14027 
14028   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14029 
14030   ins_encode %{
14031     assert(UseFMA, "Needs FMA instructions support.");
14032     __ fnmaddd(as_FloatRegister($dst$$reg),
14033                as_FloatRegister($src1$$reg),
14034                as_FloatRegister($src2$$reg),
14035                as_FloatRegister($src3$$reg));
14036   %}
14037 
14038   ins_pipe(pipe_class_default);
14039 %}
14040 
14041 // src1 * src2 - src3
14042 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14043   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14044 
14045   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14046 
14047   ins_encode %{
14048     assert(UseFMA, "Needs FMA instructions support.");
14049     __ fnmsubs(as_FloatRegister($dst$$reg),
14050                as_FloatRegister($src1$$reg),
14051                as_FloatRegister($src2$$reg),
14052                as_FloatRegister($src3$$reg));
14053   %}
14054 
14055   ins_pipe(pipe_class_default);
14056 %}
14057 
14058 // src1 * src2 - src3
14059 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14060   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14061 
14062   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14063 
14064   ins_encode %{
14065     assert(UseFMA, "Needs FMA instructions support.");
14066     // n.b. insn name should be fnmsubd
14067     __ fnmsub(as_FloatRegister($dst$$reg),
14068               as_FloatRegister($src1$$reg),
14069               as_FloatRegister($src2$$reg),
14070               as_FloatRegister($src3$$reg));
14071   %}
14072 
14073   ins_pipe(pipe_class_default);
14074 %}
14075 
14076 // Math.max(HH)H (half-precision float)
14077 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14078   match(Set dst (MaxHF src1 src2));
14079   format %{ "fmaxh $dst, $src1, $src2" %}
14080   ins_encode %{
14081     __ fmaxh($dst$$FloatRegister,
14082              $src1$$FloatRegister,
14083              $src2$$FloatRegister);
14084   %}
14085   ins_pipe(fp_dop_reg_reg_s);
14086 %}
14087 
14088 // Math.min(HH)H (half-precision float)
14089 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14090   match(Set dst (MinHF src1 src2));
14091   format %{ "fminh $dst, $src1, $src2" %}
14092   ins_encode %{
14093     __ fminh($dst$$FloatRegister,
14094              $src1$$FloatRegister,
14095              $src2$$FloatRegister);
14096   %}
14097   ins_pipe(fp_dop_reg_reg_s);
14098 %}
14099 
14100 // Math.max(FF)F
14101 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14102   match(Set dst (MaxF src1 src2));
14103 
14104   format %{ "fmaxs   $dst, $src1, $src2" %}
14105   ins_encode %{
14106     __ fmaxs(as_FloatRegister($dst$$reg),
14107              as_FloatRegister($src1$$reg),
14108              as_FloatRegister($src2$$reg));
14109   %}
14110 
14111   ins_pipe(fp_dop_reg_reg_s);
14112 %}
14113 
14114 // Math.min(FF)F
14115 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14116   match(Set dst (MinF src1 src2));
14117 
14118   format %{ "fmins   $dst, $src1, $src2" %}
14119   ins_encode %{
14120     __ fmins(as_FloatRegister($dst$$reg),
14121              as_FloatRegister($src1$$reg),
14122              as_FloatRegister($src2$$reg));
14123   %}
14124 
14125   ins_pipe(fp_dop_reg_reg_s);
14126 %}
14127 
14128 // Math.max(DD)D
14129 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14130   match(Set dst (MaxD src1 src2));
14131 
14132   format %{ "fmaxd   $dst, $src1, $src2" %}
14133   ins_encode %{
14134     __ fmaxd(as_FloatRegister($dst$$reg),
14135              as_FloatRegister($src1$$reg),
14136              as_FloatRegister($src2$$reg));
14137   %}
14138 
14139   ins_pipe(fp_dop_reg_reg_d);
14140 %}
14141 
14142 // Math.min(DD)D
14143 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14144   match(Set dst (MinD src1 src2));
14145 
14146   format %{ "fmind   $dst, $src1, $src2" %}
14147   ins_encode %{
14148     __ fmind(as_FloatRegister($dst$$reg),
14149              as_FloatRegister($src1$$reg),
14150              as_FloatRegister($src2$$reg));
14151   %}
14152 
14153   ins_pipe(fp_dop_reg_reg_d);
14154 %}
14155 
14156 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14157   match(Set dst (DivHF src1  src2));
14158   format %{ "fdivh $dst, $src1, $src2" %}
14159   ins_encode %{
14160     __ fdivh($dst$$FloatRegister,
14161              $src1$$FloatRegister,
14162              $src2$$FloatRegister);
14163   %}
14164   ins_pipe(fp_div_s);
14165 %}
14166 
14167 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14168   match(Set dst (DivF src1  src2));
14169 
14170   ins_cost(INSN_COST * 18);
14171   format %{ "fdivs   $dst, $src1, $src2" %}
14172 
14173   ins_encode %{
14174     __ fdivs(as_FloatRegister($dst$$reg),
14175              as_FloatRegister($src1$$reg),
14176              as_FloatRegister($src2$$reg));
14177   %}
14178 
14179   ins_pipe(fp_div_s);
14180 %}
14181 
14182 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14183   match(Set dst (DivD src1  src2));
14184 
14185   ins_cost(INSN_COST * 32);
14186   format %{ "fdivd   $dst, $src1, $src2" %}
14187 
14188   ins_encode %{
14189     __ fdivd(as_FloatRegister($dst$$reg),
14190              as_FloatRegister($src1$$reg),
14191              as_FloatRegister($src2$$reg));
14192   %}
14193 
14194   ins_pipe(fp_div_d);
14195 %}
14196 
14197 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14198   match(Set dst (NegF src));
14199 
14200   ins_cost(INSN_COST * 3);
14201   format %{ "fneg   $dst, $src" %}
14202 
14203   ins_encode %{
14204     __ fnegs(as_FloatRegister($dst$$reg),
14205              as_FloatRegister($src$$reg));
14206   %}
14207 
14208   ins_pipe(fp_uop_s);
14209 %}
14210 
14211 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14212   match(Set dst (NegD src));
14213 
14214   ins_cost(INSN_COST * 3);
14215   format %{ "fnegd   $dst, $src" %}
14216 
14217   ins_encode %{
14218     __ fnegd(as_FloatRegister($dst$$reg),
14219              as_FloatRegister($src$$reg));
14220   %}
14221 
14222   ins_pipe(fp_uop_d);
14223 %}
14224 
14225 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14226 %{
14227   match(Set dst (AbsI src));
14228 
14229   effect(KILL cr);
14230   ins_cost(INSN_COST * 2);
14231   format %{ "cmpw  $src, zr\n\t"
14232             "cnegw $dst, $src, Assembler::LT\t# int abs"
14233   %}
14234 
14235   ins_encode %{
14236     __ cmpw(as_Register($src$$reg), zr);
14237     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14238   %}
14239   ins_pipe(pipe_class_default);
14240 %}
14241 
14242 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14243 %{
14244   match(Set dst (AbsL src));
14245 
14246   effect(KILL cr);
14247   ins_cost(INSN_COST * 2);
14248   format %{ "cmp  $src, zr\n\t"
14249             "cneg $dst, $src, Assembler::LT\t# long abs"
14250   %}
14251 
14252   ins_encode %{
14253     __ cmp(as_Register($src$$reg), zr);
14254     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14255   %}
14256   ins_pipe(pipe_class_default);
14257 %}
14258 
14259 instruct absF_reg(vRegF dst, vRegF src) %{
14260   match(Set dst (AbsF src));
14261 
14262   ins_cost(INSN_COST * 3);
14263   format %{ "fabss   $dst, $src" %}
14264   ins_encode %{
14265     __ fabss(as_FloatRegister($dst$$reg),
14266              as_FloatRegister($src$$reg));
14267   %}
14268 
14269   ins_pipe(fp_uop_s);
14270 %}
14271 
14272 instruct absD_reg(vRegD dst, vRegD src) %{
14273   match(Set dst (AbsD src));
14274 
14275   ins_cost(INSN_COST * 3);
14276   format %{ "fabsd   $dst, $src" %}
14277   ins_encode %{
14278     __ fabsd(as_FloatRegister($dst$$reg),
14279              as_FloatRegister($src$$reg));
14280   %}
14281 
14282   ins_pipe(fp_uop_d);
14283 %}
14284 
14285 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14286   match(Set dst (AbsF (SubF src1 src2)));
14287 
14288   ins_cost(INSN_COST * 3);
14289   format %{ "fabds   $dst, $src1, $src2" %}
14290   ins_encode %{
14291     __ fabds(as_FloatRegister($dst$$reg),
14292              as_FloatRegister($src1$$reg),
14293              as_FloatRegister($src2$$reg));
14294   %}
14295 
14296   ins_pipe(fp_uop_s);
14297 %}
14298 
14299 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14300   match(Set dst (AbsD (SubD src1 src2)));
14301 
14302   ins_cost(INSN_COST * 3);
14303   format %{ "fabdd   $dst, $src1, $src2" %}
14304   ins_encode %{
14305     __ fabdd(as_FloatRegister($dst$$reg),
14306              as_FloatRegister($src1$$reg),
14307              as_FloatRegister($src2$$reg));
14308   %}
14309 
14310   ins_pipe(fp_uop_d);
14311 %}
14312 
14313 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14314   match(Set dst (SqrtD src));
14315 
14316   ins_cost(INSN_COST * 50);
14317   format %{ "fsqrtd  $dst, $src" %}
14318   ins_encode %{
14319     __ fsqrtd(as_FloatRegister($dst$$reg),
14320              as_FloatRegister($src$$reg));
14321   %}
14322 
14323   ins_pipe(fp_div_s);
14324 %}
14325 
14326 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14327   match(Set dst (SqrtF src));
14328 
14329   ins_cost(INSN_COST * 50);
14330   format %{ "fsqrts  $dst, $src" %}
14331   ins_encode %{
14332     __ fsqrts(as_FloatRegister($dst$$reg),
14333              as_FloatRegister($src$$reg));
14334   %}
14335 
14336   ins_pipe(fp_div_d);
14337 %}
14338 
14339 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
14340   match(Set dst (SqrtHF src));
14341   format %{ "fsqrth $dst, $src" %}
14342   ins_encode %{
14343     __ fsqrth($dst$$FloatRegister,
14344               $src$$FloatRegister);
14345   %}
14346   ins_pipe(fp_div_s);
14347 %}
14348 
14349 // Math.rint, floor, ceil
14350 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14351   match(Set dst (RoundDoubleMode src rmode));
14352   format %{ "frint  $dst, $src, $rmode" %}
14353   ins_encode %{
14354     switch ($rmode$$constant) {
14355       case RoundDoubleModeNode::rmode_rint:
14356         __ frintnd(as_FloatRegister($dst$$reg),
14357                    as_FloatRegister($src$$reg));
14358         break;
14359       case RoundDoubleModeNode::rmode_floor:
14360         __ frintmd(as_FloatRegister($dst$$reg),
14361                    as_FloatRegister($src$$reg));
14362         break;
14363       case RoundDoubleModeNode::rmode_ceil:
14364         __ frintpd(as_FloatRegister($dst$$reg),
14365                    as_FloatRegister($src$$reg));
14366         break;
14367     }
14368   %}
14369   ins_pipe(fp_uop_d);
14370 %}
14371 
14372 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14373   match(Set dst (CopySignD src1 (Binary src2 zero)));
14374   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14375   format %{ "CopySignD  $dst $src1 $src2" %}
14376   ins_encode %{
14377     FloatRegister dst = as_FloatRegister($dst$$reg),
14378                   src1 = as_FloatRegister($src1$$reg),
14379                   src2 = as_FloatRegister($src2$$reg),
14380                   zero = as_FloatRegister($zero$$reg);
14381     __ fnegd(dst, zero);
14382     __ bsl(dst, __ T8B, src2, src1);
14383   %}
14384   ins_pipe(fp_uop_d);
14385 %}
14386 
14387 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14388   match(Set dst (CopySignF src1 src2));
14389   effect(TEMP_DEF dst, USE src1, USE src2);
14390   format %{ "CopySignF  $dst $src1 $src2" %}
14391   ins_encode %{
14392     FloatRegister dst = as_FloatRegister($dst$$reg),
14393                   src1 = as_FloatRegister($src1$$reg),
14394                   src2 = as_FloatRegister($src2$$reg);
14395     __ movi(dst, __ T2S, 0x80, 24);
14396     __ bsl(dst, __ T8B, src2, src1);
14397   %}
14398   ins_pipe(fp_uop_d);
14399 %}
14400 
14401 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14402   match(Set dst (SignumD src (Binary zero one)));
14403   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14404   format %{ "signumD  $dst, $src" %}
14405   ins_encode %{
14406     FloatRegister src = as_FloatRegister($src$$reg),
14407                   dst = as_FloatRegister($dst$$reg),
14408                   zero = as_FloatRegister($zero$$reg),
14409                   one = as_FloatRegister($one$$reg);
14410     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14411     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14412     // Bit selection instruction gets bit from "one" for each enabled bit in
14413     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14414     // NaN the whole "src" will be copied because "dst" is zero. For all other
14415     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14416     // from "src", and all other bits are copied from 1.0.
14417     __ bsl(dst, __ T8B, one, src);
14418   %}
14419   ins_pipe(fp_uop_d);
14420 %}
14421 
14422 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14423   match(Set dst (SignumF src (Binary zero one)));
14424   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14425   format %{ "signumF  $dst, $src" %}
14426   ins_encode %{
14427     FloatRegister src = as_FloatRegister($src$$reg),
14428                   dst = as_FloatRegister($dst$$reg),
14429                   zero = as_FloatRegister($zero$$reg),
14430                   one = as_FloatRegister($one$$reg);
14431     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14432     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14433     // Bit selection instruction gets bit from "one" for each enabled bit in
14434     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14435     // NaN the whole "src" will be copied because "dst" is zero. For all other
14436     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14437     // from "src", and all other bits are copied from 1.0.
14438     __ bsl(dst, __ T8B, one, src);
14439   %}
14440   ins_pipe(fp_uop_d);
14441 %}
14442 
14443 instruct onspinwait() %{
14444   match(OnSpinWait);
14445   ins_cost(INSN_COST);
14446 
14447   format %{ "onspinwait" %}
14448 
14449   ins_encode %{
14450     __ spin_wait();
14451   %}
14452   ins_pipe(pipe_class_empty);
14453 %}
14454 
14455 // ============================================================================
14456 // Logical Instructions
14457 
14458 // Integer Logical Instructions
14459 
14460 // And Instructions
14461 
14462 
14463 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14464   match(Set dst (AndI src1 src2));
14465 
14466   format %{ "andw  $dst, $src1, $src2\t# int" %}
14467 
14468   ins_cost(INSN_COST);
14469   ins_encode %{
14470     __ andw(as_Register($dst$$reg),
14471             as_Register($src1$$reg),
14472             as_Register($src2$$reg));
14473   %}
14474 
14475   ins_pipe(ialu_reg_reg);
14476 %}
14477 
14478 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14479   match(Set dst (AndI src1 src2));
14480 
14481   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14482 
14483   ins_cost(INSN_COST);
14484   ins_encode %{
14485     __ andw(as_Register($dst$$reg),
14486             as_Register($src1$$reg),
14487             (uint64_t)($src2$$constant));
14488   %}
14489 
14490   ins_pipe(ialu_reg_imm);
14491 %}
14492 
14493 // Or Instructions
14494 
14495 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14496   match(Set dst (OrI src1 src2));
14497 
14498   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14499 
14500   ins_cost(INSN_COST);
14501   ins_encode %{
14502     __ orrw(as_Register($dst$$reg),
14503             as_Register($src1$$reg),
14504             as_Register($src2$$reg));
14505   %}
14506 
14507   ins_pipe(ialu_reg_reg);
14508 %}
14509 
14510 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14511   match(Set dst (OrI src1 src2));
14512 
14513   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14514 
14515   ins_cost(INSN_COST);
14516   ins_encode %{
14517     __ orrw(as_Register($dst$$reg),
14518             as_Register($src1$$reg),
14519             (uint64_t)($src2$$constant));
14520   %}
14521 
14522   ins_pipe(ialu_reg_imm);
14523 %}
14524 
14525 // Xor Instructions
14526 
14527 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14528   match(Set dst (XorI src1 src2));
14529 
14530   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14531 
14532   ins_cost(INSN_COST);
14533   ins_encode %{
14534     __ eorw(as_Register($dst$$reg),
14535             as_Register($src1$$reg),
14536             as_Register($src2$$reg));
14537   %}
14538 
14539   ins_pipe(ialu_reg_reg);
14540 %}
14541 
14542 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14543   match(Set dst (XorI src1 src2));
14544 
14545   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14546 
14547   ins_cost(INSN_COST);
14548   ins_encode %{
14549     __ eorw(as_Register($dst$$reg),
14550             as_Register($src1$$reg),
14551             (uint64_t)($src2$$constant));
14552   %}
14553 
14554   ins_pipe(ialu_reg_imm);
14555 %}
14556 
14557 // Long Logical Instructions
14558 // TODO
14559 
14560 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14561   match(Set dst (AndL src1 src2));
14562 
14563   format %{ "and  $dst, $src1, $src2\t# int" %}
14564 
14565   ins_cost(INSN_COST);
14566   ins_encode %{
14567     __ andr(as_Register($dst$$reg),
14568             as_Register($src1$$reg),
14569             as_Register($src2$$reg));
14570   %}
14571 
14572   ins_pipe(ialu_reg_reg);
14573 %}
14574 
14575 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14576   match(Set dst (AndL src1 src2));
14577 
14578   format %{ "and  $dst, $src1, $src2\t# int" %}
14579 
14580   ins_cost(INSN_COST);
14581   ins_encode %{
14582     __ andr(as_Register($dst$$reg),
14583             as_Register($src1$$reg),
14584             (uint64_t)($src2$$constant));
14585   %}
14586 
14587   ins_pipe(ialu_reg_imm);
14588 %}
14589 
14590 // Or Instructions
14591 
14592 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14593   match(Set dst (OrL src1 src2));
14594 
14595   format %{ "orr  $dst, $src1, $src2\t# int" %}
14596 
14597   ins_cost(INSN_COST);
14598   ins_encode %{
14599     __ orr(as_Register($dst$$reg),
14600            as_Register($src1$$reg),
14601            as_Register($src2$$reg));
14602   %}
14603 
14604   ins_pipe(ialu_reg_reg);
14605 %}
14606 
14607 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14608   match(Set dst (OrL src1 src2));
14609 
14610   format %{ "orr  $dst, $src1, $src2\t# int" %}
14611 
14612   ins_cost(INSN_COST);
14613   ins_encode %{
14614     __ orr(as_Register($dst$$reg),
14615            as_Register($src1$$reg),
14616            (uint64_t)($src2$$constant));
14617   %}
14618 
14619   ins_pipe(ialu_reg_imm);
14620 %}
14621 
14622 // Xor Instructions
14623 
14624 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14625   match(Set dst (XorL src1 src2));
14626 
14627   format %{ "eor  $dst, $src1, $src2\t# int" %}
14628 
14629   ins_cost(INSN_COST);
14630   ins_encode %{
14631     __ eor(as_Register($dst$$reg),
14632            as_Register($src1$$reg),
14633            as_Register($src2$$reg));
14634   %}
14635 
14636   ins_pipe(ialu_reg_reg);
14637 %}
14638 
14639 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14640   match(Set dst (XorL src1 src2));
14641 
14642   ins_cost(INSN_COST);
14643   format %{ "eor  $dst, $src1, $src2\t# int" %}
14644 
14645   ins_encode %{
14646     __ eor(as_Register($dst$$reg),
14647            as_Register($src1$$reg),
14648            (uint64_t)($src2$$constant));
14649   %}
14650 
14651   ins_pipe(ialu_reg_imm);
14652 %}
14653 
14654 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14655 %{
14656   match(Set dst (ConvI2L src));
14657 
14658   ins_cost(INSN_COST);
14659   format %{ "sxtw  $dst, $src\t# i2l" %}
14660   ins_encode %{
14661     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14662   %}
14663   ins_pipe(ialu_reg_shift);
14664 %}
14665 
14666 // this pattern occurs in bigmath arithmetic
14667 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14668 %{
14669   match(Set dst (AndL (ConvI2L src) mask));
14670 
14671   ins_cost(INSN_COST);
14672   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14673   ins_encode %{
14674     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14675   %}
14676 
14677   ins_pipe(ialu_reg_shift);
14678 %}
14679 
14680 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14681   match(Set dst (ConvL2I src));
14682 
14683   ins_cost(INSN_COST);
14684   format %{ "movw  $dst, $src \t// l2i" %}
14685 
14686   ins_encode %{
14687     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14688   %}
14689 
14690   ins_pipe(ialu_reg);
14691 %}
14692 
14693 instruct convD2F_reg(vRegF dst, vRegD src) %{
14694   match(Set dst (ConvD2F src));
14695 
14696   ins_cost(INSN_COST * 5);
14697   format %{ "fcvtd  $dst, $src \t// d2f" %}
14698 
14699   ins_encode %{
14700     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14701   %}
14702 
14703   ins_pipe(fp_d2f);
14704 %}
14705 
14706 instruct convF2D_reg(vRegD dst, vRegF src) %{
14707   match(Set dst (ConvF2D src));
14708 
14709   ins_cost(INSN_COST * 5);
14710   format %{ "fcvts  $dst, $src \t// f2d" %}
14711 
14712   ins_encode %{
14713     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14714   %}
14715 
14716   ins_pipe(fp_f2d);
14717 %}
14718 
14719 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14720   match(Set dst (ConvF2I src));
14721 
14722   ins_cost(INSN_COST * 5);
14723   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14724 
14725   ins_encode %{
14726     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14727   %}
14728 
14729   ins_pipe(fp_f2i);
14730 %}
14731 
14732 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14733   match(Set dst (ConvF2L src));
14734 
14735   ins_cost(INSN_COST * 5);
14736   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14737 
14738   ins_encode %{
14739     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14740   %}
14741 
14742   ins_pipe(fp_f2l);
14743 %}
14744 
14745 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14746   match(Set dst (ConvF2HF src));
14747   format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14748             "smov $dst, $tmp\t# move result from $tmp to $dst"
14749   %}
14750   effect(TEMP tmp);
14751   ins_encode %{
14752       __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14753   %}
14754   ins_pipe(pipe_slow);
14755 %}
14756 
14757 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14758   match(Set dst (ConvHF2F src));
14759   format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14760             "fcvt $dst, $tmp\t# convert half to single precision"
14761   %}
14762   effect(TEMP tmp);
14763   ins_encode %{
14764       __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14765   %}
14766   ins_pipe(pipe_slow);
14767 %}
14768 
14769 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14770   match(Set dst (ConvI2F src));
14771 
14772   ins_cost(INSN_COST * 5);
14773   format %{ "scvtfws  $dst, $src \t// i2f" %}
14774 
14775   ins_encode %{
14776     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14777   %}
14778 
14779   ins_pipe(fp_i2f);
14780 %}
14781 
14782 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14783   match(Set dst (ConvL2F src));
14784 
14785   ins_cost(INSN_COST * 5);
14786   format %{ "scvtfs  $dst, $src \t// l2f" %}
14787 
14788   ins_encode %{
14789     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14790   %}
14791 
14792   ins_pipe(fp_l2f);
14793 %}
14794 
14795 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14796   match(Set dst (ConvD2I src));
14797 
14798   ins_cost(INSN_COST * 5);
14799   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14800 
14801   ins_encode %{
14802     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14803   %}
14804 
14805   ins_pipe(fp_d2i);
14806 %}
14807 
14808 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14809   match(Set dst (ConvD2L src));
14810 
14811   ins_cost(INSN_COST * 5);
14812   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14813 
14814   ins_encode %{
14815     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14816   %}
14817 
14818   ins_pipe(fp_d2l);
14819 %}
14820 
14821 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14822   match(Set dst (ConvI2D src));
14823 
14824   ins_cost(INSN_COST * 5);
14825   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14826 
14827   ins_encode %{
14828     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14829   %}
14830 
14831   ins_pipe(fp_i2d);
14832 %}
14833 
14834 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14835   match(Set dst (ConvL2D src));
14836 
14837   ins_cost(INSN_COST * 5);
14838   format %{ "scvtfd  $dst, $src \t// l2d" %}
14839 
14840   ins_encode %{
14841     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14842   %}
14843 
14844   ins_pipe(fp_l2d);
14845 %}
14846 
14847 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14848 %{
14849   match(Set dst (RoundD src));
14850   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14851   format %{ "java_round_double $dst,$src"%}
14852   ins_encode %{
14853     __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14854                          as_FloatRegister($ftmp$$reg));
14855   %}
14856   ins_pipe(pipe_slow);
14857 %}
14858 
14859 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14860 %{
14861   match(Set dst (RoundF src));
14862   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14863   format %{ "java_round_float $dst,$src"%}
14864   ins_encode %{
14865     __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14866                         as_FloatRegister($ftmp$$reg));
14867   %}
14868   ins_pipe(pipe_slow);
14869 %}
14870 
14871 // stack <-> reg and reg <-> reg shuffles with no conversion
14872 
14873 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14874 
14875   match(Set dst (MoveF2I src));
14876 
14877   effect(DEF dst, USE src);
14878 
14879   ins_cost(4 * INSN_COST);
14880 
14881   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14882 
14883   ins_encode %{
14884     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14885   %}
14886 
14887   ins_pipe(iload_reg_reg);
14888 
14889 %}
14890 
14891 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14892 
14893   match(Set dst (MoveI2F src));
14894 
14895   effect(DEF dst, USE src);
14896 
14897   ins_cost(4 * INSN_COST);
14898 
14899   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14900 
14901   ins_encode %{
14902     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14903   %}
14904 
14905   ins_pipe(pipe_class_memory);
14906 
14907 %}
14908 
14909 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14910 
14911   match(Set dst (MoveD2L src));
14912 
14913   effect(DEF dst, USE src);
14914 
14915   ins_cost(4 * INSN_COST);
14916 
14917   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14918 
14919   ins_encode %{
14920     __ ldr($dst$$Register, Address(sp, $src$$disp));
14921   %}
14922 
14923   ins_pipe(iload_reg_reg);
14924 
14925 %}
14926 
14927 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14928 
14929   match(Set dst (MoveL2D src));
14930 
14931   effect(DEF dst, USE src);
14932 
14933   ins_cost(4 * INSN_COST);
14934 
14935   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14936 
14937   ins_encode %{
14938     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14939   %}
14940 
14941   ins_pipe(pipe_class_memory);
14942 
14943 %}
14944 
14945 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14946 
14947   match(Set dst (MoveF2I src));
14948 
14949   effect(DEF dst, USE src);
14950 
14951   ins_cost(INSN_COST);
14952 
14953   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14954 
14955   ins_encode %{
14956     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14957   %}
14958 
14959   ins_pipe(pipe_class_memory);
14960 
14961 %}
14962 
14963 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14964 
14965   match(Set dst (MoveI2F src));
14966 
14967   effect(DEF dst, USE src);
14968 
14969   ins_cost(INSN_COST);
14970 
14971   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14972 
14973   ins_encode %{
14974     __ strw($src$$Register, Address(sp, $dst$$disp));
14975   %}
14976 
14977   ins_pipe(istore_reg_reg);
14978 
14979 %}
14980 
14981 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14982 
14983   match(Set dst (MoveD2L src));
14984 
14985   effect(DEF dst, USE src);
14986 
14987   ins_cost(INSN_COST);
14988 
14989   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14990 
14991   ins_encode %{
14992     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14993   %}
14994 
14995   ins_pipe(pipe_class_memory);
14996 
14997 %}
14998 
14999 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15000 
15001   match(Set dst (MoveL2D src));
15002 
15003   effect(DEF dst, USE src);
15004 
15005   ins_cost(INSN_COST);
15006 
15007   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15008 
15009   ins_encode %{
15010     __ str($src$$Register, Address(sp, $dst$$disp));
15011   %}
15012 
15013   ins_pipe(istore_reg_reg);
15014 
15015 %}
15016 
15017 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15018 
15019   match(Set dst (MoveF2I src));
15020 
15021   effect(DEF dst, USE src);
15022 
15023   ins_cost(INSN_COST);
15024 
15025   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15026 
15027   ins_encode %{
15028     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15029   %}
15030 
15031   ins_pipe(fp_f2i);
15032 
15033 %}
15034 
15035 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15036 
15037   match(Set dst (MoveI2F src));
15038 
15039   effect(DEF dst, USE src);
15040 
15041   ins_cost(INSN_COST);
15042 
15043   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15044 
15045   ins_encode %{
15046     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15047   %}
15048 
15049   ins_pipe(fp_i2f);
15050 
15051 %}
15052 
15053 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15054 
15055   match(Set dst (MoveD2L src));
15056 
15057   effect(DEF dst, USE src);
15058 
15059   ins_cost(INSN_COST);
15060 
15061   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15062 
15063   ins_encode %{
15064     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15065   %}
15066 
15067   ins_pipe(fp_d2l);
15068 
15069 %}
15070 
15071 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15072 
15073   match(Set dst (MoveL2D src));
15074 
15075   effect(DEF dst, USE src);
15076 
15077   ins_cost(INSN_COST);
15078 
15079   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15080 
15081   ins_encode %{
15082     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15083   %}
15084 
15085   ins_pipe(fp_l2d);
15086 
15087 %}
15088 
15089 // ============================================================================
15090 // clearing of an array
15091 
15092 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15093 %{
15094   match(Set dummy (ClearArray cnt base));
15095   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15096 
15097   ins_cost(4 * INSN_COST);
15098   format %{ "ClearArray $cnt, $base" %}
15099 
15100   ins_encode %{
15101     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15102     if (tpc == nullptr) {
15103       ciEnv::current()->record_failure("CodeCache is full");
15104       return;
15105     }
15106   %}
15107 
15108   ins_pipe(pipe_class_memory);
15109 %}
15110 
15111 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15112 %{
15113   predicate((uint64_t)n->in(2)->get_long()
15114             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15115   match(Set dummy (ClearArray cnt base));
15116   effect(TEMP temp, USE_KILL base, KILL cr);
15117 
15118   ins_cost(4 * INSN_COST);
15119   format %{ "ClearArray $cnt, $base" %}
15120 
15121   ins_encode %{
15122     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15123     if (tpc == nullptr) {
15124       ciEnv::current()->record_failure("CodeCache is full");
15125       return;
15126     }
15127   %}
15128 
15129   ins_pipe(pipe_class_memory);
15130 %}
15131 
15132 // ============================================================================
15133 // Overflow Math Instructions
15134 
15135 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15136 %{
15137   match(Set cr (OverflowAddI op1 op2));
15138 
15139   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15140   ins_cost(INSN_COST);
15141   ins_encode %{
15142     __ cmnw($op1$$Register, $op2$$Register);
15143   %}
15144 
15145   ins_pipe(icmp_reg_reg);
15146 %}
15147 
15148 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15149 %{
15150   match(Set cr (OverflowAddI op1 op2));
15151 
15152   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15153   ins_cost(INSN_COST);
15154   ins_encode %{
15155     __ cmnw($op1$$Register, $op2$$constant);
15156   %}
15157 
15158   ins_pipe(icmp_reg_imm);
15159 %}
15160 
15161 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15162 %{
15163   match(Set cr (OverflowAddL op1 op2));
15164 
15165   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15166   ins_cost(INSN_COST);
15167   ins_encode %{
15168     __ cmn($op1$$Register, $op2$$Register);
15169   %}
15170 
15171   ins_pipe(icmp_reg_reg);
15172 %}
15173 
15174 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15175 %{
15176   match(Set cr (OverflowAddL op1 op2));
15177 
15178   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
15179   ins_cost(INSN_COST);
15180   ins_encode %{
15181     __ adds(zr, $op1$$Register, $op2$$constant);
15182   %}
15183 
15184   ins_pipe(icmp_reg_imm);
15185 %}
15186 
15187 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15188 %{
15189   match(Set cr (OverflowSubI op1 op2));
15190 
15191   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15192   ins_cost(INSN_COST);
15193   ins_encode %{
15194     __ cmpw($op1$$Register, $op2$$Register);
15195   %}
15196 
15197   ins_pipe(icmp_reg_reg);
15198 %}
15199 
15200 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15201 %{
15202   match(Set cr (OverflowSubI op1 op2));
15203 
15204   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15205   ins_cost(INSN_COST);
15206   ins_encode %{
15207     __ cmpw($op1$$Register, $op2$$constant);
15208   %}
15209 
15210   ins_pipe(icmp_reg_imm);
15211 %}
15212 
15213 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15214 %{
15215   match(Set cr (OverflowSubL op1 op2));
15216 
15217   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15218   ins_cost(INSN_COST);
15219   ins_encode %{
15220     __ cmp($op1$$Register, $op2$$Register);
15221   %}
15222 
15223   ins_pipe(icmp_reg_reg);
15224 %}
15225 
15226 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15227 %{
15228   match(Set cr (OverflowSubL op1 op2));
15229 
15230   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15231   ins_cost(INSN_COST);
15232   ins_encode %{
15233     __ subs(zr, $op1$$Register, $op2$$constant);
15234   %}
15235 
15236   ins_pipe(icmp_reg_imm);
15237 %}
15238 
15239 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15240 %{
15241   match(Set cr (OverflowSubI zero op1));
15242 
15243   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15244   ins_cost(INSN_COST);
15245   ins_encode %{
15246     __ cmpw(zr, $op1$$Register);
15247   %}
15248 
15249   ins_pipe(icmp_reg_imm);
15250 %}
15251 
15252 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15253 %{
15254   match(Set cr (OverflowSubL zero op1));
15255 
15256   format %{ "cmp   zr, $op1\t# overflow check long" %}
15257   ins_cost(INSN_COST);
15258   ins_encode %{
15259     __ cmp(zr, $op1$$Register);
15260   %}
15261 
15262   ins_pipe(icmp_reg_imm);
15263 %}
15264 
15265 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15266 %{
15267   match(Set cr (OverflowMulI op1 op2));
15268 
15269   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15270             "cmp   rscratch1, rscratch1, sxtw\n\t"
15271             "movw  rscratch1, #0x80000000\n\t"
15272             "cselw rscratch1, rscratch1, zr, NE\n\t"
15273             "cmpw  rscratch1, #1" %}
15274   ins_cost(5 * INSN_COST);
15275   ins_encode %{
15276     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15277     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15278     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15279     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15280     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15281   %}
15282 
15283   ins_pipe(pipe_slow);
15284 %}
15285 
15286 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15287 %{
15288   match(If cmp (OverflowMulI op1 op2));
15289   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15290             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15291   effect(USE labl, KILL cr);
15292 
15293   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15294             "cmp   rscratch1, rscratch1, sxtw\n\t"
15295             "b$cmp   $labl" %}
15296   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15297   ins_encode %{
15298     Label* L = $labl$$label;
15299     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15300     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15301     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15302     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15303   %}
15304 
15305   ins_pipe(pipe_serial);
15306 %}
15307 
15308 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15309 %{
15310   match(Set cr (OverflowMulL op1 op2));
15311 
15312   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15313             "smulh rscratch2, $op1, $op2\n\t"
15314             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15315             "movw  rscratch1, #0x80000000\n\t"
15316             "cselw rscratch1, rscratch1, zr, NE\n\t"
15317             "cmpw  rscratch1, #1" %}
15318   ins_cost(6 * INSN_COST);
15319   ins_encode %{
15320     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15321     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15322     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15323     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15324     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15325     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15326   %}
15327 
15328   ins_pipe(pipe_slow);
15329 %}
15330 
15331 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15332 %{
15333   match(If cmp (OverflowMulL op1 op2));
15334   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15335             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15336   effect(USE labl, KILL cr);
15337 
15338   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15339             "smulh rscratch2, $op1, $op2\n\t"
15340             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15341             "b$cmp $labl" %}
15342   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15343   ins_encode %{
15344     Label* L = $labl$$label;
15345     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15346     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15347     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15348     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15349     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15350   %}
15351 
15352   ins_pipe(pipe_serial);
15353 %}
15354 
15355 // ============================================================================
15356 // Compare Instructions
15357 
15358 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15359 %{
15360   match(Set cr (CmpI op1 op2));
15361 
15362   effect(DEF cr, USE op1, USE op2);
15363 
15364   ins_cost(INSN_COST);
15365   format %{ "cmpw  $op1, $op2" %}
15366 
15367   ins_encode(aarch64_enc_cmpw(op1, op2));
15368 
15369   ins_pipe(icmp_reg_reg);
15370 %}
15371 
15372 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15373 %{
15374   match(Set cr (CmpI op1 zero));
15375 
15376   effect(DEF cr, USE op1);
15377 
15378   ins_cost(INSN_COST);
15379   format %{ "cmpw $op1, 0" %}
15380 
15381   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15382 
15383   ins_pipe(icmp_reg_imm);
15384 %}
15385 
15386 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15387 %{
15388   match(Set cr (CmpI op1 op2));
15389 
15390   effect(DEF cr, USE op1);
15391 
15392   ins_cost(INSN_COST);
15393   format %{ "cmpw  $op1, $op2" %}
15394 
15395   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15396 
15397   ins_pipe(icmp_reg_imm);
15398 %}
15399 
15400 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15401 %{
15402   match(Set cr (CmpI op1 op2));
15403 
15404   effect(DEF cr, USE op1);
15405 
15406   ins_cost(INSN_COST * 2);
15407   format %{ "cmpw  $op1, $op2" %}
15408 
15409   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15410 
15411   ins_pipe(icmp_reg_imm);
15412 %}
15413 
15414 // Unsigned compare Instructions; really, same as signed compare
15415 // except it should only be used to feed an If or a CMovI which takes a
15416 // cmpOpU.
15417 
15418 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15419 %{
15420   match(Set cr (CmpU op1 op2));
15421 
15422   effect(DEF cr, USE op1, USE op2);
15423 
15424   ins_cost(INSN_COST);
15425   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15426 
15427   ins_encode(aarch64_enc_cmpw(op1, op2));
15428 
15429   ins_pipe(icmp_reg_reg);
15430 %}
15431 
15432 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15433 %{
15434   match(Set cr (CmpU op1 zero));
15435 
15436   effect(DEF cr, USE op1);
15437 
15438   ins_cost(INSN_COST);
15439   format %{ "cmpw $op1, #0\t# unsigned" %}
15440 
15441   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15442 
15443   ins_pipe(icmp_reg_imm);
15444 %}
15445 
15446 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15447 %{
15448   match(Set cr (CmpU op1 op2));
15449 
15450   effect(DEF cr, USE op1);
15451 
15452   ins_cost(INSN_COST);
15453   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15454 
15455   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15456 
15457   ins_pipe(icmp_reg_imm);
15458 %}
15459 
15460 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15461 %{
15462   match(Set cr (CmpU op1 op2));
15463 
15464   effect(DEF cr, USE op1);
15465 
15466   ins_cost(INSN_COST * 2);
15467   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15468 
15469   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15470 
15471   ins_pipe(icmp_reg_imm);
15472 %}
15473 
15474 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15475 %{
15476   match(Set cr (CmpL op1 op2));
15477 
15478   effect(DEF cr, USE op1, USE op2);
15479 
15480   ins_cost(INSN_COST);
15481   format %{ "cmp  $op1, $op2" %}
15482 
15483   ins_encode(aarch64_enc_cmp(op1, op2));
15484 
15485   ins_pipe(icmp_reg_reg);
15486 %}
15487 
15488 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15489 %{
15490   match(Set cr (CmpL op1 zero));
15491 
15492   effect(DEF cr, USE op1);
15493 
15494   ins_cost(INSN_COST);
15495   format %{ "tst  $op1" %}
15496 
15497   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15498 
15499   ins_pipe(icmp_reg_imm);
15500 %}
15501 
15502 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15503 %{
15504   match(Set cr (CmpL op1 op2));
15505 
15506   effect(DEF cr, USE op1);
15507 
15508   ins_cost(INSN_COST);
15509   format %{ "cmp  $op1, $op2" %}
15510 
15511   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15512 
15513   ins_pipe(icmp_reg_imm);
15514 %}
15515 
15516 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15517 %{
15518   match(Set cr (CmpL op1 op2));
15519 
15520   effect(DEF cr, USE op1);
15521 
15522   ins_cost(INSN_COST * 2);
15523   format %{ "cmp  $op1, $op2" %}
15524 
15525   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15526 
15527   ins_pipe(icmp_reg_imm);
15528 %}
15529 
15530 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15531 %{
15532   match(Set cr (CmpUL op1 op2));
15533 
15534   effect(DEF cr, USE op1, USE op2);
15535 
15536   ins_cost(INSN_COST);
15537   format %{ "cmp  $op1, $op2" %}
15538 
15539   ins_encode(aarch64_enc_cmp(op1, op2));
15540 
15541   ins_pipe(icmp_reg_reg);
15542 %}
15543 
15544 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15545 %{
15546   match(Set cr (CmpUL op1 zero));
15547 
15548   effect(DEF cr, USE op1);
15549 
15550   ins_cost(INSN_COST);
15551   format %{ "tst  $op1" %}
15552 
15553   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15554 
15555   ins_pipe(icmp_reg_imm);
15556 %}
15557 
15558 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15559 %{
15560   match(Set cr (CmpUL op1 op2));
15561 
15562   effect(DEF cr, USE op1);
15563 
15564   ins_cost(INSN_COST);
15565   format %{ "cmp  $op1, $op2" %}
15566 
15567   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15568 
15569   ins_pipe(icmp_reg_imm);
15570 %}
15571 
15572 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15573 %{
15574   match(Set cr (CmpUL op1 op2));
15575 
15576   effect(DEF cr, USE op1);
15577 
15578   ins_cost(INSN_COST * 2);
15579   format %{ "cmp  $op1, $op2" %}
15580 
15581   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15582 
15583   ins_pipe(icmp_reg_imm);
15584 %}
15585 
15586 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15587 %{
15588   match(Set cr (CmpP op1 op2));
15589 
15590   effect(DEF cr, USE op1, USE op2);
15591 
15592   ins_cost(INSN_COST);
15593   format %{ "cmp  $op1, $op2\t // ptr" %}
15594 
15595   ins_encode(aarch64_enc_cmpp(op1, op2));
15596 
15597   ins_pipe(icmp_reg_reg);
15598 %}
15599 
15600 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15601 %{
15602   match(Set cr (CmpN op1 op2));
15603 
15604   effect(DEF cr, USE op1, USE op2);
15605 
15606   ins_cost(INSN_COST);
15607   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15608 
15609   ins_encode(aarch64_enc_cmpn(op1, op2));
15610 
15611   ins_pipe(icmp_reg_reg);
15612 %}
15613 
15614 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15615 %{
15616   match(Set cr (CmpP op1 zero));
15617 
15618   effect(DEF cr, USE op1, USE zero);
15619 
15620   ins_cost(INSN_COST);
15621   format %{ "cmp  $op1, 0\t // ptr" %}
15622 
15623   ins_encode(aarch64_enc_testp(op1));
15624 
15625   ins_pipe(icmp_reg_imm);
15626 %}
15627 
15628 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15629 %{
15630   match(Set cr (CmpN op1 zero));
15631 
15632   effect(DEF cr, USE op1, USE zero);
15633 
15634   ins_cost(INSN_COST);
15635   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15636 
15637   ins_encode(aarch64_enc_testn(op1));
15638 
15639   ins_pipe(icmp_reg_imm);
15640 %}
15641 
15642 // FP comparisons
15643 //
15644 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15645 // using normal cmpOp. See declaration of rFlagsReg for details.
15646 
15647 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15648 %{
15649   match(Set cr (CmpF src1 src2));
15650 
15651   ins_cost(3 * INSN_COST);
15652   format %{ "fcmps $src1, $src2" %}
15653 
15654   ins_encode %{
15655     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15656   %}
15657 
15658   ins_pipe(pipe_class_compare);
15659 %}
15660 
15661 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15662 %{
15663   match(Set cr (CmpF src1 src2));
15664 
15665   ins_cost(3 * INSN_COST);
15666   format %{ "fcmps $src1, 0.0" %}
15667 
15668   ins_encode %{
15669     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15670   %}
15671 
15672   ins_pipe(pipe_class_compare);
15673 %}
15674 // FROM HERE
15675 
15676 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15677 %{
15678   match(Set cr (CmpD src1 src2));
15679 
15680   ins_cost(3 * INSN_COST);
15681   format %{ "fcmpd $src1, $src2" %}
15682 
15683   ins_encode %{
15684     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15685   %}
15686 
15687   ins_pipe(pipe_class_compare);
15688 %}
15689 
15690 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15691 %{
15692   match(Set cr (CmpD src1 src2));
15693 
15694   ins_cost(3 * INSN_COST);
15695   format %{ "fcmpd $src1, 0.0" %}
15696 
15697   ins_encode %{
15698     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15699   %}
15700 
15701   ins_pipe(pipe_class_compare);
15702 %}
15703 
15704 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15705 %{
15706   match(Set dst (CmpF3 src1 src2));
15707   effect(KILL cr);
15708 
15709   ins_cost(5 * INSN_COST);
15710   format %{ "fcmps $src1, $src2\n\t"
15711             "csinvw($dst, zr, zr, eq\n\t"
15712             "csnegw($dst, $dst, $dst, lt)"
15713   %}
15714 
15715   ins_encode %{
15716     Label done;
15717     FloatRegister s1 = as_FloatRegister($src1$$reg);
15718     FloatRegister s2 = as_FloatRegister($src2$$reg);
15719     Register d = as_Register($dst$$reg);
15720     __ fcmps(s1, s2);
15721     // installs 0 if EQ else -1
15722     __ csinvw(d, zr, zr, Assembler::EQ);
15723     // keeps -1 if less or unordered else installs 1
15724     __ csnegw(d, d, d, Assembler::LT);
15725     __ bind(done);
15726   %}
15727 
15728   ins_pipe(pipe_class_default);
15729 
15730 %}
15731 
15732 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15733 %{
15734   match(Set dst (CmpD3 src1 src2));
15735   effect(KILL cr);
15736 
15737   ins_cost(5 * INSN_COST);
15738   format %{ "fcmpd $src1, $src2\n\t"
15739             "csinvw($dst, zr, zr, eq\n\t"
15740             "csnegw($dst, $dst, $dst, lt)"
15741   %}
15742 
15743   ins_encode %{
15744     Label done;
15745     FloatRegister s1 = as_FloatRegister($src1$$reg);
15746     FloatRegister s2 = as_FloatRegister($src2$$reg);
15747     Register d = as_Register($dst$$reg);
15748     __ fcmpd(s1, s2);
15749     // installs 0 if EQ else -1
15750     __ csinvw(d, zr, zr, Assembler::EQ);
15751     // keeps -1 if less or unordered else installs 1
15752     __ csnegw(d, d, d, Assembler::LT);
15753     __ bind(done);
15754   %}
15755   ins_pipe(pipe_class_default);
15756 
15757 %}
15758 
15759 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15760 %{
15761   match(Set dst (CmpF3 src1 zero));
15762   effect(KILL cr);
15763 
15764   ins_cost(5 * INSN_COST);
15765   format %{ "fcmps $src1, 0.0\n\t"
15766             "csinvw($dst, zr, zr, eq\n\t"
15767             "csnegw($dst, $dst, $dst, lt)"
15768   %}
15769 
15770   ins_encode %{
15771     Label done;
15772     FloatRegister s1 = as_FloatRegister($src1$$reg);
15773     Register d = as_Register($dst$$reg);
15774     __ fcmps(s1, 0.0);
15775     // installs 0 if EQ else -1
15776     __ csinvw(d, zr, zr, Assembler::EQ);
15777     // keeps -1 if less or unordered else installs 1
15778     __ csnegw(d, d, d, Assembler::LT);
15779     __ bind(done);
15780   %}
15781 
15782   ins_pipe(pipe_class_default);
15783 
15784 %}
15785 
15786 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15787 %{
15788   match(Set dst (CmpD3 src1 zero));
15789   effect(KILL cr);
15790 
15791   ins_cost(5 * INSN_COST);
15792   format %{ "fcmpd $src1, 0.0\n\t"
15793             "csinvw($dst, zr, zr, eq\n\t"
15794             "csnegw($dst, $dst, $dst, lt)"
15795   %}
15796 
15797   ins_encode %{
15798     Label done;
15799     FloatRegister s1 = as_FloatRegister($src1$$reg);
15800     Register d = as_Register($dst$$reg);
15801     __ fcmpd(s1, 0.0);
15802     // installs 0 if EQ else -1
15803     __ csinvw(d, zr, zr, Assembler::EQ);
15804     // keeps -1 if less or unordered else installs 1
15805     __ csnegw(d, d, d, Assembler::LT);
15806     __ bind(done);
15807   %}
15808   ins_pipe(pipe_class_default);
15809 
15810 %}
15811 
15812 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15813 %{
15814   match(Set dst (CmpLTMask p q));
15815   effect(KILL cr);
15816 
15817   ins_cost(3 * INSN_COST);
15818 
15819   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15820             "csetw $dst, lt\n\t"
15821             "subw $dst, zr, $dst"
15822   %}
15823 
15824   ins_encode %{
15825     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15826     __ csetw(as_Register($dst$$reg), Assembler::LT);
15827     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15828   %}
15829 
15830   ins_pipe(ialu_reg_reg);
15831 %}
15832 
15833 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15834 %{
15835   match(Set dst (CmpLTMask src zero));
15836   effect(KILL cr);
15837 
15838   ins_cost(INSN_COST);
15839 
15840   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15841 
15842   ins_encode %{
15843     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15844   %}
15845 
15846   ins_pipe(ialu_reg_shift);
15847 %}
15848 
15849 // ============================================================================
15850 // Max and Min
15851 
15852 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15853 
15854 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15855 %{
15856   effect(DEF cr, USE src);
15857   ins_cost(INSN_COST);
15858   format %{ "cmpw $src, 0" %}
15859 
15860   ins_encode %{
15861     __ cmpw($src$$Register, 0);
15862   %}
15863   ins_pipe(icmp_reg_imm);
15864 %}
15865 
15866 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15867 %{
15868   match(Set dst (MinI src1 src2));
15869   ins_cost(INSN_COST * 3);
15870 
15871   expand %{
15872     rFlagsReg cr;
15873     compI_reg_reg(cr, src1, src2);
15874     cmovI_reg_reg_lt(dst, src1, src2, cr);
15875   %}
15876 %}
15877 
15878 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15879 %{
15880   match(Set dst (MaxI src1 src2));
15881   ins_cost(INSN_COST * 3);
15882 
15883   expand %{
15884     rFlagsReg cr;
15885     compI_reg_reg(cr, src1, src2);
15886     cmovI_reg_reg_gt(dst, src1, src2, cr);
15887   %}
15888 %}
15889 
15890 
15891 // ============================================================================
15892 // Branch Instructions
15893 
15894 // Direct Branch.
15895 instruct branch(label lbl)
15896 %{
15897   match(Goto);
15898 
15899   effect(USE lbl);
15900 
15901   ins_cost(BRANCH_COST);
15902   format %{ "b  $lbl" %}
15903 
15904   ins_encode(aarch64_enc_b(lbl));
15905 
15906   ins_pipe(pipe_branch);
15907 %}
15908 
15909 // Conditional Near Branch
15910 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15911 %{
15912   // Same match rule as `branchConFar'.
15913   match(If cmp cr);
15914 
15915   effect(USE lbl);
15916 
15917   ins_cost(BRANCH_COST);
15918   // If set to 1 this indicates that the current instruction is a
15919   // short variant of a long branch. This avoids using this
15920   // instruction in first-pass matching. It will then only be used in
15921   // the `Shorten_branches' pass.
15922   // ins_short_branch(1);
15923   format %{ "b$cmp  $lbl" %}
15924 
15925   ins_encode(aarch64_enc_br_con(cmp, lbl));
15926 
15927   ins_pipe(pipe_branch_cond);
15928 %}
15929 
15930 // Conditional Near Branch Unsigned
15931 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15932 %{
15933   // Same match rule as `branchConFar'.
15934   match(If cmp cr);
15935 
15936   effect(USE lbl);
15937 
15938   ins_cost(BRANCH_COST);
15939   // If set to 1 this indicates that the current instruction is a
15940   // short variant of a long branch. This avoids using this
15941   // instruction in first-pass matching. It will then only be used in
15942   // the `Shorten_branches' pass.
15943   // ins_short_branch(1);
15944   format %{ "b$cmp  $lbl\t# unsigned" %}
15945 
15946   ins_encode(aarch64_enc_br_conU(cmp, lbl));
15947 
15948   ins_pipe(pipe_branch_cond);
15949 %}
15950 
15951 // Make use of CBZ and CBNZ.  These instructions, as well as being
15952 // shorter than (cmp; branch), have the additional benefit of not
15953 // killing the flags.
15954 
15955 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15956   match(If cmp (CmpI op1 op2));
15957   effect(USE labl);
15958 
15959   ins_cost(BRANCH_COST);
15960   format %{ "cbw$cmp   $op1, $labl" %}
15961   ins_encode %{
15962     Label* L = $labl$$label;
15963     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15964     if (cond == Assembler::EQ)
15965       __ cbzw($op1$$Register, *L);
15966     else
15967       __ cbnzw($op1$$Register, *L);
15968   %}
15969   ins_pipe(pipe_cmp_branch);
15970 %}
15971 
15972 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15973   match(If cmp (CmpL op1 op2));
15974   effect(USE labl);
15975 
15976   ins_cost(BRANCH_COST);
15977   format %{ "cb$cmp   $op1, $labl" %}
15978   ins_encode %{
15979     Label* L = $labl$$label;
15980     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15981     if (cond == Assembler::EQ)
15982       __ cbz($op1$$Register, *L);
15983     else
15984       __ cbnz($op1$$Register, *L);
15985   %}
15986   ins_pipe(pipe_cmp_branch);
15987 %}
15988 
15989 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15990   match(If cmp (CmpP op1 op2));
15991   effect(USE labl);
15992 
15993   ins_cost(BRANCH_COST);
15994   format %{ "cb$cmp   $op1, $labl" %}
15995   ins_encode %{
15996     Label* L = $labl$$label;
15997     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15998     if (cond == Assembler::EQ)
15999       __ cbz($op1$$Register, *L);
16000     else
16001       __ cbnz($op1$$Register, *L);
16002   %}
16003   ins_pipe(pipe_cmp_branch);
16004 %}
16005 
16006 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16007   match(If cmp (CmpN op1 op2));
16008   effect(USE labl);
16009 
16010   ins_cost(BRANCH_COST);
16011   format %{ "cbw$cmp   $op1, $labl" %}
16012   ins_encode %{
16013     Label* L = $labl$$label;
16014     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16015     if (cond == Assembler::EQ)
16016       __ cbzw($op1$$Register, *L);
16017     else
16018       __ cbnzw($op1$$Register, *L);
16019   %}
16020   ins_pipe(pipe_cmp_branch);
16021 %}
16022 
16023 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16024   match(If cmp (CmpP (DecodeN oop) zero));
16025   effect(USE labl);
16026 
16027   ins_cost(BRANCH_COST);
16028   format %{ "cb$cmp   $oop, $labl" %}
16029   ins_encode %{
16030     Label* L = $labl$$label;
16031     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16032     if (cond == Assembler::EQ)
16033       __ cbzw($oop$$Register, *L);
16034     else
16035       __ cbnzw($oop$$Register, *L);
16036   %}
16037   ins_pipe(pipe_cmp_branch);
16038 %}
16039 
16040 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16041   match(If cmp (CmpU op1 op2));
16042   effect(USE labl);
16043 
16044   ins_cost(BRANCH_COST);
16045   format %{ "cbw$cmp   $op1, $labl" %}
16046   ins_encode %{
16047     Label* L = $labl$$label;
16048     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16049     if (cond == Assembler::EQ || cond == Assembler::LS) {
16050       __ cbzw($op1$$Register, *L);
16051     } else {
16052       assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16053       __ cbnzw($op1$$Register, *L);
16054     }
16055   %}
16056   ins_pipe(pipe_cmp_branch);
16057 %}
16058 
16059 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
16060   match(If cmp (CmpUL op1 op2));
16061   effect(USE labl);
16062 
16063   ins_cost(BRANCH_COST);
16064   format %{ "cb$cmp   $op1, $labl" %}
16065   ins_encode %{
16066     Label* L = $labl$$label;
16067     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16068     if (cond == Assembler::EQ || cond == Assembler::LS) {
16069       __ cbz($op1$$Register, *L);
16070     } else {
16071       assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16072       __ cbnz($op1$$Register, *L);
16073     }
16074   %}
16075   ins_pipe(pipe_cmp_branch);
16076 %}
16077 
16078 // Test bit and Branch
16079 
16080 // Patterns for short (< 32KiB) variants
16081 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16082   match(If cmp (CmpL op1 op2));
16083   effect(USE labl);
16084 
16085   ins_cost(BRANCH_COST);
16086   format %{ "cb$cmp   $op1, $labl # long" %}
16087   ins_encode %{
16088     Label* L = $labl$$label;
16089     Assembler::Condition cond =
16090       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16091     __ tbr(cond, $op1$$Register, 63, *L);
16092   %}
16093   ins_pipe(pipe_cmp_branch);
16094   ins_short_branch(1);
16095 %}
16096 
16097 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16098   match(If cmp (CmpI op1 op2));
16099   effect(USE labl);
16100 
16101   ins_cost(BRANCH_COST);
16102   format %{ "cb$cmp   $op1, $labl # int" %}
16103   ins_encode %{
16104     Label* L = $labl$$label;
16105     Assembler::Condition cond =
16106       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16107     __ tbr(cond, $op1$$Register, 31, *L);
16108   %}
16109   ins_pipe(pipe_cmp_branch);
16110   ins_short_branch(1);
16111 %}
16112 
16113 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16114   match(If cmp (CmpL (AndL op1 op2) op3));
16115   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16116   effect(USE labl);
16117 
16118   ins_cost(BRANCH_COST);
16119   format %{ "tb$cmp   $op1, $op2, $labl" %}
16120   ins_encode %{
16121     Label* L = $labl$$label;
16122     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16123     int bit = exact_log2_long($op2$$constant);
16124     __ tbr(cond, $op1$$Register, bit, *L);
16125   %}
16126   ins_pipe(pipe_cmp_branch);
16127   ins_short_branch(1);
16128 %}
16129 
16130 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16131   match(If cmp (CmpI (AndI op1 op2) op3));
16132   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16133   effect(USE labl);
16134 
16135   ins_cost(BRANCH_COST);
16136   format %{ "tb$cmp   $op1, $op2, $labl" %}
16137   ins_encode %{
16138     Label* L = $labl$$label;
16139     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16140     int bit = exact_log2((juint)$op2$$constant);
16141     __ tbr(cond, $op1$$Register, bit, *L);
16142   %}
16143   ins_pipe(pipe_cmp_branch);
16144   ins_short_branch(1);
16145 %}
16146 
16147 // And far variants
16148 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16149   match(If cmp (CmpL op1 op2));
16150   effect(USE labl);
16151 
16152   ins_cost(BRANCH_COST);
16153   format %{ "cb$cmp   $op1, $labl # long" %}
16154   ins_encode %{
16155     Label* L = $labl$$label;
16156     Assembler::Condition cond =
16157       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16158     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16159   %}
16160   ins_pipe(pipe_cmp_branch);
16161 %}
16162 
16163 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16164   match(If cmp (CmpI op1 op2));
16165   effect(USE labl);
16166 
16167   ins_cost(BRANCH_COST);
16168   format %{ "cb$cmp   $op1, $labl # int" %}
16169   ins_encode %{
16170     Label* L = $labl$$label;
16171     Assembler::Condition cond =
16172       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16173     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16174   %}
16175   ins_pipe(pipe_cmp_branch);
16176 %}
16177 
16178 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16179   match(If cmp (CmpL (AndL op1 op2) op3));
16180   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16181   effect(USE labl);
16182 
16183   ins_cost(BRANCH_COST);
16184   format %{ "tb$cmp   $op1, $op2, $labl" %}
16185   ins_encode %{
16186     Label* L = $labl$$label;
16187     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16188     int bit = exact_log2_long($op2$$constant);
16189     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16190   %}
16191   ins_pipe(pipe_cmp_branch);
16192 %}
16193 
16194 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16195   match(If cmp (CmpI (AndI op1 op2) op3));
16196   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16197   effect(USE labl);
16198 
16199   ins_cost(BRANCH_COST);
16200   format %{ "tb$cmp   $op1, $op2, $labl" %}
16201   ins_encode %{
16202     Label* L = $labl$$label;
16203     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16204     int bit = exact_log2((juint)$op2$$constant);
16205     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16206   %}
16207   ins_pipe(pipe_cmp_branch);
16208 %}
16209 
16210 // Test bits
16211 
16212 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16213   match(Set cr (CmpL (AndL op1 op2) op3));
16214   predicate(Assembler::operand_valid_for_logical_immediate
16215             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16216 
16217   ins_cost(INSN_COST);
16218   format %{ "tst $op1, $op2 # long" %}
16219   ins_encode %{
16220     __ tst($op1$$Register, $op2$$constant);
16221   %}
16222   ins_pipe(ialu_reg_reg);
16223 %}
16224 
16225 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16226   match(Set cr (CmpI (AndI op1 op2) op3));
16227   predicate(Assembler::operand_valid_for_logical_immediate
16228             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16229 
16230   ins_cost(INSN_COST);
16231   format %{ "tst $op1, $op2 # int" %}
16232   ins_encode %{
16233     __ tstw($op1$$Register, $op2$$constant);
16234   %}
16235   ins_pipe(ialu_reg_reg);
16236 %}
16237 
16238 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16239   match(Set cr (CmpL (AndL op1 op2) op3));
16240 
16241   ins_cost(INSN_COST);
16242   format %{ "tst $op1, $op2 # long" %}
16243   ins_encode %{
16244     __ tst($op1$$Register, $op2$$Register);
16245   %}
16246   ins_pipe(ialu_reg_reg);
16247 %}
16248 
16249 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16250   match(Set cr (CmpI (AndI op1 op2) op3));
16251 
16252   ins_cost(INSN_COST);
16253   format %{ "tstw $op1, $op2 # int" %}
16254   ins_encode %{
16255     __ tstw($op1$$Register, $op2$$Register);
16256   %}
16257   ins_pipe(ialu_reg_reg);
16258 %}
16259 
16260 
16261 // Conditional Far Branch
16262 // Conditional Far Branch Unsigned
16263 // TODO: fixme
16264 
16265 // counted loop end branch near
16266 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16267 %{
16268   match(CountedLoopEnd cmp cr);
16269 
16270   effect(USE lbl);
16271 
16272   ins_cost(BRANCH_COST);
16273   // short variant.
16274   // ins_short_branch(1);
16275   format %{ "b$cmp $lbl \t// counted loop end" %}
16276 
16277   ins_encode(aarch64_enc_br_con(cmp, lbl));
16278 
16279   ins_pipe(pipe_branch);
16280 %}
16281 
16282 // counted loop end branch far
16283 // TODO: fixme
16284 
16285 // ============================================================================
16286 // inlined locking and unlocking
16287 
16288 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16289 %{
16290   match(Set cr (FastLock object box));
16291   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16292 
16293   ins_cost(5 * INSN_COST);
16294   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16295 
16296   ins_encode %{
16297     __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16298   %}
16299 
16300   ins_pipe(pipe_serial);
16301 %}
16302 
16303 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16304 %{
16305   match(Set cr (FastUnlock object box));
16306   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16307 
16308   ins_cost(5 * INSN_COST);
16309   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16310 
16311   ins_encode %{
16312     __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16313   %}
16314 
16315   ins_pipe(pipe_serial);
16316 %}
16317 
16318 // ============================================================================
16319 // Safepoint Instructions
16320 
16321 // TODO
16322 // provide a near and far version of this code
16323 
16324 instruct safePoint(rFlagsReg cr, iRegP poll)
16325 %{
16326   match(SafePoint poll);
16327   effect(KILL cr);
16328 
16329   format %{
16330     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16331   %}
16332   ins_encode %{
16333     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16334   %}
16335   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16336 %}
16337 
16338 
16339 // ============================================================================
16340 // Procedure Call/Return Instructions
16341 
16342 // Call Java Static Instruction
16343 
16344 instruct CallStaticJavaDirect(method meth)
16345 %{
16346   match(CallStaticJava);
16347 
16348   effect(USE meth);
16349 
16350   ins_cost(CALL_COST);
16351 
16352   format %{ "call,static $meth \t// ==> " %}
16353 
16354   ins_encode(aarch64_enc_java_static_call(meth),
16355              aarch64_enc_call_epilog);
16356 
16357   ins_pipe(pipe_class_call);
16358 %}
16359 
16360 // TO HERE
16361 
16362 // Call Java Dynamic Instruction
16363 instruct CallDynamicJavaDirect(method meth)
16364 %{
16365   match(CallDynamicJava);
16366 
16367   effect(USE meth);
16368 
16369   ins_cost(CALL_COST);
16370 
16371   format %{ "CALL,dynamic $meth \t// ==> " %}
16372 
16373   ins_encode(aarch64_enc_java_dynamic_call(meth),
16374              aarch64_enc_call_epilog);
16375 
16376   ins_pipe(pipe_class_call);
16377 %}
16378 
16379 // Call Runtime Instruction
16380 
16381 instruct CallRuntimeDirect(method meth)
16382 %{
16383   match(CallRuntime);
16384 
16385   effect(USE meth);
16386 
16387   ins_cost(CALL_COST);
16388 
16389   format %{ "CALL, runtime $meth" %}
16390 
16391   ins_encode( aarch64_enc_java_to_runtime(meth) );
16392 
16393   ins_pipe(pipe_class_call);
16394 %}
16395 
16396 // Call Runtime Instruction
16397 
16398 instruct CallLeafDirect(method meth)
16399 %{
16400   match(CallLeaf);
16401 
16402   effect(USE meth);
16403 
16404   ins_cost(CALL_COST);
16405 
16406   format %{ "CALL, runtime leaf $meth" %}
16407 
16408   ins_encode( aarch64_enc_java_to_runtime(meth) );
16409 
16410   ins_pipe(pipe_class_call);
16411 %}
16412 
16413 // Call Runtime Instruction without safepoint and with vector arguments
16414 instruct CallLeafDirectVector(method meth)
16415 %{
16416   match(CallLeafVector);
16417 
16418   effect(USE meth);
16419 
16420   ins_cost(CALL_COST);
16421 
16422   format %{ "CALL, runtime leaf vector $meth" %}
16423 
16424   ins_encode(aarch64_enc_java_to_runtime(meth));
16425 
16426   ins_pipe(pipe_class_call);
16427 %}
16428 
16429 // Call Runtime Instruction
16430 
16431 instruct CallLeafNoFPDirect(method meth)
16432 %{
16433   match(CallLeafNoFP);
16434 
16435   effect(USE meth);
16436 
16437   ins_cost(CALL_COST);
16438 
16439   format %{ "CALL, runtime leaf nofp $meth" %}
16440 
16441   ins_encode( aarch64_enc_java_to_runtime(meth) );
16442 
16443   ins_pipe(pipe_class_call);
16444 %}
16445 
16446 // Tail Call; Jump from runtime stub to Java code.
16447 // Also known as an 'interprocedural jump'.
16448 // Target of jump will eventually return to caller.
16449 // TailJump below removes the return address.
16450 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16451 // emitted just above the TailCall which has reset rfp to the caller state.
16452 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16453 %{
16454   match(TailCall jump_target method_ptr);
16455 
16456   ins_cost(CALL_COST);
16457 
16458   format %{ "br $jump_target\t# $method_ptr holds method" %}
16459 
16460   ins_encode(aarch64_enc_tail_call(jump_target));
16461 
16462   ins_pipe(pipe_class_call);
16463 %}
16464 
16465 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16466 %{
16467   match(TailJump jump_target ex_oop);
16468 
16469   ins_cost(CALL_COST);
16470 
16471   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16472 
16473   ins_encode(aarch64_enc_tail_jmp(jump_target));
16474 
16475   ins_pipe(pipe_class_call);
16476 %}
16477 
16478 // Forward exception.
16479 instruct ForwardExceptionjmp()
16480 %{
16481   match(ForwardException);
16482   ins_cost(CALL_COST);
16483 
16484   format %{ "b forward_exception_stub" %}
16485   ins_encode %{
16486     __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16487   %}
16488   ins_pipe(pipe_class_call);
16489 %}
16490 
16491 // Create exception oop: created by stack-crawling runtime code.
16492 // Created exception is now available to this handler, and is setup
16493 // just prior to jumping to this handler. No code emitted.
16494 // TODO check
16495 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16496 instruct CreateException(iRegP_R0 ex_oop)
16497 %{
16498   match(Set ex_oop (CreateEx));
16499 
16500   format %{ " -- \t// exception oop; no code emitted" %}
16501 
16502   size(0);
16503 
16504   ins_encode( /*empty*/ );
16505 
16506   ins_pipe(pipe_class_empty);
16507 %}
16508 
16509 // Rethrow exception: The exception oop will come in the first
16510 // argument position. Then JUMP (not call) to the rethrow stub code.
16511 instruct RethrowException() %{
16512   match(Rethrow);
16513   ins_cost(CALL_COST);
16514 
16515   format %{ "b rethrow_stub" %}
16516 
16517   ins_encode( aarch64_enc_rethrow() );
16518 
16519   ins_pipe(pipe_class_call);
16520 %}
16521 
16522 
16523 // Return Instruction
16524 // epilog node loads ret address into lr as part of frame pop
16525 instruct Ret()
16526 %{
16527   match(Return);
16528 
16529   format %{ "ret\t// return register" %}
16530 
16531   ins_encode( aarch64_enc_ret() );
16532 
16533   ins_pipe(pipe_branch);
16534 %}
16535 
16536 // Die now.
16537 instruct ShouldNotReachHere() %{
16538   match(Halt);
16539 
16540   ins_cost(CALL_COST);
16541   format %{ "ShouldNotReachHere" %}
16542 
16543   ins_encode %{
16544     if (is_reachable()) {
16545       const char* str = __ code_string(_halt_reason);
16546       __ stop(str);
16547     }
16548   %}
16549 
16550   ins_pipe(pipe_class_default);
16551 %}
16552 
16553 // ============================================================================
16554 // Partial Subtype Check
16555 //
16556 // superklass array for an instance of the superklass.  Set a hidden
16557 // internal cache on a hit (cache is checked with exposed code in
16558 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16559 // encoding ALSO sets flags.
16560 
16561 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16562 %{
16563   match(Set result (PartialSubtypeCheck sub super));
16564   predicate(!UseSecondarySupersTable);
16565   effect(KILL cr, KILL temp);
16566 
16567   ins_cost(20 * INSN_COST);  // slightly larger than the next version
16568   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16569 
16570   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16571 
16572   opcode(0x1); // Force zero of result reg on hit
16573 
16574   ins_pipe(pipe_class_memory);
16575 %}
16576 
16577 // Two versions of partialSubtypeCheck, both used when we need to
16578 // search for a super class in the secondary supers array. The first
16579 // is used when we don't know _a priori_ the class being searched
16580 // for. The second, far more common, is used when we do know: this is
16581 // used for instanceof, checkcast, and any case where C2 can determine
16582 // it by constant propagation.
16583 
16584 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16585                                      iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16586                                      rFlagsReg cr)
16587 %{
16588   match(Set result (PartialSubtypeCheck sub super));
16589   predicate(UseSecondarySupersTable);
16590   effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16591 
16592   ins_cost(10 * INSN_COST);  // slightly larger than the next version
16593   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16594 
16595   ins_encode %{
16596     __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16597                                          $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16598                                          $vtemp$$FloatRegister,
16599                                          $result$$Register, /*L_success*/nullptr);
16600   %}
16601 
16602   ins_pipe(pipe_class_memory);
16603 %}
16604 
16605 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16606                                        iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16607                                        rFlagsReg cr)
16608 %{
16609   match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16610   predicate(UseSecondarySupersTable);
16611   effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16612 
16613   ins_cost(5 * INSN_COST);  // smaller than the next version
16614   format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16615 
16616   ins_encode %{
16617     bool success = false;
16618     u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16619     if (InlineSecondarySupersTest) {
16620       success =
16621         __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16622                                                $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16623                                                $vtemp$$FloatRegister,
16624                                                $result$$Register,
16625                                                super_klass_slot);
16626     } else {
16627       address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16628       success = (call != nullptr);
16629     }
16630     if (!success) {
16631       ciEnv::current()->record_failure("CodeCache is full");
16632       return;
16633     }
16634   %}
16635 
16636   ins_pipe(pipe_class_memory);
16637 %}
16638 
16639 // Intrisics for String.compareTo()
16640 
16641 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16642                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16643 %{
16644   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16645   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16646   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16647 
16648   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16649   ins_encode %{
16650     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16651     __ string_compare($str1$$Register, $str2$$Register,
16652                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16653                       $tmp1$$Register, $tmp2$$Register,
16654                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16655   %}
16656   ins_pipe(pipe_class_memory);
16657 %}
16658 
16659 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16660                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16661 %{
16662   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16663   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16664   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16665 
16666   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16667   ins_encode %{
16668     __ string_compare($str1$$Register, $str2$$Register,
16669                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16670                       $tmp1$$Register, $tmp2$$Register,
16671                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16672   %}
16673   ins_pipe(pipe_class_memory);
16674 %}
16675 
16676 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16677                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16678                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16679 %{
16680   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16681   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16682   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16683          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16684 
16685   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16686   ins_encode %{
16687     __ string_compare($str1$$Register, $str2$$Register,
16688                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16689                       $tmp1$$Register, $tmp2$$Register,
16690                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16691                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16692   %}
16693   ins_pipe(pipe_class_memory);
16694 %}
16695 
16696 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16697                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16698                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16699 %{
16700   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16701   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16702   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16703          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16704 
16705   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16706   ins_encode %{
16707     __ string_compare($str1$$Register, $str2$$Register,
16708                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16709                       $tmp1$$Register, $tmp2$$Register,
16710                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16711                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16712   %}
16713   ins_pipe(pipe_class_memory);
16714 %}
16715 
16716 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16717 // these string_compare variants as NEON register type for convenience so that the prototype of
16718 // string_compare can be shared with all variants.
16719 
16720 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16721                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16722                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16723                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16724 %{
16725   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16726   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16727   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16728          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16729 
16730   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16731   ins_encode %{
16732     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16733     __ string_compare($str1$$Register, $str2$$Register,
16734                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16735                       $tmp1$$Register, $tmp2$$Register,
16736                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16737                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16738                       StrIntrinsicNode::LL);
16739   %}
16740   ins_pipe(pipe_class_memory);
16741 %}
16742 
16743 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16744                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16745                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16746                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16747 %{
16748   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16749   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16750   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16751          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16752 
16753   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16754   ins_encode %{
16755     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16756     __ string_compare($str1$$Register, $str2$$Register,
16757                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16758                       $tmp1$$Register, $tmp2$$Register,
16759                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16760                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16761                       StrIntrinsicNode::LU);
16762   %}
16763   ins_pipe(pipe_class_memory);
16764 %}
16765 
16766 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16767                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16768                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16769                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16770 %{
16771   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16772   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16773   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16774          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16775 
16776   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16777   ins_encode %{
16778     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16779     __ string_compare($str1$$Register, $str2$$Register,
16780                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16781                       $tmp1$$Register, $tmp2$$Register,
16782                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16783                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16784                       StrIntrinsicNode::UL);
16785   %}
16786   ins_pipe(pipe_class_memory);
16787 %}
16788 
16789 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16790                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16791                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16792                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16793 %{
16794   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16795   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16796   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16797          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16798 
16799   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16800   ins_encode %{
16801     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16802     __ string_compare($str1$$Register, $str2$$Register,
16803                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16804                       $tmp1$$Register, $tmp2$$Register,
16805                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16806                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16807                       StrIntrinsicNode::UU);
16808   %}
16809   ins_pipe(pipe_class_memory);
16810 %}
16811 
16812 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16813                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16814                           iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16815                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16816 %{
16817   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16818   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16819   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16820          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16821          TEMP vtmp0, TEMP vtmp1, KILL cr);
16822   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16823             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16824 
16825   ins_encode %{
16826     __ string_indexof($str1$$Register, $str2$$Register,
16827                       $cnt1$$Register, $cnt2$$Register,
16828                       $tmp1$$Register, $tmp2$$Register,
16829                       $tmp3$$Register, $tmp4$$Register,
16830                       $tmp5$$Register, $tmp6$$Register,
16831                       -1, $result$$Register, StrIntrinsicNode::UU);
16832   %}
16833   ins_pipe(pipe_class_memory);
16834 %}
16835 
16836 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16837                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16838                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16839                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16840 %{
16841   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16842   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16843   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16844          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16845          TEMP vtmp0, TEMP vtmp1, KILL cr);
16846   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16847             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16848 
16849   ins_encode %{
16850     __ string_indexof($str1$$Register, $str2$$Register,
16851                       $cnt1$$Register, $cnt2$$Register,
16852                       $tmp1$$Register, $tmp2$$Register,
16853                       $tmp3$$Register, $tmp4$$Register,
16854                       $tmp5$$Register, $tmp6$$Register,
16855                       -1, $result$$Register, StrIntrinsicNode::LL);
16856   %}
16857   ins_pipe(pipe_class_memory);
16858 %}
16859 
16860 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16861                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16862                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16863                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16864 %{
16865   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16866   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16867   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16868          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16869          TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16870   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16871             "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16872 
16873   ins_encode %{
16874     __ string_indexof($str1$$Register, $str2$$Register,
16875                       $cnt1$$Register, $cnt2$$Register,
16876                       $tmp1$$Register, $tmp2$$Register,
16877                       $tmp3$$Register, $tmp4$$Register,
16878                       $tmp5$$Register, $tmp6$$Register,
16879                       -1, $result$$Register, StrIntrinsicNode::UL);
16880   %}
16881   ins_pipe(pipe_class_memory);
16882 %}
16883 
16884 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16885                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16886                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16887 %{
16888   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16889   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16890   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16891          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16892   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16893             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16894 
16895   ins_encode %{
16896     int icnt2 = (int)$int_cnt2$$constant;
16897     __ string_indexof($str1$$Register, $str2$$Register,
16898                       $cnt1$$Register, zr,
16899                       $tmp1$$Register, $tmp2$$Register,
16900                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16901                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16902   %}
16903   ins_pipe(pipe_class_memory);
16904 %}
16905 
16906 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16907                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16908                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16909 %{
16910   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16911   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16912   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16913          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16914   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16915             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16916 
16917   ins_encode %{
16918     int icnt2 = (int)$int_cnt2$$constant;
16919     __ string_indexof($str1$$Register, $str2$$Register,
16920                       $cnt1$$Register, zr,
16921                       $tmp1$$Register, $tmp2$$Register,
16922                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16923                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16924   %}
16925   ins_pipe(pipe_class_memory);
16926 %}
16927 
16928 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16929                               immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16930                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16931 %{
16932   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16933   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16934   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16935          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16936   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16937             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16938 
16939   ins_encode %{
16940     int icnt2 = (int)$int_cnt2$$constant;
16941     __ string_indexof($str1$$Register, $str2$$Register,
16942                       $cnt1$$Register, zr,
16943                       $tmp1$$Register, $tmp2$$Register,
16944                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16945                       icnt2, $result$$Register, StrIntrinsicNode::UL);
16946   %}
16947   ins_pipe(pipe_class_memory);
16948 %}
16949 
16950 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16951                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16952                              iRegINoSp tmp3, rFlagsReg cr)
16953 %{
16954   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16955   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16956   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16957          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16958 
16959   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16960 
16961   ins_encode %{
16962     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16963                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16964                            $tmp3$$Register);
16965   %}
16966   ins_pipe(pipe_class_memory);
16967 %}
16968 
16969 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16970                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16971                               iRegINoSp tmp3, rFlagsReg cr)
16972 %{
16973   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16974   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16975   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16976          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16977 
16978   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16979 
16980   ins_encode %{
16981     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16982                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
16983                             $tmp3$$Register);
16984   %}
16985   ins_pipe(pipe_class_memory);
16986 %}
16987 
16988 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16989                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16990                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16991   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16992   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16993   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16994   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16995   ins_encode %{
16996     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16997                                $result$$Register, $ztmp1$$FloatRegister,
16998                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16999                                $ptmp$$PRegister, true /* isL */);
17000   %}
17001   ins_pipe(pipe_class_memory);
17002 %}
17003 
17004 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17005                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17006                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17007   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17008   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17009   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17010   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17011   ins_encode %{
17012     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17013                                $result$$Register, $ztmp1$$FloatRegister,
17014                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17015                                $ptmp$$PRegister, false /* isL */);
17016   %}
17017   ins_pipe(pipe_class_memory);
17018 %}
17019 
17020 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17021                         iRegI_R0 result, rFlagsReg cr)
17022 %{
17023   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17024   match(Set result (StrEquals (Binary str1 str2) cnt));
17025   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17026 
17027   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17028   ins_encode %{
17029     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17030     __ string_equals($str1$$Register, $str2$$Register,
17031                      $result$$Register, $cnt$$Register);
17032   %}
17033   ins_pipe(pipe_class_memory);
17034 %}
17035 
17036 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17037                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17038                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17039                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17040                        iRegP_R10 tmp, rFlagsReg cr)
17041 %{
17042   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17043   match(Set result (AryEq ary1 ary2));
17044   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17045          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17046          TEMP vtmp6, TEMP vtmp7, KILL cr);
17047 
17048   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17049   ins_encode %{
17050     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17051                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17052                                    $result$$Register, $tmp$$Register, 1);
17053     if (tpc == nullptr) {
17054       ciEnv::current()->record_failure("CodeCache is full");
17055       return;
17056     }
17057   %}
17058   ins_pipe(pipe_class_memory);
17059 %}
17060 
17061 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17062                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17063                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17064                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17065                        iRegP_R10 tmp, rFlagsReg cr)
17066 %{
17067   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17068   match(Set result (AryEq ary1 ary2));
17069   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17070          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17071          TEMP vtmp6, TEMP vtmp7, KILL cr);
17072 
17073   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17074   ins_encode %{
17075     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17076                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17077                                    $result$$Register, $tmp$$Register, 2);
17078     if (tpc == nullptr) {
17079       ciEnv::current()->record_failure("CodeCache is full");
17080       return;
17081     }
17082   %}
17083   ins_pipe(pipe_class_memory);
17084 %}
17085 
17086 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
17087                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17088                          vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17089                          vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
17090 %{
17091   match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
17092   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
17093          TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
17094 
17095   format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result   // KILL all" %}
17096   ins_encode %{
17097     address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
17098                                      $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
17099                                      $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
17100                                      $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
17101                                      $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
17102                                      $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
17103                                      (BasicType)$basic_type$$constant);
17104     if (tpc == nullptr) {
17105       ciEnv::current()->record_failure("CodeCache is full");
17106       return;
17107     }
17108   %}
17109   ins_pipe(pipe_class_memory);
17110 %}
17111 
17112 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17113 %{
17114   match(Set result (CountPositives ary1 len));
17115   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17116   format %{ "count positives byte[] $ary1,$len -> $result" %}
17117   ins_encode %{
17118     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17119     if (tpc == nullptr) {
17120       ciEnv::current()->record_failure("CodeCache is full");
17121       return;
17122     }
17123   %}
17124   ins_pipe( pipe_slow );
17125 %}
17126 
17127 // fast char[] to byte[] compression
17128 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17129                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17130                          vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17131                          iRegI_R0 result, rFlagsReg cr)
17132 %{
17133   match(Set result (StrCompressedCopy src (Binary dst len)));
17134   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17135          USE_KILL src, USE_KILL dst, USE len, KILL cr);
17136 
17137   format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17138   ins_encode %{
17139     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17140                            $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17141                            $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17142                            $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17143   %}
17144   ins_pipe(pipe_slow);
17145 %}
17146 
17147 // fast byte[] to char[] inflation
17148 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17149                         vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17150                         vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17151 %{
17152   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17153   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17154          TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17155          USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17156 
17157   format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17158   ins_encode %{
17159     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17160                                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17161                                         $vtmp2$$FloatRegister, $tmp$$Register);
17162     if (tpc == nullptr) {
17163       ciEnv::current()->record_failure("CodeCache is full");
17164       return;
17165     }
17166   %}
17167   ins_pipe(pipe_class_memory);
17168 %}
17169 
17170 // encode char[] to byte[] in ISO_8859_1
17171 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17172                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17173                           vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17174                           iRegI_R0 result, rFlagsReg cr)
17175 %{
17176   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17177   match(Set result (EncodeISOArray src (Binary dst len)));
17178   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17179          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17180 
17181   format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17182   ins_encode %{
17183     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17184                         $result$$Register, false,
17185                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17186                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17187                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17188   %}
17189   ins_pipe(pipe_class_memory);
17190 %}
17191 
17192 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17193                             vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17194                             vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17195                             iRegI_R0 result, rFlagsReg cr)
17196 %{
17197   predicate(((EncodeISOArrayNode*)n)->is_ascii());
17198   match(Set result (EncodeISOArray src (Binary dst len)));
17199   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17200          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17201 
17202   format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17203   ins_encode %{
17204     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17205                         $result$$Register, true,
17206                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17207                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17208                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17209   %}
17210   ins_pipe(pipe_class_memory);
17211 %}
17212 
17213 //----------------------------- CompressBits/ExpandBits ------------------------
17214 
17215 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17216                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17217   match(Set dst (CompressBits src mask));
17218   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17219   format %{ "mov    $tsrc, $src\n\t"
17220             "mov    $tmask, $mask\n\t"
17221             "bext   $tdst, $tsrc, $tmask\n\t"
17222             "mov    $dst, $tdst"
17223           %}
17224   ins_encode %{
17225     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17226     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17227     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17228     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17229   %}
17230   ins_pipe(pipe_slow);
17231 %}
17232 
17233 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17234                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17235   match(Set dst (CompressBits (LoadI mem) mask));
17236   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17237   format %{ "ldrs   $tsrc, $mem\n\t"
17238             "ldrs   $tmask, $mask\n\t"
17239             "bext   $tdst, $tsrc, $tmask\n\t"
17240             "mov    $dst, $tdst"
17241           %}
17242   ins_encode %{
17243     loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17244               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17245     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17246     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17247     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17248   %}
17249   ins_pipe(pipe_slow);
17250 %}
17251 
17252 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17253                            vRegD tdst, vRegD tsrc, vRegD tmask) %{
17254   match(Set dst (CompressBits src mask));
17255   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17256   format %{ "mov    $tsrc, $src\n\t"
17257             "mov    $tmask, $mask\n\t"
17258             "bext   $tdst, $tsrc, $tmask\n\t"
17259             "mov    $dst, $tdst"
17260           %}
17261   ins_encode %{
17262     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17263     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17264     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17265     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17266   %}
17267   ins_pipe(pipe_slow);
17268 %}
17269 
17270 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17271                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17272   match(Set dst (CompressBits (LoadL mem) mask));
17273   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17274   format %{ "ldrd   $tsrc, $mem\n\t"
17275             "ldrd   $tmask, $mask\n\t"
17276             "bext   $tdst, $tsrc, $tmask\n\t"
17277             "mov    $dst, $tdst"
17278           %}
17279   ins_encode %{
17280     loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17281               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17282     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17283     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17284     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17285   %}
17286   ins_pipe(pipe_slow);
17287 %}
17288 
17289 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17290                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17291   match(Set dst (ExpandBits src mask));
17292   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17293   format %{ "mov    $tsrc, $src\n\t"
17294             "mov    $tmask, $mask\n\t"
17295             "bdep   $tdst, $tsrc, $tmask\n\t"
17296             "mov    $dst, $tdst"
17297           %}
17298   ins_encode %{
17299     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17300     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17301     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17302     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17303   %}
17304   ins_pipe(pipe_slow);
17305 %}
17306 
17307 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17308                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17309   match(Set dst (ExpandBits (LoadI mem) mask));
17310   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17311   format %{ "ldrs   $tsrc, $mem\n\t"
17312             "ldrs   $tmask, $mask\n\t"
17313             "bdep   $tdst, $tsrc, $tmask\n\t"
17314             "mov    $dst, $tdst"
17315           %}
17316   ins_encode %{
17317     loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17318               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17319     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17320     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17321     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17322   %}
17323   ins_pipe(pipe_slow);
17324 %}
17325 
17326 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17327                          vRegD tdst, vRegD tsrc, vRegD tmask) %{
17328   match(Set dst (ExpandBits src mask));
17329   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17330   format %{ "mov    $tsrc, $src\n\t"
17331             "mov    $tmask, $mask\n\t"
17332             "bdep   $tdst, $tsrc, $tmask\n\t"
17333             "mov    $dst, $tdst"
17334           %}
17335   ins_encode %{
17336     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17337     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17338     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17339     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17340   %}
17341   ins_pipe(pipe_slow);
17342 %}
17343 
17344 
17345 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17346                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17347   match(Set dst (ExpandBits (LoadL mem) mask));
17348   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17349   format %{ "ldrd   $tsrc, $mem\n\t"
17350             "ldrd   $tmask, $mask\n\t"
17351             "bdep   $tdst, $tsrc, $tmask\n\t"
17352             "mov    $dst, $tdst"
17353           %}
17354   ins_encode %{
17355     loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17356               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17357     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17358     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17359     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17360   %}
17361   ins_pipe(pipe_slow);
17362 %}
17363 
17364 //----------------------------- Reinterpret ----------------------------------
17365 // Reinterpret a half-precision float value in a floating point register to a general purpose register
17366 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
17367   match(Set dst (ReinterpretHF2S src));
17368   format %{ "reinterpretHF2S $dst, $src" %}
17369   ins_encode %{
17370     __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
17371   %}
17372   ins_pipe(pipe_slow);
17373 %}
17374 
17375 // Reinterpret a half-precision float value in a general purpose register to a floating point register
17376 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
17377   match(Set dst (ReinterpretS2HF src));
17378   format %{ "reinterpretS2HF $dst, $src" %}
17379   ins_encode %{
17380     __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
17381   %}
17382   ins_pipe(pipe_slow);
17383 %}
17384 
17385 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
17386 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
17387 // fcvt $tmp1_fpr, $src_fpr    // Convert float to half-precision float
17388 // mov  $tmp2_gpr, $tmp1_fpr   // Move half-precision float in FPR to a GPR
17389 // mov  $dst_fpr,  $tmp2_gpr   // Move the result from a GPR to an FPR
17390 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
17391 // can be omitted in this pattern, resulting in -
17392 // fcvt $dst, $src  // Convert float to half-precision float
17393 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
17394 %{
17395   match(Set dst (ReinterpretS2HF (ConvF2HF src)));
17396   format %{ "convF2HFAndS2HF $dst, $src" %}
17397   ins_encode %{
17398     __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
17399   %}
17400   ins_pipe(pipe_slow);
17401 %}
17402 
17403 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
17404 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
17405 // mov  $tmp1_gpr, $src_fpr  // Move the half-precision float from an FPR to a GPR
17406 // mov  $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
17407 // fcvt $dst_fpr,  $tmp2_fpr // Convert the half-precision float to 32-bit float
17408 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
17409 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
17410 // resulting in -
17411 // fcvt $dst, $src  // Convert half-precision float to a 32-bit float
17412 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
17413 %{
17414   match(Set dst (ConvHF2F (ReinterpretHF2S src)));
17415   format %{ "convHF2SAndHF2F $dst, $src" %}
17416   ins_encode %{
17417     __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
17418   %}
17419   ins_pipe(pipe_slow);
17420 %}
17421 
17422 // ============================================================================
17423 // This name is KNOWN by the ADLC and cannot be changed.
17424 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17425 // for this guy.
17426 instruct tlsLoadP(thread_RegP dst)
17427 %{
17428   match(Set dst (ThreadLocal));
17429 
17430   ins_cost(0);
17431 
17432   format %{ " -- \t// $dst=Thread::current(), empty" %}
17433 
17434   size(0);
17435 
17436   ins_encode( /*empty*/ );
17437 
17438   ins_pipe(pipe_class_empty);
17439 %}
17440 
17441 //----------PEEPHOLE RULES-----------------------------------------------------
17442 // These must follow all instruction definitions as they use the names
17443 // defined in the instructions definitions.
17444 //
17445 // peepmatch ( root_instr_name [preceding_instruction]* );
17446 //
17447 // peepconstraint %{
17448 // (instruction_number.operand_name relational_op instruction_number.operand_name
17449 //  [, ...] );
17450 // // instruction numbers are zero-based using left to right order in peepmatch
17451 //
17452 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17453 // // provide an instruction_number.operand_name for each operand that appears
17454 // // in the replacement instruction's match rule
17455 //
17456 // ---------VM FLAGS---------------------------------------------------------
17457 //
17458 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17459 //
17460 // Each peephole rule is given an identifying number starting with zero and
17461 // increasing by one in the order seen by the parser.  An individual peephole
17462 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17463 // on the command-line.
17464 //
17465 // ---------CURRENT LIMITATIONS----------------------------------------------
17466 //
17467 // Only match adjacent instructions in same basic block
17468 // Only equality constraints
17469 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17470 // Only one replacement instruction
17471 //
17472 // ---------EXAMPLE----------------------------------------------------------
17473 //
17474 // // pertinent parts of existing instructions in architecture description
17475 // instruct movI(iRegINoSp dst, iRegI src)
17476 // %{
17477 //   match(Set dst (CopyI src));
17478 // %}
17479 //
17480 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17481 // %{
17482 //   match(Set dst (AddI dst src));
17483 //   effect(KILL cr);
17484 // %}
17485 //
17486 // // Change (inc mov) to lea
17487 // peephole %{
17488 //   // increment preceded by register-register move
17489 //   peepmatch ( incI_iReg movI );
17490 //   // require that the destination register of the increment
17491 //   // match the destination register of the move
17492 //   peepconstraint ( 0.dst == 1.dst );
17493 //   // construct a replacement instruction that sets
17494 //   // the destination to ( move's source register + one )
17495 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17496 // %}
17497 //
17498 
17499 // Implementation no longer uses movX instructions since
17500 // machine-independent system no longer uses CopyX nodes.
17501 //
17502 // peephole
17503 // %{
17504 //   peepmatch (incI_iReg movI);
17505 //   peepconstraint (0.dst == 1.dst);
17506 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17507 // %}
17508 
17509 // peephole
17510 // %{
17511 //   peepmatch (decI_iReg movI);
17512 //   peepconstraint (0.dst == 1.dst);
17513 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17514 // %}
17515 
17516 // peephole
17517 // %{
17518 //   peepmatch (addI_iReg_imm movI);
17519 //   peepconstraint (0.dst == 1.dst);
17520 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17521 // %}
17522 
17523 // peephole
17524 // %{
17525 //   peepmatch (incL_iReg movL);
17526 //   peepconstraint (0.dst == 1.dst);
17527 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17528 // %}
17529 
17530 // peephole
17531 // %{
17532 //   peepmatch (decL_iReg movL);
17533 //   peepconstraint (0.dst == 1.dst);
17534 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17535 // %}
17536 
17537 // peephole
17538 // %{
17539 //   peepmatch (addL_iReg_imm movL);
17540 //   peepconstraint (0.dst == 1.dst);
17541 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17542 // %}
17543 
17544 // peephole
17545 // %{
17546 //   peepmatch (addP_iReg_imm movP);
17547 //   peepconstraint (0.dst == 1.dst);
17548 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17549 // %}
17550 
17551 // // Change load of spilled value to only a spill
17552 // instruct storeI(memory mem, iRegI src)
17553 // %{
17554 //   match(Set mem (StoreI mem src));
17555 // %}
17556 //
17557 // instruct loadI(iRegINoSp dst, memory mem)
17558 // %{
17559 //   match(Set dst (LoadI mem));
17560 // %}
17561 //
17562 
17563 //----------SMARTSPILL RULES---------------------------------------------------
17564 // These must follow all instruction definitions as they use the names
17565 // defined in the instructions definitions.
17566 
17567 // Local Variables:
17568 // mode: c++
17569 // End: