1 //
    2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // architecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182 
  183   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  184   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  185   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  186   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  187 
  188   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  189   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  190   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  191   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  192 
  193   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  194   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  195   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  196   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  197 
  198   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  199   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  200   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  201   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  202 
  203   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  204   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  205   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  206   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  207 
  208   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  209   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  210   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  211   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  212 
  213   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  214   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  215   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  216   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  217 
  218   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  219   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  220   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  221   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  222 
  223   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  224   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  225   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  226   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  227 
  228   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  229   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  230   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  231   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  232 
  233   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  234   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  235   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  236   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  237 
  238   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  239   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  240   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  241   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  242 
  243   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  244   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  245   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  246   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  247 
  248   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  249   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  250   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  251   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  252 
  253   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  254   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  255   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  256   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  257 
  258   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  259   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  260   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  261   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  262 
  263   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  264   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  265   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  266   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  267 
  268   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  269   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  270   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  271   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  272 
  273   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  274   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  275   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  276   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  277 
  278   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  279   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  280   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  281   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  282 
  283   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  284   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  285   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  286   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  287 
  288   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  289   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  290   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  291   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  292 
  293   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  294   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  295   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  296   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  297 
  298   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  299   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  300   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  301   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  302 
  303   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  304   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  305   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  306   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  307 
  308   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  309   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  310   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  311   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  312 
  313   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  314   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  315   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  316   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  317 
  318   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  319   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  320   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  321   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  322 
  323   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  324   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  325   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  326   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  327 
  328   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  329   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  330   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  331   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  332 
  333   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  334   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  335   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  336   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  337 
  338 // ----------------------------
  339 // SVE Predicate Registers
  340 // ----------------------------
  341   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  342   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  343   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  344   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  345   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  346   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  347   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  348   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  349   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  350   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  351   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  352   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  353   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  354   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  355   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  356   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  357 
  358 // ----------------------------
  359 // Special Registers
  360 // ----------------------------
  361 
  362 // the AArch64 CSPR status flag register is not directly accessible as
  363 // instruction operand. the FPSR status flag register is a system
  364 // register which can be written/read using MSR/MRS but again does not
  365 // appear as an operand (a code identifying the FSPR occurs as an
  366 // immediate value in the instruction).
  367 
  368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  369 
  370 // Specify priority of register selection within phases of register
  371 // allocation.  Highest priority is first.  A useful heuristic is to
  372 // give registers a low priority when they are required by machine
  373 // instructions, like EAX and EDX on I486, and choose no-save registers
  374 // before save-on-call, & save-on-call before save-on-entry.  Registers
  375 // which participate in fixed calling sequences should come last.
  376 // Registers which are used as pairs must fall on an even boundary.
  377 
  378 alloc_class chunk0(
  379     // volatiles
  380     R10, R10_H,
  381     R11, R11_H,
  382     R12, R12_H,
  383     R13, R13_H,
  384     R14, R14_H,
  385     R15, R15_H,
  386     R16, R16_H,
  387     R17, R17_H,
  388     R18, R18_H,
  389 
  390     // arg registers
  391     R0, R0_H,
  392     R1, R1_H,
  393     R2, R2_H,
  394     R3, R3_H,
  395     R4, R4_H,
  396     R5, R5_H,
  397     R6, R6_H,
  398     R7, R7_H,
  399 
  400     // non-volatiles
  401     R19, R19_H,
  402     R20, R20_H,
  403     R21, R21_H,
  404     R22, R22_H,
  405     R23, R23_H,
  406     R24, R24_H,
  407     R25, R25_H,
  408     R26, R26_H,
  409 
  410     // non-allocatable registers
  411 
  412     R27, R27_H, // heapbase
  413     R28, R28_H, // thread
  414     R29, R29_H, // fp
  415     R30, R30_H, // lr
  416     R31, R31_H, // sp
  417     R8, R8_H,   // rscratch1
  418     R9, R9_H,   // rscratch2
  419 );
  420 
  421 alloc_class chunk1(
  422 
  423     // no save
  424     V16, V16_H, V16_J, V16_K,
  425     V17, V17_H, V17_J, V17_K,
  426     V18, V18_H, V18_J, V18_K,
  427     V19, V19_H, V19_J, V19_K,
  428     V20, V20_H, V20_J, V20_K,
  429     V21, V21_H, V21_J, V21_K,
  430     V22, V22_H, V22_J, V22_K,
  431     V23, V23_H, V23_J, V23_K,
  432     V24, V24_H, V24_J, V24_K,
  433     V25, V25_H, V25_J, V25_K,
  434     V26, V26_H, V26_J, V26_K,
  435     V27, V27_H, V27_J, V27_K,
  436     V28, V28_H, V28_J, V28_K,
  437     V29, V29_H, V29_J, V29_K,
  438     V30, V30_H, V30_J, V30_K,
  439     V31, V31_H, V31_J, V31_K,
  440 
  441     // arg registers
  442     V0, V0_H, V0_J, V0_K,
  443     V1, V1_H, V1_J, V1_K,
  444     V2, V2_H, V2_J, V2_K,
  445     V3, V3_H, V3_J, V3_K,
  446     V4, V4_H, V4_J, V4_K,
  447     V5, V5_H, V5_J, V5_K,
  448     V6, V6_H, V6_J, V6_K,
  449     V7, V7_H, V7_J, V7_K,
  450 
  451     // non-volatiles
  452     V8, V8_H, V8_J, V8_K,
  453     V9, V9_H, V9_J, V9_K,
  454     V10, V10_H, V10_J, V10_K,
  455     V11, V11_H, V11_J, V11_K,
  456     V12, V12_H, V12_J, V12_K,
  457     V13, V13_H, V13_J, V13_K,
  458     V14, V14_H, V14_J, V14_K,
  459     V15, V15_H, V15_J, V15_K,
  460 );
  461 
  462 alloc_class chunk2 (
  463     // Governing predicates for load/store and arithmetic
  464     P0,
  465     P1,
  466     P2,
  467     P3,
  468     P4,
  469     P5,
  470     P6,
  471 
  472     // Extra predicates
  473     P8,
  474     P9,
  475     P10,
  476     P11,
  477     P12,
  478     P13,
  479     P14,
  480     P15,
  481 
  482     // Preserved for all-true predicate
  483     P7,
  484 );
  485 
  486 alloc_class chunk3(RFLAGS);
  487 
  488 //----------Architecture Description Register Classes--------------------------
  489 // Several register classes are automatically defined based upon information in
  490 // this architecture description.
  491 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  493 //
  494 
  495 // Class for all 32 bit general purpose registers
  496 reg_class all_reg32(
  497     R0,
  498     R1,
  499     R2,
  500     R3,
  501     R4,
  502     R5,
  503     R6,
  504     R7,
  505     R10,
  506     R11,
  507     R12,
  508     R13,
  509     R14,
  510     R15,
  511     R16,
  512     R17,
  513     R18,
  514     R19,
  515     R20,
  516     R21,
  517     R22,
  518     R23,
  519     R24,
  520     R25,
  521     R26,
  522     R27,
  523     R28,
  524     R29,
  525     R30,
  526     R31
  527 );
  528 
  529 
  530 // Class for all 32 bit integer registers (excluding SP which
  531 // will never be used as an integer register)
  532 reg_class any_reg32 %{
  533   return _ANY_REG32_mask;
  534 %}
  535 
  536 // Singleton class for R0 int register
  537 reg_class int_r0_reg(R0);
  538 
  539 // Singleton class for R2 int register
  540 reg_class int_r2_reg(R2);
  541 
  542 // Singleton class for R3 int register
  543 reg_class int_r3_reg(R3);
  544 
  545 // Singleton class for R4 int register
  546 reg_class int_r4_reg(R4);
  547 
  548 // Singleton class for R31 int register
  549 reg_class int_r31_reg(R31);
  550 
  551 // Class for all 64 bit general purpose registers
  552 reg_class all_reg(
  553     R0, R0_H,
  554     R1, R1_H,
  555     R2, R2_H,
  556     R3, R3_H,
  557     R4, R4_H,
  558     R5, R5_H,
  559     R6, R6_H,
  560     R7, R7_H,
  561     R10, R10_H,
  562     R11, R11_H,
  563     R12, R12_H,
  564     R13, R13_H,
  565     R14, R14_H,
  566     R15, R15_H,
  567     R16, R16_H,
  568     R17, R17_H,
  569     R18, R18_H,
  570     R19, R19_H,
  571     R20, R20_H,
  572     R21, R21_H,
  573     R22, R22_H,
  574     R23, R23_H,
  575     R24, R24_H,
  576     R25, R25_H,
  577     R26, R26_H,
  578     R27, R27_H,
  579     R28, R28_H,
  580     R29, R29_H,
  581     R30, R30_H,
  582     R31, R31_H
  583 );
  584 
  585 // Class for all long integer registers (including SP)
  586 reg_class any_reg %{
  587   return _ANY_REG_mask;
  588 %}
  589 
  590 // Class for non-allocatable 32 bit registers
  591 reg_class non_allocatable_reg32(
  592 #ifdef R18_RESERVED
  593     // See comment in register_aarch64.hpp
  594     R18,                        // tls on Windows
  595 #endif
  596     R28,                        // thread
  597     R30,                        // lr
  598     R31                         // sp
  599 );
  600 
  601 // Class for non-allocatable 64 bit registers
  602 reg_class non_allocatable_reg(
  603 #ifdef R18_RESERVED
  604     // See comment in register_aarch64.hpp
  605     R18, R18_H,                 // tls on Windows, platform register on macOS
  606 #endif
  607     R28, R28_H,                 // thread
  608     R30, R30_H,                 // lr
  609     R31, R31_H                  // sp
  610 );
  611 
  612 // Class for all non-special integer registers
  613 reg_class no_special_reg32 %{
  614   return _NO_SPECIAL_REG32_mask;
  615 %}
  616 
  617 // Class for all non-special long integer registers
  618 reg_class no_special_reg %{
  619   return _NO_SPECIAL_REG_mask;
  620 %}
  621 
  622 // Class for 64 bit register r0
  623 reg_class r0_reg(
  624     R0, R0_H
  625 );
  626 
  627 // Class for 64 bit register r1
  628 reg_class r1_reg(
  629     R1, R1_H
  630 );
  631 
  632 // Class for 64 bit register r2
  633 reg_class r2_reg(
  634     R2, R2_H
  635 );
  636 
  637 // Class for 64 bit register r3
  638 reg_class r3_reg(
  639     R3, R3_H
  640 );
  641 
  642 // Class for 64 bit register r4
  643 reg_class r4_reg(
  644     R4, R4_H
  645 );
  646 
  647 // Class for 64 bit register r5
  648 reg_class r5_reg(
  649     R5, R5_H
  650 );
  651 
  652 // Class for 64 bit register r10
  653 reg_class r10_reg(
  654     R10, R10_H
  655 );
  656 
  657 // Class for 64 bit register r11
  658 reg_class r11_reg(
  659     R11, R11_H
  660 );
  661 
  662 // Class for method register
  663 reg_class method_reg(
  664     R12, R12_H
  665 );
  666 
  667 // Class for thread register
  668 reg_class thread_reg(
  669     R28, R28_H
  670 );
  671 
  672 // Class for frame pointer register
  673 reg_class fp_reg(
  674     R29, R29_H
  675 );
  676 
  677 // Class for link register
  678 reg_class lr_reg(
  679     R30, R30_H
  680 );
  681 
  682 // Class for long sp register
  683 reg_class sp_reg(
  684   R31, R31_H
  685 );
  686 
  687 // Class for all pointer registers
  688 reg_class ptr_reg %{
  689   return _PTR_REG_mask;
  690 %}
  691 
  692 // Class for all non_special pointer registers
  693 reg_class no_special_ptr_reg %{
  694   return _NO_SPECIAL_PTR_REG_mask;
  695 %}
  696 
  697 // Class for all non_special pointer registers (excluding rfp)
  698 reg_class no_special_no_rfp_ptr_reg %{
  699   return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
  700 %}
  701 
  702 // Class for all float registers
  703 reg_class float_reg(
  704     V0,
  705     V1,
  706     V2,
  707     V3,
  708     V4,
  709     V5,
  710     V6,
  711     V7,
  712     V8,
  713     V9,
  714     V10,
  715     V11,
  716     V12,
  717     V13,
  718     V14,
  719     V15,
  720     V16,
  721     V17,
  722     V18,
  723     V19,
  724     V20,
  725     V21,
  726     V22,
  727     V23,
  728     V24,
  729     V25,
  730     V26,
  731     V27,
  732     V28,
  733     V29,
  734     V30,
  735     V31
  736 );
  737 
  738 // Double precision float registers have virtual `high halves' that
  739 // are needed by the allocator.
  740 // Class for all double registers
  741 reg_class double_reg(
  742     V0, V0_H,
  743     V1, V1_H,
  744     V2, V2_H,
  745     V3, V3_H,
  746     V4, V4_H,
  747     V5, V5_H,
  748     V6, V6_H,
  749     V7, V7_H,
  750     V8, V8_H,
  751     V9, V9_H,
  752     V10, V10_H,
  753     V11, V11_H,
  754     V12, V12_H,
  755     V13, V13_H,
  756     V14, V14_H,
  757     V15, V15_H,
  758     V16, V16_H,
  759     V17, V17_H,
  760     V18, V18_H,
  761     V19, V19_H,
  762     V20, V20_H,
  763     V21, V21_H,
  764     V22, V22_H,
  765     V23, V23_H,
  766     V24, V24_H,
  767     V25, V25_H,
  768     V26, V26_H,
  769     V27, V27_H,
  770     V28, V28_H,
  771     V29, V29_H,
  772     V30, V30_H,
  773     V31, V31_H
  774 );
  775 
  776 // Class for all SVE vector registers.
  777 reg_class vectora_reg (
  778     V0, V0_H, V0_J, V0_K,
  779     V1, V1_H, V1_J, V1_K,
  780     V2, V2_H, V2_J, V2_K,
  781     V3, V3_H, V3_J, V3_K,
  782     V4, V4_H, V4_J, V4_K,
  783     V5, V5_H, V5_J, V5_K,
  784     V6, V6_H, V6_J, V6_K,
  785     V7, V7_H, V7_J, V7_K,
  786     V8, V8_H, V8_J, V8_K,
  787     V9, V9_H, V9_J, V9_K,
  788     V10, V10_H, V10_J, V10_K,
  789     V11, V11_H, V11_J, V11_K,
  790     V12, V12_H, V12_J, V12_K,
  791     V13, V13_H, V13_J, V13_K,
  792     V14, V14_H, V14_J, V14_K,
  793     V15, V15_H, V15_J, V15_K,
  794     V16, V16_H, V16_J, V16_K,
  795     V17, V17_H, V17_J, V17_K,
  796     V18, V18_H, V18_J, V18_K,
  797     V19, V19_H, V19_J, V19_K,
  798     V20, V20_H, V20_J, V20_K,
  799     V21, V21_H, V21_J, V21_K,
  800     V22, V22_H, V22_J, V22_K,
  801     V23, V23_H, V23_J, V23_K,
  802     V24, V24_H, V24_J, V24_K,
  803     V25, V25_H, V25_J, V25_K,
  804     V26, V26_H, V26_J, V26_K,
  805     V27, V27_H, V27_J, V27_K,
  806     V28, V28_H, V28_J, V28_K,
  807     V29, V29_H, V29_J, V29_K,
  808     V30, V30_H, V30_J, V30_K,
  809     V31, V31_H, V31_J, V31_K,
  810 );
  811 
  812 // Class for all 64bit vector registers
  813 reg_class vectord_reg(
  814     V0, V0_H,
  815     V1, V1_H,
  816     V2, V2_H,
  817     V3, V3_H,
  818     V4, V4_H,
  819     V5, V5_H,
  820     V6, V6_H,
  821     V7, V7_H,
  822     V8, V8_H,
  823     V9, V9_H,
  824     V10, V10_H,
  825     V11, V11_H,
  826     V12, V12_H,
  827     V13, V13_H,
  828     V14, V14_H,
  829     V15, V15_H,
  830     V16, V16_H,
  831     V17, V17_H,
  832     V18, V18_H,
  833     V19, V19_H,
  834     V20, V20_H,
  835     V21, V21_H,
  836     V22, V22_H,
  837     V23, V23_H,
  838     V24, V24_H,
  839     V25, V25_H,
  840     V26, V26_H,
  841     V27, V27_H,
  842     V28, V28_H,
  843     V29, V29_H,
  844     V30, V30_H,
  845     V31, V31_H
  846 );
  847 
  848 // Class for all 128bit vector registers
  849 reg_class vectorx_reg(
  850     V0, V0_H, V0_J, V0_K,
  851     V1, V1_H, V1_J, V1_K,
  852     V2, V2_H, V2_J, V2_K,
  853     V3, V3_H, V3_J, V3_K,
  854     V4, V4_H, V4_J, V4_K,
  855     V5, V5_H, V5_J, V5_K,
  856     V6, V6_H, V6_J, V6_K,
  857     V7, V7_H, V7_J, V7_K,
  858     V8, V8_H, V8_J, V8_K,
  859     V9, V9_H, V9_J, V9_K,
  860     V10, V10_H, V10_J, V10_K,
  861     V11, V11_H, V11_J, V11_K,
  862     V12, V12_H, V12_J, V12_K,
  863     V13, V13_H, V13_J, V13_K,
  864     V14, V14_H, V14_J, V14_K,
  865     V15, V15_H, V15_J, V15_K,
  866     V16, V16_H, V16_J, V16_K,
  867     V17, V17_H, V17_J, V17_K,
  868     V18, V18_H, V18_J, V18_K,
  869     V19, V19_H, V19_J, V19_K,
  870     V20, V20_H, V20_J, V20_K,
  871     V21, V21_H, V21_J, V21_K,
  872     V22, V22_H, V22_J, V22_K,
  873     V23, V23_H, V23_J, V23_K,
  874     V24, V24_H, V24_J, V24_K,
  875     V25, V25_H, V25_J, V25_K,
  876     V26, V26_H, V26_J, V26_K,
  877     V27, V27_H, V27_J, V27_K,
  878     V28, V28_H, V28_J, V28_K,
  879     V29, V29_H, V29_J, V29_K,
  880     V30, V30_H, V30_J, V30_K,
  881     V31, V31_H, V31_J, V31_K
  882 );
  883 
  884 // Class for vector register V10
  885 reg_class v10_veca_reg(
  886     V10, V10_H, V10_J, V10_K
  887 );
  888 
  889 // Class for vector register V11
  890 reg_class v11_veca_reg(
  891     V11, V11_H, V11_J, V11_K
  892 );
  893 
  894 // Class for vector register V12
  895 reg_class v12_veca_reg(
  896     V12, V12_H, V12_J, V12_K
  897 );
  898 
  899 // Class for vector register V13
  900 reg_class v13_veca_reg(
  901     V13, V13_H, V13_J, V13_K
  902 );
  903 
  904 // Class for vector register V17
  905 reg_class v17_veca_reg(
  906     V17, V17_H, V17_J, V17_K
  907 );
  908 
  909 // Class for vector register V18
  910 reg_class v18_veca_reg(
  911     V18, V18_H, V18_J, V18_K
  912 );
  913 
  914 // Class for vector register V23
  915 reg_class v23_veca_reg(
  916     V23, V23_H, V23_J, V23_K
  917 );
  918 
  919 // Class for vector register V24
  920 reg_class v24_veca_reg(
  921     V24, V24_H, V24_J, V24_K
  922 );
  923 
  924 // Class for 128 bit register v0
  925 reg_class v0_reg(
  926     V0, V0_H
  927 );
  928 
  929 // Class for 128 bit register v1
  930 reg_class v1_reg(
  931     V1, V1_H
  932 );
  933 
  934 // Class for 128 bit register v2
  935 reg_class v2_reg(
  936     V2, V2_H
  937 );
  938 
  939 // Class for 128 bit register v3
  940 reg_class v3_reg(
  941     V3, V3_H
  942 );
  943 
  944 // Class for 128 bit register v4
  945 reg_class v4_reg(
  946     V4, V4_H
  947 );
  948 
  949 // Class for 128 bit register v5
  950 reg_class v5_reg(
  951     V5, V5_H
  952 );
  953 
  954 // Class for 128 bit register v6
  955 reg_class v6_reg(
  956     V6, V6_H
  957 );
  958 
  959 // Class for 128 bit register v7
  960 reg_class v7_reg(
  961     V7, V7_H
  962 );
  963 
  964 // Class for 128 bit register v8
  965 reg_class v8_reg(
  966     V8, V8_H
  967 );
  968 
  969 // Class for 128 bit register v9
  970 reg_class v9_reg(
  971     V9, V9_H
  972 );
  973 
  974 // Class for 128 bit register v10
  975 reg_class v10_reg(
  976     V10, V10_H
  977 );
  978 
  979 // Class for 128 bit register v11
  980 reg_class v11_reg(
  981     V11, V11_H
  982 );
  983 
  984 // Class for 128 bit register v12
  985 reg_class v12_reg(
  986     V12, V12_H
  987 );
  988 
  989 // Class for 128 bit register v13
  990 reg_class v13_reg(
  991     V13, V13_H
  992 );
  993 
  994 // Class for 128 bit register v14
  995 reg_class v14_reg(
  996     V14, V14_H
  997 );
  998 
  999 // Class for 128 bit register v15
 1000 reg_class v15_reg(
 1001     V15, V15_H
 1002 );
 1003 
 1004 // Class for 128 bit register v16
 1005 reg_class v16_reg(
 1006     V16, V16_H
 1007 );
 1008 
 1009 // Class for 128 bit register v17
 1010 reg_class v17_reg(
 1011     V17, V17_H
 1012 );
 1013 
 1014 // Class for 128 bit register v18
 1015 reg_class v18_reg(
 1016     V18, V18_H
 1017 );
 1018 
 1019 // Class for 128 bit register v19
 1020 reg_class v19_reg(
 1021     V19, V19_H
 1022 );
 1023 
 1024 // Class for 128 bit register v20
 1025 reg_class v20_reg(
 1026     V20, V20_H
 1027 );
 1028 
 1029 // Class for 128 bit register v21
 1030 reg_class v21_reg(
 1031     V21, V21_H
 1032 );
 1033 
 1034 // Class for 128 bit register v22
 1035 reg_class v22_reg(
 1036     V22, V22_H
 1037 );
 1038 
 1039 // Class for 128 bit register v23
 1040 reg_class v23_reg(
 1041     V23, V23_H
 1042 );
 1043 
 1044 // Class for 128 bit register v24
 1045 reg_class v24_reg(
 1046     V24, V24_H
 1047 );
 1048 
 1049 // Class for 128 bit register v25
 1050 reg_class v25_reg(
 1051     V25, V25_H
 1052 );
 1053 
 1054 // Class for 128 bit register v26
 1055 reg_class v26_reg(
 1056     V26, V26_H
 1057 );
 1058 
 1059 // Class for 128 bit register v27
 1060 reg_class v27_reg(
 1061     V27, V27_H
 1062 );
 1063 
 1064 // Class for 128 bit register v28
 1065 reg_class v28_reg(
 1066     V28, V28_H
 1067 );
 1068 
 1069 // Class for 128 bit register v29
 1070 reg_class v29_reg(
 1071     V29, V29_H
 1072 );
 1073 
 1074 // Class for 128 bit register v30
 1075 reg_class v30_reg(
 1076     V30, V30_H
 1077 );
 1078 
 1079 // Class for 128 bit register v31
 1080 reg_class v31_reg(
 1081     V31, V31_H
 1082 );
 1083 
 1084 // Class for all SVE predicate registers.
 1085 reg_class pr_reg (
 1086     P0,
 1087     P1,
 1088     P2,
 1089     P3,
 1090     P4,
 1091     P5,
 1092     P6,
 1093     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1094     P8,
 1095     P9,
 1096     P10,
 1097     P11,
 1098     P12,
 1099     P13,
 1100     P14,
 1101     P15
 1102 );
 1103 
 1104 // Class for SVE governing predicate registers, which are used
 1105 // to determine the active elements of a predicated instruction.
 1106 reg_class gov_pr (
 1107     P0,
 1108     P1,
 1109     P2,
 1110     P3,
 1111     P4,
 1112     P5,
 1113     P6,
 1114     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1115 );
 1116 
 1117 reg_class p0_reg(P0);
 1118 reg_class p1_reg(P1);
 1119 
 1120 // Singleton class for condition codes
 1121 reg_class int_flags(RFLAGS);
 1122 
 1123 %}
 1124 
 1125 //----------DEFINITION BLOCK---------------------------------------------------
 1126 // Define name --> value mappings to inform the ADLC of an integer valued name
 1127 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1128 // Format:
 1129 //        int_def  <name>         ( <int_value>, <expression>);
 1130 // Generated Code in ad_<arch>.hpp
 1131 //        #define  <name>   (<expression>)
 1132 //        // value == <int_value>
 1133 // Generated code in ad_<arch>.cpp adlc_verification()
 1134 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1135 //
 1136 
 1137 // we follow the ppc-aix port in using a simple cost model which ranks
 1138 // register operations as cheap, memory ops as more expensive and
 1139 // branches as most expensive. the first two have a low as well as a
 1140 // normal cost. huge cost appears to be a way of saying don't do
 1141 // something
 1142 
 1143 definitions %{
 1144   // The default cost (of a register move instruction).
 1145   int_def INSN_COST            (    100,     100);
 1146   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1147   int_def CALL_COST            (    200,     2 * INSN_COST);
 1148   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1149 %}
 1150 
 1151 
 1152 //----------SOURCE BLOCK-------------------------------------------------------
 1153 // This is a block of C++ code which provides values, functions, and
 1154 // definitions necessary in the rest of the architecture description
 1155 
 1156 source_hpp %{
 1157 
 1158 #include "asm/macroAssembler.hpp"
 1159 #include "gc/shared/barrierSetAssembler.hpp"
 1160 #include "gc/shared/cardTable.hpp"
 1161 #include "gc/shared/cardTableBarrierSet.hpp"
 1162 #include "gc/shared/collectedHeap.hpp"
 1163 #include "opto/addnode.hpp"
 1164 #include "opto/convertnode.hpp"
 1165 #include "runtime/objectMonitor.hpp"
 1166 
 1167 extern RegMask _ANY_REG32_mask;
 1168 extern RegMask _ANY_REG_mask;
 1169 extern RegMask _PTR_REG_mask;
 1170 extern RegMask _NO_SPECIAL_REG32_mask;
 1171 extern RegMask _NO_SPECIAL_REG_mask;
 1172 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1173 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
 1174 
 1175 class CallStubImpl {
 1176 
 1177   //--------------------------------------------------------------
 1178   //---<  Used for optimization in Compile::shorten_branches  >---
 1179   //--------------------------------------------------------------
 1180 
 1181  public:
 1182   // Size of call trampoline stub.
 1183   static uint size_call_trampoline() {
 1184     return MacroAssembler::max_trampoline_stub_size(); // no call trampolines on this platform
 1185   }
 1186 
 1187   // number of relocations needed by a call trampoline stub
 1188   static uint reloc_call_trampoline() {
 1189     return 0; // no call trampolines on this platform
 1190   }
 1191 };
 1192 
 1193 class HandlerImpl {
 1194 
 1195  public:
 1196 
 1197   static int emit_exception_handler(C2_MacroAssembler *masm);
 1198   static int emit_deopt_handler(C2_MacroAssembler* masm);
 1199 
 1200   static uint size_exception_handler() {
 1201     return MacroAssembler::far_codestub_branch_size();
 1202   }
 1203 
 1204   static uint size_deopt_handler() {
 1205     // count one adr and one far branch instruction
 1206     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1207   }
 1208 };
 1209 
 1210 class Node::PD {
 1211 public:
 1212   enum NodeFlags {
 1213     _last_flag = Node::_last_flag
 1214   };
 1215 };
 1216 
 1217   bool is_CAS(int opcode, bool maybe_volatile);
 1218 
 1219   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1220 
 1221   bool unnecessary_acquire(const Node *barrier);
 1222   bool needs_acquiring_load(const Node *load);
 1223 
 1224   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1225 
 1226   bool unnecessary_release(const Node *barrier);
 1227   bool unnecessary_volatile(const Node *barrier);
 1228   bool needs_releasing_store(const Node *store);
 1229 
 1230   // predicate controlling translation of CompareAndSwapX
 1231   bool needs_acquiring_load_exclusive(const Node *load);
 1232 
 1233   // predicate controlling addressing modes
 1234   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1235 
 1236   // Convert BootTest condition to Assembler condition.
 1237   // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 1238   Assembler::Condition to_assembler_cond(BoolTest::mask cond);
 1239 %}
 1240 
 1241 source %{
 1242 
 1243   // Derived RegMask with conditionally allocatable registers
 1244 
 1245   void PhaseOutput::pd_perform_mach_node_analysis() {
 1246   }
 1247 
 1248   int MachNode::pd_alignment_required() const {
 1249     return 1;
 1250   }
 1251 
 1252   int MachNode::compute_padding(int current_offset) const {
 1253     return 0;
 1254   }
 1255 
 1256   RegMask _ANY_REG32_mask;
 1257   RegMask _ANY_REG_mask;
 1258   RegMask _PTR_REG_mask;
 1259   RegMask _NO_SPECIAL_REG32_mask;
 1260   RegMask _NO_SPECIAL_REG_mask;
 1261   RegMask _NO_SPECIAL_PTR_REG_mask;
 1262   RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
 1263 
 1264   void reg_mask_init() {
 1265     // We derive below RegMask(s) from the ones which are auto-generated from
 1266     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1267     // registers conditionally reserved.
 1268 
 1269     _ANY_REG32_mask = _ALL_REG32_mask;
 1270     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1271 
 1272     _ANY_REG_mask = _ALL_REG_mask;
 1273 
 1274     _PTR_REG_mask = _ALL_REG_mask;
 1275 
 1276     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1277     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1278 
 1279     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1280     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1281 
 1282     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1283     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1284 
 1285     // r27 is not allocatable when compressed oops is on and heapbase is not
 1286     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1287     if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
 1288       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1289       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1290       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1291     }
 1292 
 1293     // r29 is not allocatable when PreserveFramePointer is on
 1294     if (PreserveFramePointer) {
 1295       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1296       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1297       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1298     }
 1299 
 1300     _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
 1301     _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1302   }
 1303 
 1304   // Optimizaton of volatile gets and puts
 1305   // -------------------------------------
 1306   //
 1307   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1308   // use to implement volatile reads and writes. For a volatile read
 1309   // we simply need
 1310   //
 1311   //   ldar<x>
 1312   //
 1313   // and for a volatile write we need
 1314   //
 1315   //   stlr<x>
 1316   //
 1317   // Alternatively, we can implement them by pairing a normal
 1318   // load/store with a memory barrier. For a volatile read we need
 1319   //
 1320   //   ldr<x>
 1321   //   dmb ishld
 1322   //
 1323   // for a volatile write
 1324   //
 1325   //   dmb ish
 1326   //   str<x>
 1327   //   dmb ish
 1328   //
 1329   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1330   // sequences. These are normally translated to an instruction
 1331   // sequence like the following
 1332   //
 1333   //   dmb      ish
 1334   // retry:
 1335   //   ldxr<x>   rval raddr
 1336   //   cmp       rval rold
 1337   //   b.ne done
 1338   //   stlxr<x>  rval, rnew, rold
 1339   //   cbnz      rval retry
 1340   // done:
 1341   //   cset      r0, eq
 1342   //   dmb ishld
 1343   //
 1344   // Note that the exclusive store is already using an stlxr
 1345   // instruction. That is required to ensure visibility to other
 1346   // threads of the exclusive write (assuming it succeeds) before that
 1347   // of any subsequent writes.
 1348   //
 1349   // The following instruction sequence is an improvement on the above
 1350   //
 1351   // retry:
 1352   //   ldaxr<x>  rval raddr
 1353   //   cmp       rval rold
 1354   //   b.ne done
 1355   //   stlxr<x>  rval, rnew, rold
 1356   //   cbnz      rval retry
 1357   // done:
 1358   //   cset      r0, eq
 1359   //
 1360   // We don't need the leading dmb ish since the stlxr guarantees
 1361   // visibility of prior writes in the case that the swap is
 1362   // successful. Crucially we don't have to worry about the case where
 1363   // the swap is not successful since no valid program should be
 1364   // relying on visibility of prior changes by the attempting thread
 1365   // in the case where the CAS fails.
 1366   //
 1367   // Similarly, we don't need the trailing dmb ishld if we substitute
 1368   // an ldaxr instruction since that will provide all the guarantees we
 1369   // require regarding observation of changes made by other threads
 1370   // before any change to the CAS address observed by the load.
 1371   //
 1372   // In order to generate the desired instruction sequence we need to
 1373   // be able to identify specific 'signature' ideal graph node
 1374   // sequences which i) occur as a translation of a volatile reads or
 1375   // writes or CAS operations and ii) do not occur through any other
 1376   // translation or graph transformation. We can then provide
 1377   // alternative aldc matching rules which translate these node
 1378   // sequences to the desired machine code sequences. Selection of the
 1379   // alternative rules can be implemented by predicates which identify
 1380   // the relevant node sequences.
 1381   //
 1382   // The ideal graph generator translates a volatile read to the node
 1383   // sequence
 1384   //
 1385   //   LoadX[mo_acquire]
 1386   //   MemBarAcquire
 1387   //
 1388   // As a special case when using the compressed oops optimization we
 1389   // may also see this variant
 1390   //
 1391   //   LoadN[mo_acquire]
 1392   //   DecodeN
 1393   //   MemBarAcquire
 1394   //
 1395   // A volatile write is translated to the node sequence
 1396   //
 1397   //   MemBarRelease
 1398   //   StoreX[mo_release] {CardMark}-optional
 1399   //   MemBarVolatile
 1400   //
 1401   // n.b. the above node patterns are generated with a strict
 1402   // 'signature' configuration of input and output dependencies (see
 1403   // the predicates below for exact details). The card mark may be as
 1404   // simple as a few extra nodes or, in a few GC configurations, may
 1405   // include more complex control flow between the leading and
 1406   // trailing memory barriers. However, whatever the card mark
 1407   // configuration these signatures are unique to translated volatile
 1408   // reads/stores -- they will not appear as a result of any other
 1409   // bytecode translation or inlining nor as a consequence of
 1410   // optimizing transforms.
 1411   //
 1412   // We also want to catch inlined unsafe volatile gets and puts and
 1413   // be able to implement them using either ldar<x>/stlr<x> or some
 1414   // combination of ldr<x>/stlr<x> and dmb instructions.
 1415   //
 1416   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1417   // normal volatile put node sequence containing an extra cpuorder
 1418   // membar
 1419   //
 1420   //   MemBarRelease
 1421   //   MemBarCPUOrder
 1422   //   StoreX[mo_release] {CardMark}-optional
 1423   //   MemBarCPUOrder
 1424   //   MemBarVolatile
 1425   //
 1426   // n.b. as an aside, a cpuorder membar is not itself subject to
 1427   // matching and translation by adlc rules.  However, the rule
 1428   // predicates need to detect its presence in order to correctly
 1429   // select the desired adlc rules.
 1430   //
 1431   // Inlined unsafe volatile gets manifest as a slightly different
 1432   // node sequence to a normal volatile get because of the
 1433   // introduction of some CPUOrder memory barriers to bracket the
 1434   // Load. However, but the same basic skeleton of a LoadX feeding a
 1435   // MemBarAcquire, possibly through an optional DecodeN, is still
 1436   // present
 1437   //
 1438   //   MemBarCPUOrder
 1439   //        ||       \\
 1440   //   MemBarCPUOrder LoadX[mo_acquire]
 1441   //        ||            |
 1442   //        ||       {DecodeN} optional
 1443   //        ||       /
 1444   //     MemBarAcquire
 1445   //
 1446   // In this case the acquire membar does not directly depend on the
 1447   // load. However, we can be sure that the load is generated from an
 1448   // inlined unsafe volatile get if we see it dependent on this unique
 1449   // sequence of membar nodes. Similarly, given an acquire membar we
 1450   // can know that it was added because of an inlined unsafe volatile
 1451   // get if it is fed and feeds a cpuorder membar and if its feed
 1452   // membar also feeds an acquiring load.
 1453   //
 1454   // Finally an inlined (Unsafe) CAS operation is translated to the
 1455   // following ideal graph
 1456   //
 1457   //   MemBarRelease
 1458   //   MemBarCPUOrder
 1459   //   CompareAndSwapX {CardMark}-optional
 1460   //   MemBarCPUOrder
 1461   //   MemBarAcquire
 1462   //
 1463   // So, where we can identify these volatile read and write
 1464   // signatures we can choose to plant either of the above two code
 1465   // sequences. For a volatile read we can simply plant a normal
 1466   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1467   // also choose to inhibit translation of the MemBarAcquire and
 1468   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1469   //
 1470   // When we recognise a volatile store signature we can choose to
 1471   // plant at a dmb ish as a translation for the MemBarRelease, a
 1472   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1473   // Alternatively, we can inhibit translation of the MemBarRelease
 1474   // and MemBarVolatile and instead plant a simple stlr<x>
 1475   // instruction.
 1476   //
 1477   // when we recognise a CAS signature we can choose to plant a dmb
 1478   // ish as a translation for the MemBarRelease, the conventional
 1479   // macro-instruction sequence for the CompareAndSwap node (which
 1480   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1481   // Alternatively, we can elide generation of the dmb instructions
 1482   // and plant the alternative CompareAndSwap macro-instruction
 1483   // sequence (which uses ldaxr<x>).
 1484   //
 1485   // Of course, the above only applies when we see these signature
 1486   // configurations. We still want to plant dmb instructions in any
 1487   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1488   // MemBarVolatile. For example, at the end of a constructor which
 1489   // writes final/volatile fields we will see a MemBarRelease
 1490   // instruction and this needs a 'dmb ish' lest we risk the
 1491   // constructed object being visible without making the
 1492   // final/volatile field writes visible.
 1493   //
 1494   // n.b. the translation rules below which rely on detection of the
 1495   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1496   // If we see anything other than the signature configurations we
 1497   // always just translate the loads and stores to ldr<x> and str<x>
 1498   // and translate acquire, release and volatile membars to the
 1499   // relevant dmb instructions.
 1500   //
 1501 
 1502   // is_CAS(int opcode, bool maybe_volatile)
 1503   //
 1504   // return true if opcode is one of the possible CompareAndSwapX
 1505   // values otherwise false.
 1506 
 1507   bool is_CAS(int opcode, bool maybe_volatile)
 1508   {
 1509     switch(opcode) {
 1510       // We handle these
 1511     case Op_CompareAndSwapI:
 1512     case Op_CompareAndSwapL:
 1513     case Op_CompareAndSwapP:
 1514     case Op_CompareAndSwapN:
 1515     case Op_ShenandoahCompareAndSwapP:
 1516     case Op_ShenandoahCompareAndSwapN:
 1517     case Op_CompareAndSwapB:
 1518     case Op_CompareAndSwapS:
 1519     case Op_GetAndSetI:
 1520     case Op_GetAndSetL:
 1521     case Op_GetAndSetP:
 1522     case Op_GetAndSetN:
 1523     case Op_GetAndAddI:
 1524     case Op_GetAndAddL:
 1525       return true;
 1526     case Op_CompareAndExchangeI:
 1527     case Op_CompareAndExchangeN:
 1528     case Op_CompareAndExchangeB:
 1529     case Op_CompareAndExchangeS:
 1530     case Op_CompareAndExchangeL:
 1531     case Op_CompareAndExchangeP:
 1532     case Op_WeakCompareAndSwapB:
 1533     case Op_WeakCompareAndSwapS:
 1534     case Op_WeakCompareAndSwapI:
 1535     case Op_WeakCompareAndSwapL:
 1536     case Op_WeakCompareAndSwapP:
 1537     case Op_WeakCompareAndSwapN:
 1538     case Op_ShenandoahWeakCompareAndSwapP:
 1539     case Op_ShenandoahWeakCompareAndSwapN:
 1540     case Op_ShenandoahCompareAndExchangeP:
 1541     case Op_ShenandoahCompareAndExchangeN:
 1542       return maybe_volatile;
 1543     default:
 1544       return false;
 1545     }
 1546   }
 1547 
 1548   // helper to determine the maximum number of Phi nodes we may need to
 1549   // traverse when searching from a card mark membar for the merge mem
 1550   // feeding a trailing membar or vice versa
 1551 
 1552 // predicates controlling emit of ldr<x>/ldar<x>
 1553 
 1554 bool unnecessary_acquire(const Node *barrier)
 1555 {
 1556   assert(barrier->is_MemBar(), "expecting a membar");
 1557 
 1558   MemBarNode* mb = barrier->as_MemBar();
 1559 
 1560   if (mb->trailing_load()) {
 1561     return true;
 1562   }
 1563 
 1564   if (mb->trailing_load_store()) {
 1565     Node* load_store = mb->in(MemBarNode::Precedent);
 1566     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1567     return is_CAS(load_store->Opcode(), true);
 1568   }
 1569 
 1570   return false;
 1571 }
 1572 
 1573 bool needs_acquiring_load(const Node *n)
 1574 {
 1575   assert(n->is_Load(), "expecting a load");
 1576   LoadNode *ld = n->as_Load();
 1577   return ld->is_acquire();
 1578 }
 1579 
 1580 bool unnecessary_release(const Node *n)
 1581 {
 1582   assert((n->is_MemBar() &&
 1583           n->Opcode() == Op_MemBarRelease),
 1584          "expecting a release membar");
 1585 
 1586   MemBarNode *barrier = n->as_MemBar();
 1587   if (!barrier->leading()) {
 1588     return false;
 1589   } else {
 1590     Node* trailing = barrier->trailing_membar();
 1591     MemBarNode* trailing_mb = trailing->as_MemBar();
 1592     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1593     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1594 
 1595     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1596     if (mem->is_Store()) {
 1597       assert(mem->as_Store()->is_release(), "");
 1598       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1599       return true;
 1600     } else {
 1601       assert(mem->is_LoadStore(), "");
 1602       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1603       return is_CAS(mem->Opcode(), true);
 1604     }
 1605   }
 1606   return false;
 1607 }
 1608 
 1609 bool unnecessary_volatile(const Node *n)
 1610 {
 1611   // assert n->is_MemBar();
 1612   MemBarNode *mbvol = n->as_MemBar();
 1613 
 1614   bool release = mbvol->trailing_store();
 1615   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1616 #ifdef ASSERT
 1617   if (release) {
 1618     Node* leading = mbvol->leading_membar();
 1619     assert(leading->Opcode() == Op_MemBarRelease, "");
 1620     assert(leading->as_MemBar()->leading_store(), "");
 1621     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1622   }
 1623 #endif
 1624 
 1625   return release;
 1626 }
 1627 
 1628 // predicates controlling emit of str<x>/stlr<x>
 1629 
 1630 bool needs_releasing_store(const Node *n)
 1631 {
 1632   // assert n->is_Store();
 1633   StoreNode *st = n->as_Store();
 1634   return st->trailing_membar() != nullptr;
 1635 }
 1636 
 1637 // predicate controlling translation of CAS
 1638 //
 1639 // returns true if CAS needs to use an acquiring load otherwise false
 1640 
 1641 bool needs_acquiring_load_exclusive(const Node *n)
 1642 {
 1643   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1644   LoadStoreNode* ldst = n->as_LoadStore();
 1645   if (is_CAS(n->Opcode(), false)) {
 1646     assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
 1647   } else {
 1648     return ldst->trailing_membar() != nullptr;
 1649   }
 1650 
 1651   // so we can just return true here
 1652   return true;
 1653 }
 1654 
 1655 #define __ masm->
 1656 
 1657 // advance declarations for helper functions to convert register
 1658 // indices to register objects
 1659 
 1660 // the ad file has to provide implementations of certain methods
 1661 // expected by the generic code
 1662 //
 1663 // REQUIRED FUNCTIONALITY
 1664 
 1665 //=============================================================================
 1666 
 1667 // !!!!! Special hack to get all types of calls to specify the byte offset
 1668 //       from the start of the call to the point where the return address
 1669 //       will point.
 1670 
 1671 int MachCallStaticJavaNode::ret_addr_offset()
 1672 {
 1673   // call should be a simple bl
 1674   int off = 4;
 1675   return off;
 1676 }
 1677 
 1678 int MachCallDynamicJavaNode::ret_addr_offset()
 1679 {
 1680   return 16; // movz, movk, movk, bl
 1681 }
 1682 
 1683 int MachCallRuntimeNode::ret_addr_offset() {
 1684   // for generated stubs the call will be
 1685   //   bl(addr)
 1686   // or with far branches
 1687   //   bl(trampoline_stub)
 1688   // for real runtime callouts it will be six instructions
 1689   // see aarch64_enc_java_to_runtime
 1690   //   adr(rscratch2, retaddr)
 1691   //   str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
 1692   //   lea(rscratch1, RuntimeAddress(addr)
 1693   //   blr(rscratch1)
 1694   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1695   if (cb) {
 1696     return 1 * NativeInstruction::instruction_size;
 1697   } else {
 1698     return 6 * NativeInstruction::instruction_size;
 1699   }
 1700 }
 1701 
 1702 //=============================================================================
 1703 
 1704 #ifndef PRODUCT
 1705 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1706   st->print("BREAKPOINT");
 1707 }
 1708 #endif
 1709 
 1710 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1711   __ brk(0);
 1712 }
 1713 
 1714 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1715   return MachNode::size(ra_);
 1716 }
 1717 
 1718 //=============================================================================
 1719 
 1720 #ifndef PRODUCT
 1721   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1722     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1723   }
 1724 #endif
 1725 
 1726   void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
 1727     for (int i = 0; i < _count; i++) {
 1728       __ nop();
 1729     }
 1730   }
 1731 
 1732   uint MachNopNode::size(PhaseRegAlloc*) const {
 1733     return _count * NativeInstruction::instruction_size;
 1734   }
 1735 
 1736 //=============================================================================
 1737 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1738 
 1739 int ConstantTable::calculate_table_base_offset() const {
 1740   return 0;  // absolute addressing, no offset
 1741 }
 1742 
 1743 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1744 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1745   ShouldNotReachHere();
 1746 }
 1747 
 1748 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
 1749   // Empty encoding
 1750 }
 1751 
 1752 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1753   return 0;
 1754 }
 1755 
 1756 #ifndef PRODUCT
 1757 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1758   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1759 }
 1760 #endif
 1761 
 1762 #ifndef PRODUCT
 1763 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1764   Compile* C = ra_->C;
 1765 
 1766   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1767 
 1768   if (C->output()->need_stack_bang(framesize))
 1769     st->print("# stack bang size=%d\n\t", framesize);
 1770 
 1771   if (VM_Version::use_rop_protection()) {
 1772     st->print("ldr  zr, [lr]\n\t");
 1773     st->print("paciaz\n\t");
 1774   }
 1775   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1776     st->print("sub  sp, sp, #%d\n\t", framesize);
 1777     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1778     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1779   } else {
 1780     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1781     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1782     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1783     st->print("sub  sp, sp, rscratch1");
 1784   }
 1785   if (C->stub_function() == nullptr) {
 1786     st->print("\n\t");
 1787     st->print("ldr  rscratch1, [guard]\n\t");
 1788     st->print("dmb ishld\n\t");
 1789     st->print("ldr  rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
 1790     st->print("cmp  rscratch1, rscratch2\n\t");
 1791     st->print("b.eq skip");
 1792     st->print("\n\t");
 1793     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1794     st->print("b skip\n\t");
 1795     st->print("guard: int\n\t");
 1796     st->print("\n\t");
 1797     st->print("skip:\n\t");
 1798   }
 1799 }
 1800 #endif
 1801 
 1802 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1803   Compile* C = ra_->C;
 1804 
 1805   // n.b. frame size includes space for return pc and rfp
 1806   const int framesize = C->output()->frame_size_in_bytes();
 1807 
 1808   if (C->clinit_barrier_on_entry()) {
 1809     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1810 
 1811     Label L_skip_barrier;
 1812 
 1813     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1814     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1815     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1816     __ bind(L_skip_barrier);
 1817   }
 1818 
 1819   if (C->max_vector_size() > 0) {
 1820     __ reinitialize_ptrue();
 1821   }
 1822 
 1823   int bangsize = C->output()->bang_size_in_bytes();
 1824   if (C->output()->need_stack_bang(bangsize))
 1825     __ generate_stack_overflow_check(bangsize);
 1826 
 1827   __ build_frame(framesize);
 1828 
 1829   if (C->stub_function() == nullptr) {
 1830     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1831     // Dummy labels for just measuring the code size
 1832     Label dummy_slow_path;
 1833     Label dummy_continuation;
 1834     Label dummy_guard;
 1835     Label* slow_path = &dummy_slow_path;
 1836     Label* continuation = &dummy_continuation;
 1837     Label* guard = &dummy_guard;
 1838     if (!Compile::current()->output()->in_scratch_emit_size()) {
 1839       // Use real labels from actual stub when not emitting code for the purpose of measuring its size
 1840       C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
 1841       Compile::current()->output()->add_stub(stub);
 1842       slow_path = &stub->entry();
 1843       continuation = &stub->continuation();
 1844       guard = &stub->guard();
 1845     }
 1846     // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
 1847     bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
 1848   }
 1849 
 1850   if (VerifyStackAtCalls) {
 1851     Unimplemented();
 1852   }
 1853 
 1854   C->output()->set_frame_complete(__ offset());
 1855 
 1856   if (C->has_mach_constant_base_node()) {
 1857     // NOTE: We set the table base offset here because users might be
 1858     // emitted before MachConstantBaseNode.
 1859     ConstantTable& constant_table = C->output()->constant_table();
 1860     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1861   }
 1862 }
 1863 
 1864 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1865 {
 1866   return MachNode::size(ra_); // too many variables; just compute it
 1867                               // the hard way
 1868 }
 1869 
 1870 int MachPrologNode::reloc() const
 1871 {
 1872   return 0;
 1873 }
 1874 
 1875 //=============================================================================
 1876 
 1877 #ifndef PRODUCT
 1878 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1879   Compile* C = ra_->C;
 1880   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1881 
 1882   st->print("# pop frame %d\n\t",framesize);
 1883 
 1884   if (framesize == 0) {
 1885     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1886   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1887     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1888     st->print("add  sp, sp, #%d\n\t", framesize);
 1889   } else {
 1890     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1891     st->print("add  sp, sp, rscratch1\n\t");
 1892     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1893   }
 1894   if (VM_Version::use_rop_protection()) {
 1895     st->print("autiaz\n\t");
 1896     st->print("ldr  zr, [lr]\n\t");
 1897   }
 1898 
 1899   if (do_polling() && C->is_method_compilation()) {
 1900     st->print("# test polling word\n\t");
 1901     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1902     st->print("cmp  sp, rscratch1\n\t");
 1903     st->print("bhi #slow_path");
 1904   }
 1905 }
 1906 #endif
 1907 
 1908 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1909   Compile* C = ra_->C;
 1910   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1911 
 1912   __ remove_frame(framesize);
 1913 
 1914   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1915     __ reserved_stack_check();
 1916   }
 1917 
 1918   if (do_polling() && C->is_method_compilation()) {
 1919     Label dummy_label;
 1920     Label* code_stub = &dummy_label;
 1921     if (!C->output()->in_scratch_emit_size()) {
 1922       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
 1923       C->output()->add_stub(stub);
 1924       code_stub = &stub->entry();
 1925     }
 1926     __ relocate(relocInfo::poll_return_type);
 1927     __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
 1928   }
 1929 }
 1930 
 1931 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 1932   // Variable size. Determine dynamically.
 1933   return MachNode::size(ra_);
 1934 }
 1935 
 1936 int MachEpilogNode::reloc() const {
 1937   // Return number of relocatable values contained in this instruction.
 1938   return 1; // 1 for polling page.
 1939 }
 1940 
 1941 const Pipeline * MachEpilogNode::pipeline() const {
 1942   return MachNode::pipeline_class();
 1943 }
 1944 
 1945 //=============================================================================
 1946 
 1947 static enum RC rc_class(OptoReg::Name reg) {
 1948 
 1949   if (reg == OptoReg::Bad) {
 1950     return rc_bad;
 1951   }
 1952 
 1953   // we have 32 int registers * 2 halves
 1954   int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
 1955 
 1956   if (reg < slots_of_int_registers) {
 1957     return rc_int;
 1958   }
 1959 
 1960   // we have 32 float register * 8 halves
 1961   int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
 1962   if (reg < slots_of_int_registers + slots_of_float_registers) {
 1963     return rc_float;
 1964   }
 1965 
 1966   int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
 1967   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 1968     return rc_predicate;
 1969   }
 1970 
 1971   // Between predicate regs & stack is the flags.
 1972   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 1973 
 1974   return rc_stack;
 1975 }
 1976 
 1977 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 1978   Compile* C = ra_->C;
 1979 
 1980   // Get registers to move.
 1981   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 1982   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 1983   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 1984   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 1985 
 1986   enum RC src_hi_rc = rc_class(src_hi);
 1987   enum RC src_lo_rc = rc_class(src_lo);
 1988   enum RC dst_hi_rc = rc_class(dst_hi);
 1989   enum RC dst_lo_rc = rc_class(dst_lo);
 1990 
 1991   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 1992 
 1993   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 1994     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 1995            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 1996            "expected aligned-adjacent pairs");
 1997   }
 1998 
 1999   if (src_lo == dst_lo && src_hi == dst_hi) {
 2000     return 0;            // Self copy, no move.
 2001   }
 2002 
 2003   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 2004               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 2005   int src_offset = ra_->reg2offset(src_lo);
 2006   int dst_offset = ra_->reg2offset(dst_lo);
 2007 
 2008   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2009     uint ireg = ideal_reg();
 2010     if (ireg == Op_VecA && masm) {
 2011       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 2012       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2013         // stack->stack
 2014         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 2015                                                 sve_vector_reg_size_in_bytes);
 2016       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2017         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2018                             sve_vector_reg_size_in_bytes);
 2019       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2020         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2021                               sve_vector_reg_size_in_bytes);
 2022       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2023         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2024                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 2025                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 2026       } else {
 2027         ShouldNotReachHere();
 2028       }
 2029     } else if (masm) {
 2030       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 2031       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 2032       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2033         // stack->stack
 2034         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2035         if (ireg == Op_VecD) {
 2036           __ unspill(rscratch1, true, src_offset);
 2037           __ spill(rscratch1, true, dst_offset);
 2038         } else {
 2039           __ spill_copy128(src_offset, dst_offset);
 2040         }
 2041       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2042         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2043                ireg == Op_VecD ? __ T8B : __ T16B,
 2044                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2045       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2046         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2047                  ireg == Op_VecD ? __ D : __ Q,
 2048                  ra_->reg2offset(dst_lo));
 2049       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2050         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2051                    ireg == Op_VecD ? __ D : __ Q,
 2052                    ra_->reg2offset(src_lo));
 2053       } else {
 2054         ShouldNotReachHere();
 2055       }
 2056     }
 2057   } else if (masm) {
 2058     switch (src_lo_rc) {
 2059     case rc_int:
 2060       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2061         if (is64) {
 2062             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2063                    as_Register(Matcher::_regEncode[src_lo]));
 2064         } else {
 2065             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2066                     as_Register(Matcher::_regEncode[src_lo]));
 2067         }
 2068       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2069         if (is64) {
 2070             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2071                      as_Register(Matcher::_regEncode[src_lo]));
 2072         } else {
 2073             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2074                      as_Register(Matcher::_regEncode[src_lo]));
 2075         }
 2076       } else {                    // gpr --> stack spill
 2077         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2078         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2079       }
 2080       break;
 2081     case rc_float:
 2082       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2083         if (is64) {
 2084             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2085                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2086         } else {
 2087             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2088                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2089         }
 2090       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2091         if (is64) {
 2092             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2093                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2094         } else {
 2095             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2096                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2097         }
 2098       } else {                    // fpr --> stack spill
 2099         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2100         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2101                  is64 ? __ D : __ S, dst_offset);
 2102       }
 2103       break;
 2104     case rc_stack:
 2105       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2106         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2107       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2108         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2109                    is64 ? __ D : __ S, src_offset);
 2110       } else if (dst_lo_rc == rc_predicate) {
 2111         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2112                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2113       } else {                    // stack --> stack copy
 2114         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2115         if (ideal_reg() == Op_RegVectMask) {
 2116           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2117                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2118         } else {
 2119           __ unspill(rscratch1, is64, src_offset);
 2120           __ spill(rscratch1, is64, dst_offset);
 2121         }
 2122       }
 2123       break;
 2124     case rc_predicate:
 2125       if (dst_lo_rc == rc_predicate) {
 2126         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2127       } else if (dst_lo_rc == rc_stack) {
 2128         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2129                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2130       } else {
 2131         assert(false, "bad src and dst rc_class combination.");
 2132         ShouldNotReachHere();
 2133       }
 2134       break;
 2135     default:
 2136       assert(false, "bad rc_class for spill");
 2137       ShouldNotReachHere();
 2138     }
 2139   }
 2140 
 2141   if (st) {
 2142     st->print("spill ");
 2143     if (src_lo_rc == rc_stack) {
 2144       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2145     } else {
 2146       st->print("%s -> ", Matcher::regName[src_lo]);
 2147     }
 2148     if (dst_lo_rc == rc_stack) {
 2149       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2150     } else {
 2151       st->print("%s", Matcher::regName[dst_lo]);
 2152     }
 2153     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2154       int vsize = 0;
 2155       switch (ideal_reg()) {
 2156       case Op_VecD:
 2157         vsize = 64;
 2158         break;
 2159       case Op_VecX:
 2160         vsize = 128;
 2161         break;
 2162       case Op_VecA:
 2163         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2164         break;
 2165       default:
 2166         assert(false, "bad register type for spill");
 2167         ShouldNotReachHere();
 2168       }
 2169       st->print("\t# vector spill size = %d", vsize);
 2170     } else if (ideal_reg() == Op_RegVectMask) {
 2171       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2172       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2173       st->print("\t# predicate spill size = %d", vsize);
 2174     } else {
 2175       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2176     }
 2177   }
 2178 
 2179   return 0;
 2180 
 2181 }
 2182 
 2183 #ifndef PRODUCT
 2184 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2185   if (!ra_)
 2186     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2187   else
 2188     implementation(nullptr, ra_, false, st);
 2189 }
 2190 #endif
 2191 
 2192 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 2193   implementation(masm, ra_, false, nullptr);
 2194 }
 2195 
 2196 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2197   return MachNode::size(ra_);
 2198 }
 2199 
 2200 //=============================================================================
 2201 
 2202 #ifndef PRODUCT
 2203 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2204   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2205   int reg = ra_->get_reg_first(this);
 2206   st->print("add %s, rsp, #%d]\t# box lock",
 2207             Matcher::regName[reg], offset);
 2208 }
 2209 #endif
 2210 
 2211 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 2212   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2213   int reg    = ra_->get_encode(this);
 2214 
 2215   // This add will handle any 24-bit signed offset. 24 bits allows an
 2216   // 8 megabyte stack frame.
 2217   __ add(as_Register(reg), sp, offset);
 2218 }
 2219 
 2220 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2221   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2222   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2223 
 2224   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2225     return NativeInstruction::instruction_size;
 2226   } else {
 2227     return 2 * NativeInstruction::instruction_size;
 2228   }
 2229 }
 2230 
 2231 //=============================================================================
 2232 
 2233 #ifndef PRODUCT
 2234 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2235 {
 2236   st->print_cr("# MachUEPNode");
 2237   if (UseCompressedClassPointers) {
 2238     st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2239     st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
 2240     st->print_cr("\tcmpw rscratch1, r10");
 2241   } else {
 2242     st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2243     st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
 2244     st->print_cr("\tcmp rscratch1, r10");
 2245   }
 2246   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2247 }
 2248 #endif
 2249 
 2250 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 2251 {
 2252   __ ic_check(InteriorEntryAlignment);
 2253 }
 2254 
 2255 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2256 {
 2257   return MachNode::size(ra_);
 2258 }
 2259 
 2260 // REQUIRED EMIT CODE
 2261 
 2262 //=============================================================================
 2263 
 2264 // Emit exception handler code.
 2265 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
 2266 {
 2267   // mov rscratch1 #exception_blob_entry_point
 2268   // br rscratch1
 2269   // Note that the code buffer's insts_mark is always relative to insts.
 2270   // That's why we must use the macroassembler to generate a handler.
 2271   address base = __ start_a_stub(size_exception_handler());
 2272   if (base == nullptr) {
 2273     ciEnv::current()->record_failure("CodeCache is full");
 2274     return 0;  // CodeBuffer::expand failed
 2275   }
 2276   int offset = __ offset();
 2277   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2278   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2279   __ end_a_stub();
 2280   return offset;
 2281 }
 2282 
 2283 // Emit deopt handler code.
 2284 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
 2285 {
 2286   // Note that the code buffer's insts_mark is always relative to insts.
 2287   // That's why we must use the macroassembler to generate a handler.
 2288   address base = __ start_a_stub(size_deopt_handler());
 2289   if (base == nullptr) {
 2290     ciEnv::current()->record_failure("CodeCache is full");
 2291     return 0;  // CodeBuffer::expand failed
 2292   }
 2293   int offset = __ offset();
 2294 
 2295   __ adr(lr, __ pc());
 2296   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2297 
 2298   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2299   __ end_a_stub();
 2300   return offset;
 2301 }
 2302 
 2303 // REQUIRED MATCHER CODE
 2304 
 2305 //=============================================================================
 2306 
 2307 bool Matcher::match_rule_supported(int opcode) {
 2308   if (!has_match_rule(opcode))
 2309     return false;
 2310 
 2311   switch (opcode) {
 2312     case Op_OnSpinWait:
 2313       return VM_Version::supports_on_spin_wait();
 2314     case Op_CacheWB:
 2315     case Op_CacheWBPreSync:
 2316     case Op_CacheWBPostSync:
 2317       if (!VM_Version::supports_data_cache_line_flush()) {
 2318         return false;
 2319       }
 2320       break;
 2321     case Op_ExpandBits:
 2322     case Op_CompressBits:
 2323       if (!VM_Version::supports_svebitperm()) {
 2324         return false;
 2325       }
 2326       break;
 2327     case Op_FmaF:
 2328     case Op_FmaD:
 2329     case Op_FmaVF:
 2330     case Op_FmaVD:
 2331       if (!UseFMA) {
 2332         return false;
 2333       }
 2334       break;
 2335     case Op_FmaHF:
 2336       // UseFMA flag also needs to be checked along with FEAT_FP16
 2337       if (!UseFMA || !is_feat_fp16_supported()) {
 2338         return false;
 2339       }
 2340       break;
 2341     case Op_AddHF:
 2342     case Op_SubHF:
 2343     case Op_MulHF:
 2344     case Op_DivHF:
 2345     case Op_MinHF:
 2346     case Op_MaxHF:
 2347     case Op_SqrtHF:
 2348       // Half-precision floating point scalar operations require FEAT_FP16
 2349       // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
 2350       // features are supported.
 2351       if (!is_feat_fp16_supported()) {
 2352         return false;
 2353       }
 2354       break;
 2355   }
 2356 
 2357   return true; // Per default match rules are supported.
 2358 }
 2359 
 2360 const RegMask* Matcher::predicate_reg_mask(void) {
 2361   return &_PR_REG_mask;
 2362 }
 2363 
 2364 bool Matcher::supports_vector_calling_convention(void) {
 2365   return EnableVectorSupport;
 2366 }
 2367 
 2368 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2369   assert(EnableVectorSupport, "sanity");
 2370   int lo = V0_num;
 2371   int hi = V0_H_num;
 2372   if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
 2373     hi = V0_K_num;
 2374   }
 2375   return OptoRegPair(hi, lo);
 2376 }
 2377 
 2378 // Is this branch offset short enough that a short branch can be used?
 2379 //
 2380 // NOTE: If the platform does not provide any short branch variants, then
 2381 //       this method should return false for offset 0.
 2382 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2383   // The passed offset is relative to address of the branch.
 2384 
 2385   return (-32768 <= offset && offset < 32768);
 2386 }
 2387 
 2388 // Vector width in bytes.
 2389 int Matcher::vector_width_in_bytes(BasicType bt) {
 2390   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2391   int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
 2392   // Minimum 2 values in vector
 2393   if (size < 2*type2aelembytes(bt)) size = 0;
 2394   // But never < 4
 2395   if (size < 4) size = 0;
 2396   return size;
 2397 }
 2398 
 2399 // Limits on vector size (number of elements) loaded into vector.
 2400 int Matcher::max_vector_size(const BasicType bt) {
 2401   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2402 }
 2403 
 2404 int Matcher::min_vector_size(const BasicType bt) {
 2405   // Usually, the shortest vector length supported by AArch64 ISA and
 2406   // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
 2407   // vectors in a few special cases.
 2408   int size;
 2409   switch(bt) {
 2410     case T_BOOLEAN:
 2411       // Load/store a vector mask with only 2 elements for vector types
 2412       // such as "2I/2F/2L/2D".
 2413       size = 2;
 2414       break;
 2415     case T_BYTE:
 2416       // Generate a "4B" vector, to support vector cast between "8B/16B"
 2417       // and "4S/4I/4L/4F/4D".
 2418       size = 4;
 2419       break;
 2420     case T_SHORT:
 2421       // Generate a "2S" vector, to support vector cast between "4S/8S"
 2422       // and "2I/2L/2F/2D".
 2423       size = 2;
 2424       break;
 2425     default:
 2426       // Limit the min vector length to 64-bit.
 2427       size = 8 / type2aelembytes(bt);
 2428       // The number of elements in a vector should be at least 2.
 2429       size = MAX2(size, 2);
 2430   }
 2431 
 2432   int max_size = max_vector_size(bt);
 2433   return MIN2(size, max_size);
 2434 }
 2435 
 2436 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
 2437   return Matcher::max_vector_size(bt);
 2438 }
 2439 
 2440 // Actual max scalable vector register length.
 2441 int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2442   return Matcher::max_vector_size(bt);
 2443 }
 2444 
 2445 // Vector ideal reg.
 2446 uint Matcher::vector_ideal_reg(int len) {
 2447   if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
 2448     return Op_VecA;
 2449   }
 2450   switch(len) {
 2451     // For 16-bit/32-bit mask vector, reuse VecD.
 2452     case  2:
 2453     case  4:
 2454     case  8: return Op_VecD;
 2455     case 16: return Op_VecX;
 2456   }
 2457   ShouldNotReachHere();
 2458   return 0;
 2459 }
 2460 
 2461 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
 2462   assert(Matcher::is_generic_vector(generic_opnd), "not generic");
 2463   switch (ideal_reg) {
 2464     case Op_VecA: return new vecAOper();
 2465     case Op_VecD: return new vecDOper();
 2466     case Op_VecX: return new vecXOper();
 2467   }
 2468   ShouldNotReachHere();
 2469   return nullptr;
 2470 }
 2471 
 2472 bool Matcher::is_reg2reg_move(MachNode* m) {
 2473   return false;
 2474 }
 2475 
 2476 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2477   return opnd->opcode() == VREG;
 2478 }
 2479 
 2480 // Return whether or not this register is ever used as an argument.
 2481 // This function is used on startup to build the trampoline stubs in
 2482 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2483 // call in the trampoline, and arguments in those registers not be
 2484 // available to the callee.
 2485 bool Matcher::can_be_java_arg(int reg)
 2486 {
 2487   return
 2488     reg ==  R0_num || reg == R0_H_num ||
 2489     reg ==  R1_num || reg == R1_H_num ||
 2490     reg ==  R2_num || reg == R2_H_num ||
 2491     reg ==  R3_num || reg == R3_H_num ||
 2492     reg ==  R4_num || reg == R4_H_num ||
 2493     reg ==  R5_num || reg == R5_H_num ||
 2494     reg ==  R6_num || reg == R6_H_num ||
 2495     reg ==  R7_num || reg == R7_H_num ||
 2496     reg ==  V0_num || reg == V0_H_num ||
 2497     reg ==  V1_num || reg == V1_H_num ||
 2498     reg ==  V2_num || reg == V2_H_num ||
 2499     reg ==  V3_num || reg == V3_H_num ||
 2500     reg ==  V4_num || reg == V4_H_num ||
 2501     reg ==  V5_num || reg == V5_H_num ||
 2502     reg ==  V6_num || reg == V6_H_num ||
 2503     reg ==  V7_num || reg == V7_H_num;
 2504 }
 2505 
 2506 bool Matcher::is_spillable_arg(int reg)
 2507 {
 2508   return can_be_java_arg(reg);
 2509 }
 2510 
 2511 uint Matcher::int_pressure_limit()
 2512 {
 2513   // JDK-8183543: When taking the number of available registers as int
 2514   // register pressure threshold, the jtreg test:
 2515   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2516   // failed due to C2 compilation failure with
 2517   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2518   //
 2519   // A derived pointer is live at CallNode and then is flagged by RA
 2520   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2521   // derived pointers and lastly fail to spill after reaching maximum
 2522   // number of iterations. Lowering the default pressure threshold to
 2523   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2524   // a high register pressure area of the code so that split_DEF can
 2525   // generate DefinitionSpillCopy for the derived pointer.
 2526   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2527   if (!PreserveFramePointer) {
 2528     // When PreserveFramePointer is off, frame pointer is allocatable,
 2529     // but different from other SOC registers, it is excluded from
 2530     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2531     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2532     // See check_pressure_at_fatproj().
 2533     default_int_pressure_threshold--;
 2534   }
 2535   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2536 }
 2537 
 2538 uint Matcher::float_pressure_limit()
 2539 {
 2540   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2541   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2542 }
 2543 
 2544 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2545   return false;
 2546 }
 2547 
 2548 RegMask Matcher::divI_proj_mask() {
 2549   ShouldNotReachHere();
 2550   return RegMask();
 2551 }
 2552 
 2553 // Register for MODI projection of divmodI.
 2554 RegMask Matcher::modI_proj_mask() {
 2555   ShouldNotReachHere();
 2556   return RegMask();
 2557 }
 2558 
 2559 // Register for DIVL projection of divmodL.
 2560 RegMask Matcher::divL_proj_mask() {
 2561   ShouldNotReachHere();
 2562   return RegMask();
 2563 }
 2564 
 2565 // Register for MODL projection of divmodL.
 2566 RegMask Matcher::modL_proj_mask() {
 2567   ShouldNotReachHere();
 2568   return RegMask();
 2569 }
 2570 
 2571 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2572   return FP_REG_mask();
 2573 }
 2574 
 2575 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2576   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2577     Node* u = addp->fast_out(i);
 2578     if (u->is_LoadStore()) {
 2579       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2580       // instructions) only take register indirect as an operand, so
 2581       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2582       // must fail.
 2583       return false;
 2584     }
 2585     if (u->is_Mem()) {
 2586       int opsize = u->as_Mem()->memory_size();
 2587       assert(opsize > 0, "unexpected memory operand size");
 2588       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2589         return false;
 2590       }
 2591     }
 2592   }
 2593   return true;
 2594 }
 2595 
 2596 // Convert BootTest condition to Assembler condition.
 2597 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 2598 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
 2599   Assembler::Condition result;
 2600   switch(cond) {
 2601     case BoolTest::eq:
 2602       result = Assembler::EQ; break;
 2603     case BoolTest::ne:
 2604       result = Assembler::NE; break;
 2605     case BoolTest::le:
 2606       result = Assembler::LE; break;
 2607     case BoolTest::ge:
 2608       result = Assembler::GE; break;
 2609     case BoolTest::lt:
 2610       result = Assembler::LT; break;
 2611     case BoolTest::gt:
 2612       result = Assembler::GT; break;
 2613     case BoolTest::ule:
 2614       result = Assembler::LS; break;
 2615     case BoolTest::uge:
 2616       result = Assembler::HS; break;
 2617     case BoolTest::ult:
 2618       result = Assembler::LO; break;
 2619     case BoolTest::ugt:
 2620       result = Assembler::HI; break;
 2621     case BoolTest::overflow:
 2622       result = Assembler::VS; break;
 2623     case BoolTest::no_overflow:
 2624       result = Assembler::VC; break;
 2625     default:
 2626       ShouldNotReachHere();
 2627       return Assembler::Condition(-1);
 2628   }
 2629 
 2630   // Check conversion
 2631   if (cond & BoolTest::unsigned_compare) {
 2632     assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
 2633   } else {
 2634     assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
 2635   }
 2636 
 2637   return result;
 2638 }
 2639 
 2640 // Binary src (Replicate con)
 2641 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
 2642   if (n == nullptr || m == nullptr) {
 2643     return false;
 2644   }
 2645 
 2646   if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
 2647     return false;
 2648   }
 2649 
 2650   Node* imm_node = m->in(1);
 2651   if (!imm_node->is_Con()) {
 2652     return false;
 2653   }
 2654 
 2655   const Type* t = imm_node->bottom_type();
 2656   if (!(t->isa_int() || t->isa_long())) {
 2657     return false;
 2658   }
 2659 
 2660   switch (n->Opcode()) {
 2661   case Op_AndV:
 2662   case Op_OrV:
 2663   case Op_XorV: {
 2664     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
 2665     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2666     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2667   }
 2668   case Op_AddVB:
 2669     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2670   case Op_AddVS:
 2671   case Op_AddVI:
 2672     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2673   case Op_AddVL:
 2674     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2675   default:
 2676     return false;
 2677   }
 2678 }
 2679 
 2680 // (XorV src (Replicate m1))
 2681 // (XorVMask src (MaskAll m1))
 2682 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
 2683   if (n != nullptr && m != nullptr) {
 2684     return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
 2685            VectorNode::is_all_ones_vector(m);
 2686   }
 2687   return false;
 2688 }
 2689 
 2690 // Should the matcher clone input 'm' of node 'n'?
 2691 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2692   if (is_vshift_con_pattern(n, m) ||
 2693       is_vector_bitwise_not_pattern(n, m) ||
 2694       is_valid_sve_arith_imm_pattern(n, m) ||
 2695       is_encode_and_store_pattern(n, m)) {
 2696     mstack.push(m, Visit);
 2697     return true;
 2698   }
 2699   return false;
 2700 }
 2701 
 2702 // Should the Matcher clone shifts on addressing modes, expecting them
 2703 // to be subsumed into complex addressing expressions or compute them
 2704 // into registers?
 2705 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2706 
 2707   // Loads and stores with indirect memory input (e.g., volatile loads and
 2708   // stores) do not subsume the input into complex addressing expressions. If
 2709   // the addressing expression is input to at least one such load or store, do
 2710   // not clone the addressing expression. Query needs_acquiring_load and
 2711   // needs_releasing_store as a proxy for indirect memory input, as it is not
 2712   // possible to directly query for indirect memory input at this stage.
 2713   for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
 2714     Node* n = m->fast_out(i);
 2715     if (n->is_Load() && needs_acquiring_load(n)) {
 2716       return false;
 2717     }
 2718     if (n->is_Store() && needs_releasing_store(n)) {
 2719       return false;
 2720     }
 2721   }
 2722 
 2723   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2724     return true;
 2725   }
 2726 
 2727   Node *off = m->in(AddPNode::Offset);
 2728   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2729       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2730       // Are there other uses besides address expressions?
 2731       !is_visited(off)) {
 2732     address_visited.set(off->_idx); // Flag as address_visited
 2733     mstack.push(off->in(2), Visit);
 2734     Node *conv = off->in(1);
 2735     if (conv->Opcode() == Op_ConvI2L &&
 2736         // Are there other uses besides address expressions?
 2737         !is_visited(conv)) {
 2738       address_visited.set(conv->_idx); // Flag as address_visited
 2739       mstack.push(conv->in(1), Pre_Visit);
 2740     } else {
 2741       mstack.push(conv, Pre_Visit);
 2742     }
 2743     address_visited.test_set(m->_idx); // Flag as address_visited
 2744     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2745     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2746     return true;
 2747   } else if (off->Opcode() == Op_ConvI2L &&
 2748              // Are there other uses besides address expressions?
 2749              !is_visited(off)) {
 2750     address_visited.test_set(m->_idx); // Flag as address_visited
 2751     address_visited.set(off->_idx); // Flag as address_visited
 2752     mstack.push(off->in(1), Pre_Visit);
 2753     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2754     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2755     return true;
 2756   }
 2757   return false;
 2758 }
 2759 
 2760 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2761   {                                                                     \
 2762     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2763     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2764     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2765     __ INSN(REG, as_Register(BASE));                                    \
 2766   }
 2767 
 2768 
 2769 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2770   {
 2771     Address::extend scale;
 2772 
 2773     // Hooboy, this is fugly.  We need a way to communicate to the
 2774     // encoder that the index needs to be sign extended, so we have to
 2775     // enumerate all the cases.
 2776     switch (opcode) {
 2777     case INDINDEXSCALEDI2L:
 2778     case INDINDEXSCALEDI2LN:
 2779     case INDINDEXI2L:
 2780     case INDINDEXI2LN:
 2781       scale = Address::sxtw(size);
 2782       break;
 2783     default:
 2784       scale = Address::lsl(size);
 2785     }
 2786 
 2787     if (index == -1) {
 2788       return Address(base, disp);
 2789     } else {
 2790       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2791       return Address(base, as_Register(index), scale);
 2792     }
 2793   }
 2794 
 2795 
 2796 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2797 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2798 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2799 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2800                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2801 
 2802   // Used for all non-volatile memory accesses.  The use of
 2803   // $mem->opcode() to discover whether this pattern uses sign-extended
 2804   // offsets is something of a kludge.
 2805   static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
 2806                         Register reg, int opcode,
 2807                         Register base, int index, int scale, int disp,
 2808                         int size_in_memory)
 2809   {
 2810     Address addr = mem2address(opcode, base, index, scale, disp);
 2811     if (addr.getMode() == Address::base_plus_offset) {
 2812       /* Fix up any out-of-range offsets. */
 2813       assert_different_registers(rscratch1, base);
 2814       assert_different_registers(rscratch1, reg);
 2815       addr = __ legitimize_address(addr, size_in_memory, rscratch1);
 2816     }
 2817     (masm->*insn)(reg, addr);
 2818   }
 2819 
 2820   static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
 2821                         FloatRegister reg, int opcode,
 2822                         Register base, int index, int size, int disp,
 2823                         int size_in_memory)
 2824   {
 2825     Address::extend scale;
 2826 
 2827     switch (opcode) {
 2828     case INDINDEXSCALEDI2L:
 2829     case INDINDEXSCALEDI2LN:
 2830       scale = Address::sxtw(size);
 2831       break;
 2832     default:
 2833       scale = Address::lsl(size);
 2834     }
 2835 
 2836     if (index == -1) {
 2837       // Fix up any out-of-range offsets.
 2838       assert_different_registers(rscratch1, base);
 2839       Address addr = Address(base, disp);
 2840       addr = __ legitimize_address(addr, size_in_memory, rscratch1);
 2841       (masm->*insn)(reg, addr);
 2842     } else {
 2843       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2844       (masm->*insn)(reg, Address(base, as_Register(index), scale));
 2845     }
 2846   }
 2847 
 2848   static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
 2849                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2850                         int opcode, Register base, int index, int size, int disp)
 2851   {
 2852     if (index == -1) {
 2853       (masm->*insn)(reg, T, Address(base, disp));
 2854     } else {
 2855       assert(disp == 0, "unsupported address mode");
 2856       (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2857     }
 2858   }
 2859 
 2860 %}
 2861 
 2862 
 2863 
 2864 //----------ENCODING BLOCK-----------------------------------------------------
 2865 // This block specifies the encoding classes used by the compiler to
 2866 // output byte streams.  Encoding classes are parameterized macros
 2867 // used by Machine Instruction Nodes in order to generate the bit
 2868 // encoding of the instruction.  Operands specify their base encoding
 2869 // interface with the interface keyword.  There are currently
 2870 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2871 // COND_INTER.  REG_INTER causes an operand to generate a function
 2872 // which returns its register number when queried.  CONST_INTER causes
 2873 // an operand to generate a function which returns the value of the
 2874 // constant when queried.  MEMORY_INTER causes an operand to generate
 2875 // four functions which return the Base Register, the Index Register,
 2876 // the Scale Value, and the Offset Value of the operand when queried.
 2877 // COND_INTER causes an operand to generate six functions which return
 2878 // the encoding code (ie - encoding bits for the instruction)
 2879 // associated with each basic boolean condition for a conditional
 2880 // instruction.
 2881 //
 2882 // Instructions specify two basic values for encoding.  Again, a
 2883 // function is available to check if the constant displacement is an
 2884 // oop. They use the ins_encode keyword to specify their encoding
 2885 // classes (which must be a sequence of enc_class names, and their
 2886 // parameters, specified in the encoding block), and they use the
 2887 // opcode keyword to specify, in order, their primary, secondary, and
 2888 // tertiary opcode.  Only the opcode sections which a particular
 2889 // instruction needs for encoding need to be specified.
 2890 encode %{
 2891   // Build emit functions for each basic byte or larger field in the
 2892   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2893   // from C++ code in the enc_class source block.  Emit functions will
 2894   // live in the main source block for now.  In future, we can
 2895   // generalize this by adding a syntax that specifies the sizes of
 2896   // fields in an order, so that the adlc can build the emit functions
 2897   // automagically
 2898 
 2899   // catch all for unimplemented encodings
 2900   enc_class enc_unimplemented %{
 2901     __ unimplemented("C2 catch all");
 2902   %}
 2903 
 2904   // BEGIN Non-volatile memory access
 2905 
 2906   // This encoding class is generated automatically from ad_encode.m4.
 2907   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2908   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2909     Register dst_reg = as_Register($dst$$reg);
 2910     loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2911                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2912   %}
 2913 
 2914   // This encoding class is generated automatically from ad_encode.m4.
 2915   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2916   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2917     Register dst_reg = as_Register($dst$$reg);
 2918     loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2919                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2920   %}
 2921 
 2922   // This encoding class is generated automatically from ad_encode.m4.
 2923   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2924   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2925     Register dst_reg = as_Register($dst$$reg);
 2926     loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2927                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2928   %}
 2929 
 2930   // This encoding class is generated automatically from ad_encode.m4.
 2931   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2932   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2933     Register dst_reg = as_Register($dst$$reg);
 2934     loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2935                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2936   %}
 2937 
 2938   // This encoding class is generated automatically from ad_encode.m4.
 2939   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2940   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2941     Register dst_reg = as_Register($dst$$reg);
 2942     loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2943                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2944   %}
 2945 
 2946   // This encoding class is generated automatically from ad_encode.m4.
 2947   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2948   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2949     Register dst_reg = as_Register($dst$$reg);
 2950     loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2951                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2952   %}
 2953 
 2954   // This encoding class is generated automatically from ad_encode.m4.
 2955   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2956   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2957     Register dst_reg = as_Register($dst$$reg);
 2958     loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2959                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2960   %}
 2961 
 2962   // This encoding class is generated automatically from ad_encode.m4.
 2963   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2964   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2965     Register dst_reg = as_Register($dst$$reg);
 2966     loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2967                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2968   %}
 2969 
 2970   // This encoding class is generated automatically from ad_encode.m4.
 2971   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2972   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2973     Register dst_reg = as_Register($dst$$reg);
 2974     loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2975                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2976   %}
 2977 
 2978   // This encoding class is generated automatically from ad_encode.m4.
 2979   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2980   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2981     Register dst_reg = as_Register($dst$$reg);
 2982     loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2983                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2984   %}
 2985 
 2986   // This encoding class is generated automatically from ad_encode.m4.
 2987   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2988   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2989     Register dst_reg = as_Register($dst$$reg);
 2990     loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2991                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2992   %}
 2993 
 2994   // This encoding class is generated automatically from ad_encode.m4.
 2995   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2996   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2997     Register dst_reg = as_Register($dst$$reg);
 2998     loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2999                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3000   %}
 3001 
 3002   // This encoding class is generated automatically from ad_encode.m4.
 3003   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3004   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 3005     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3006     loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 3007                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3008   %}
 3009 
 3010   // This encoding class is generated automatically from ad_encode.m4.
 3011   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3012   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 3013     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3014     loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 3015                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3016   %}
 3017 
 3018   // This encoding class is generated automatically from ad_encode.m4.
 3019   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3020   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 3021     Register src_reg = as_Register($src$$reg);
 3022     loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
 3023                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3024   %}
 3025 
 3026   // This encoding class is generated automatically from ad_encode.m4.
 3027   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3028   enc_class aarch64_enc_strb0(memory1 mem) %{
 3029     loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3030                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3031   %}
 3032 
 3033   // This encoding class is generated automatically from ad_encode.m4.
 3034   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3035   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 3036     Register src_reg = as_Register($src$$reg);
 3037     loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
 3038                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3039   %}
 3040 
 3041   // This encoding class is generated automatically from ad_encode.m4.
 3042   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3043   enc_class aarch64_enc_strh0(memory2 mem) %{
 3044     loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
 3045                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3046   %}
 3047 
 3048   // This encoding class is generated automatically from ad_encode.m4.
 3049   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3050   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 3051     Register src_reg = as_Register($src$$reg);
 3052     loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
 3053                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3054   %}
 3055 
 3056   // This encoding class is generated automatically from ad_encode.m4.
 3057   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3058   enc_class aarch64_enc_strw0(memory4 mem) %{
 3059     loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
 3060                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3061   %}
 3062 
 3063   // This encoding class is generated automatically from ad_encode.m4.
 3064   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3065   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 3066     Register src_reg = as_Register($src$$reg);
 3067     // we sometimes get asked to store the stack pointer into the
 3068     // current thread -- we cannot do that directly on AArch64
 3069     if (src_reg == r31_sp) {
 3070       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3071       __ mov(rscratch2, sp);
 3072       src_reg = rscratch2;
 3073     }
 3074     loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
 3075                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3076   %}
 3077 
 3078   // This encoding class is generated automatically from ad_encode.m4.
 3079   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3080   enc_class aarch64_enc_str0(memory8 mem) %{
 3081     loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
 3082                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3083   %}
 3084 
 3085   // This encoding class is generated automatically from ad_encode.m4.
 3086   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3087   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3088     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3089     loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
 3090                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3091   %}
 3092 
 3093   // This encoding class is generated automatically from ad_encode.m4.
 3094   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3095   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3096     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3097     loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
 3098                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3099   %}
 3100 
 3101   // This encoding class is generated automatically from ad_encode.m4.
 3102   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3103   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3104       __ membar(Assembler::StoreStore);
 3105       loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3106                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3107   %}
 3108 
 3109   // END Non-volatile memory access
 3110 
 3111   // Vector loads and stores
 3112   enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
 3113     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3114     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3115        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3116   %}
 3117 
 3118   enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
 3119     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3120     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3121        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3122   %}
 3123 
 3124   enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
 3125     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3126     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3127        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3128   %}
 3129 
 3130   enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
 3131     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3132     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3133        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3134   %}
 3135 
 3136   enc_class aarch64_enc_strvH(vReg src, memory mem) %{
 3137     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3138     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
 3139        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3140   %}
 3141 
 3142   enc_class aarch64_enc_strvS(vReg src, memory mem) %{
 3143     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3144     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
 3145        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3146   %}
 3147 
 3148   enc_class aarch64_enc_strvD(vReg src, memory mem) %{
 3149     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3150     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
 3151        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3152   %}
 3153 
 3154   enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
 3155     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3156     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3157        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3158   %}
 3159 
 3160   // volatile loads and stores
 3161 
 3162   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3163     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3164                  rscratch1, stlrb);
 3165   %}
 3166 
 3167   enc_class aarch64_enc_stlrb0(memory mem) %{
 3168     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3169                  rscratch1, stlrb);
 3170   %}
 3171 
 3172   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3173     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3174                  rscratch1, stlrh);
 3175   %}
 3176 
 3177   enc_class aarch64_enc_stlrh0(memory mem) %{
 3178     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3179                  rscratch1, stlrh);
 3180   %}
 3181 
 3182   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3183     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3184                  rscratch1, stlrw);
 3185   %}
 3186 
 3187   enc_class aarch64_enc_stlrw0(memory mem) %{
 3188     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3189                  rscratch1, stlrw);
 3190   %}
 3191 
 3192   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3193     Register dst_reg = as_Register($dst$$reg);
 3194     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3195              rscratch1, ldarb);
 3196     __ sxtbw(dst_reg, dst_reg);
 3197   %}
 3198 
 3199   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3200     Register dst_reg = as_Register($dst$$reg);
 3201     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3202              rscratch1, ldarb);
 3203     __ sxtb(dst_reg, dst_reg);
 3204   %}
 3205 
 3206   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3207     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3208              rscratch1, ldarb);
 3209   %}
 3210 
 3211   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3212     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3213              rscratch1, ldarb);
 3214   %}
 3215 
 3216   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3217     Register dst_reg = as_Register($dst$$reg);
 3218     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3219              rscratch1, ldarh);
 3220     __ sxthw(dst_reg, dst_reg);
 3221   %}
 3222 
 3223   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3224     Register dst_reg = as_Register($dst$$reg);
 3225     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3226              rscratch1, ldarh);
 3227     __ sxth(dst_reg, dst_reg);
 3228   %}
 3229 
 3230   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3231     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3232              rscratch1, ldarh);
 3233   %}
 3234 
 3235   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3236     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3237              rscratch1, ldarh);
 3238   %}
 3239 
 3240   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3241     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3242              rscratch1, ldarw);
 3243   %}
 3244 
 3245   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3246     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3247              rscratch1, ldarw);
 3248   %}
 3249 
 3250   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3251     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3252              rscratch1, ldar);
 3253   %}
 3254 
 3255   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3256     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3257              rscratch1, ldarw);
 3258     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3259   %}
 3260 
 3261   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3262     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3263              rscratch1, ldar);
 3264     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3265   %}
 3266 
 3267   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3268     Register src_reg = as_Register($src$$reg);
 3269     // we sometimes get asked to store the stack pointer into the
 3270     // current thread -- we cannot do that directly on AArch64
 3271     if (src_reg == r31_sp) {
 3272       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3273       __ mov(rscratch2, sp);
 3274       src_reg = rscratch2;
 3275     }
 3276     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3277                  rscratch1, stlr);
 3278   %}
 3279 
 3280   enc_class aarch64_enc_stlr0(memory mem) %{
 3281     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3282                  rscratch1, stlr);
 3283   %}
 3284 
 3285   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3286     {
 3287       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3288       __ fmovs(rscratch2, src_reg);
 3289     }
 3290     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3291                  rscratch1, stlrw);
 3292   %}
 3293 
 3294   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3295     {
 3296       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3297       __ fmovd(rscratch2, src_reg);
 3298     }
 3299     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3300                  rscratch1, stlr);
 3301   %}
 3302 
 3303   // synchronized read/update encodings
 3304 
 3305   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3306     Register dst_reg = as_Register($dst$$reg);
 3307     Register base = as_Register($mem$$base);
 3308     int index = $mem$$index;
 3309     int scale = $mem$$scale;
 3310     int disp = $mem$$disp;
 3311     if (index == -1) {
 3312        if (disp != 0) {
 3313         __ lea(rscratch1, Address(base, disp));
 3314         __ ldaxr(dst_reg, rscratch1);
 3315       } else {
 3316         // TODO
 3317         // should we ever get anything other than this case?
 3318         __ ldaxr(dst_reg, base);
 3319       }
 3320     } else {
 3321       Register index_reg = as_Register(index);
 3322       if (disp == 0) {
 3323         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3324         __ ldaxr(dst_reg, rscratch1);
 3325       } else {
 3326         __ lea(rscratch1, Address(base, disp));
 3327         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3328         __ ldaxr(dst_reg, rscratch1);
 3329       }
 3330     }
 3331   %}
 3332 
 3333   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3334     Register src_reg = as_Register($src$$reg);
 3335     Register base = as_Register($mem$$base);
 3336     int index = $mem$$index;
 3337     int scale = $mem$$scale;
 3338     int disp = $mem$$disp;
 3339     if (index == -1) {
 3340        if (disp != 0) {
 3341         __ lea(rscratch2, Address(base, disp));
 3342         __ stlxr(rscratch1, src_reg, rscratch2);
 3343       } else {
 3344         // TODO
 3345         // should we ever get anything other than this case?
 3346         __ stlxr(rscratch1, src_reg, base);
 3347       }
 3348     } else {
 3349       Register index_reg = as_Register(index);
 3350       if (disp == 0) {
 3351         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3352         __ stlxr(rscratch1, src_reg, rscratch2);
 3353       } else {
 3354         __ lea(rscratch2, Address(base, disp));
 3355         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3356         __ stlxr(rscratch1, src_reg, rscratch2);
 3357       }
 3358     }
 3359     __ cmpw(rscratch1, zr);
 3360   %}
 3361 
 3362   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3363     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3364     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3365                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3366                /*weak*/ false, noreg);
 3367   %}
 3368 
 3369   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3370     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3371     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3372                Assembler::word, /*acquire*/ false, /*release*/ true,
 3373                /*weak*/ false, noreg);
 3374   %}
 3375 
 3376   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3377     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3378     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3379                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3380                /*weak*/ false, noreg);
 3381   %}
 3382 
 3383   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3384     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3385     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3386                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3387                /*weak*/ false, noreg);
 3388   %}
 3389 
 3390 
 3391   // The only difference between aarch64_enc_cmpxchg and
 3392   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3393   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3394   // lock.
 3395   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3396     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3397     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3398                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3399                /*weak*/ false, noreg);
 3400   %}
 3401 
 3402   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3403     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3404     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3405                Assembler::word, /*acquire*/ true, /*release*/ true,
 3406                /*weak*/ false, noreg);
 3407   %}
 3408 
 3409   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3410     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3411     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3412                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3413                /*weak*/ false, noreg);
 3414   %}
 3415 
 3416   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3417     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3418     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3419                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3420                /*weak*/ false, noreg);
 3421   %}
 3422 
 3423   // auxiliary used for CompareAndSwapX to set result register
 3424   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3425     Register res_reg = as_Register($res$$reg);
 3426     __ cset(res_reg, Assembler::EQ);
 3427   %}
 3428 
 3429   // prefetch encodings
 3430 
 3431   enc_class aarch64_enc_prefetchw(memory mem) %{
 3432     Register base = as_Register($mem$$base);
 3433     int index = $mem$$index;
 3434     int scale = $mem$$scale;
 3435     int disp = $mem$$disp;
 3436     if (index == -1) {
 3437       // Fix up any out-of-range offsets.
 3438       assert_different_registers(rscratch1, base);
 3439       Address addr = Address(base, disp);
 3440       addr = __ legitimize_address(addr, 8, rscratch1);
 3441       __ prfm(addr, PSTL1KEEP);
 3442     } else {
 3443       Register index_reg = as_Register(index);
 3444       if (disp == 0) {
 3445         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3446       } else {
 3447         __ lea(rscratch1, Address(base, disp));
 3448 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3449       }
 3450     }
 3451   %}
 3452 
 3453   // mov encodings
 3454 
 3455   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3456     uint32_t con = (uint32_t)$src$$constant;
 3457     Register dst_reg = as_Register($dst$$reg);
 3458     if (con == 0) {
 3459       __ movw(dst_reg, zr);
 3460     } else {
 3461       __ movw(dst_reg, con);
 3462     }
 3463   %}
 3464 
 3465   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3466     Register dst_reg = as_Register($dst$$reg);
 3467     uint64_t con = (uint64_t)$src$$constant;
 3468     if (con == 0) {
 3469       __ mov(dst_reg, zr);
 3470     } else {
 3471       __ mov(dst_reg, con);
 3472     }
 3473   %}
 3474 
 3475   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3476     Register dst_reg = as_Register($dst$$reg);
 3477     address con = (address)$src$$constant;
 3478     if (con == nullptr || con == (address)1) {
 3479       ShouldNotReachHere();
 3480     } else {
 3481       relocInfo::relocType rtype = $src->constant_reloc();
 3482       if (rtype == relocInfo::oop_type) {
 3483         __ movoop(dst_reg, (jobject)con);
 3484       } else if (rtype == relocInfo::metadata_type) {
 3485         __ mov_metadata(dst_reg, (Metadata*)con);
 3486       } else {
 3487         assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
 3488         if (! __ is_valid_AArch64_address(con) ||
 3489             con < (address)(uintptr_t)os::vm_page_size()) {
 3490           __ mov(dst_reg, con);
 3491         } else {
 3492           uint64_t offset;
 3493           __ adrp(dst_reg, con, offset);
 3494           __ add(dst_reg, dst_reg, offset);
 3495         }
 3496       }
 3497     }
 3498   %}
 3499 
 3500   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3501     Register dst_reg = as_Register($dst$$reg);
 3502     __ mov(dst_reg, zr);
 3503   %}
 3504 
 3505   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3506     Register dst_reg = as_Register($dst$$reg);
 3507     __ mov(dst_reg, (uint64_t)1);
 3508   %}
 3509 
 3510   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3511     Register dst_reg = as_Register($dst$$reg);
 3512     address con = (address)$src$$constant;
 3513     if (con == nullptr) {
 3514       ShouldNotReachHere();
 3515     } else {
 3516       relocInfo::relocType rtype = $src->constant_reloc();
 3517       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3518       __ set_narrow_oop(dst_reg, (jobject)con);
 3519     }
 3520   %}
 3521 
 3522   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3523     Register dst_reg = as_Register($dst$$reg);
 3524     __ mov(dst_reg, zr);
 3525   %}
 3526 
 3527   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3528     Register dst_reg = as_Register($dst$$reg);
 3529     address con = (address)$src$$constant;
 3530     if (con == nullptr) {
 3531       ShouldNotReachHere();
 3532     } else {
 3533       relocInfo::relocType rtype = $src->constant_reloc();
 3534       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3535       __ set_narrow_klass(dst_reg, (Klass *)con);
 3536     }
 3537   %}
 3538 
 3539   // arithmetic encodings
 3540 
 3541   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3542     Register dst_reg = as_Register($dst$$reg);
 3543     Register src_reg = as_Register($src1$$reg);
 3544     int32_t con = (int32_t)$src2$$constant;
 3545     // add has primary == 0, subtract has primary == 1
 3546     if ($primary) { con = -con; }
 3547     if (con < 0) {
 3548       __ subw(dst_reg, src_reg, -con);
 3549     } else {
 3550       __ addw(dst_reg, src_reg, con);
 3551     }
 3552   %}
 3553 
 3554   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3555     Register dst_reg = as_Register($dst$$reg);
 3556     Register src_reg = as_Register($src1$$reg);
 3557     int32_t con = (int32_t)$src2$$constant;
 3558     // add has primary == 0, subtract has primary == 1
 3559     if ($primary) { con = -con; }
 3560     if (con < 0) {
 3561       __ sub(dst_reg, src_reg, -con);
 3562     } else {
 3563       __ add(dst_reg, src_reg, con);
 3564     }
 3565   %}
 3566 
 3567   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3568    Register dst_reg = as_Register($dst$$reg);
 3569    Register src1_reg = as_Register($src1$$reg);
 3570    Register src2_reg = as_Register($src2$$reg);
 3571     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3572   %}
 3573 
 3574   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3575    Register dst_reg = as_Register($dst$$reg);
 3576    Register src1_reg = as_Register($src1$$reg);
 3577    Register src2_reg = as_Register($src2$$reg);
 3578     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3579   %}
 3580 
 3581   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3582    Register dst_reg = as_Register($dst$$reg);
 3583    Register src1_reg = as_Register($src1$$reg);
 3584    Register src2_reg = as_Register($src2$$reg);
 3585     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3586   %}
 3587 
 3588   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3589    Register dst_reg = as_Register($dst$$reg);
 3590    Register src1_reg = as_Register($src1$$reg);
 3591    Register src2_reg = as_Register($src2$$reg);
 3592     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3593   %}
 3594 
 3595   // compare instruction encodings
 3596 
 3597   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3598     Register reg1 = as_Register($src1$$reg);
 3599     Register reg2 = as_Register($src2$$reg);
 3600     __ cmpw(reg1, reg2);
 3601   %}
 3602 
 3603   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3604     Register reg = as_Register($src1$$reg);
 3605     int32_t val = $src2$$constant;
 3606     if (val >= 0) {
 3607       __ subsw(zr, reg, val);
 3608     } else {
 3609       __ addsw(zr, reg, -val);
 3610     }
 3611   %}
 3612 
 3613   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3614     Register reg1 = as_Register($src1$$reg);
 3615     uint32_t val = (uint32_t)$src2$$constant;
 3616     __ movw(rscratch1, val);
 3617     __ cmpw(reg1, rscratch1);
 3618   %}
 3619 
 3620   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3621     Register reg1 = as_Register($src1$$reg);
 3622     Register reg2 = as_Register($src2$$reg);
 3623     __ cmp(reg1, reg2);
 3624   %}
 3625 
 3626   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3627     Register reg = as_Register($src1$$reg);
 3628     int64_t val = $src2$$constant;
 3629     if (val >= 0) {
 3630       __ subs(zr, reg, val);
 3631     } else if (val != -val) {
 3632       __ adds(zr, reg, -val);
 3633     } else {
 3634     // aargh, Long.MIN_VALUE is a special case
 3635       __ orr(rscratch1, zr, (uint64_t)val);
 3636       __ subs(zr, reg, rscratch1);
 3637     }
 3638   %}
 3639 
 3640   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3641     Register reg1 = as_Register($src1$$reg);
 3642     uint64_t val = (uint64_t)$src2$$constant;
 3643     __ mov(rscratch1, val);
 3644     __ cmp(reg1, rscratch1);
 3645   %}
 3646 
 3647   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3648     Register reg1 = as_Register($src1$$reg);
 3649     Register reg2 = as_Register($src2$$reg);
 3650     __ cmp(reg1, reg2);
 3651   %}
 3652 
 3653   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3654     Register reg1 = as_Register($src1$$reg);
 3655     Register reg2 = as_Register($src2$$reg);
 3656     __ cmpw(reg1, reg2);
 3657   %}
 3658 
 3659   enc_class aarch64_enc_testp(iRegP src) %{
 3660     Register reg = as_Register($src$$reg);
 3661     __ cmp(reg, zr);
 3662   %}
 3663 
 3664   enc_class aarch64_enc_testn(iRegN src) %{
 3665     Register reg = as_Register($src$$reg);
 3666     __ cmpw(reg, zr);
 3667   %}
 3668 
 3669   enc_class aarch64_enc_b(label lbl) %{
 3670     Label *L = $lbl$$label;
 3671     __ b(*L);
 3672   %}
 3673 
 3674   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3675     Label *L = $lbl$$label;
 3676     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3677   %}
 3678 
 3679   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3680     Label *L = $lbl$$label;
 3681     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3682   %}
 3683 
 3684   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3685   %{
 3686      Register sub_reg = as_Register($sub$$reg);
 3687      Register super_reg = as_Register($super$$reg);
 3688      Register temp_reg = as_Register($temp$$reg);
 3689      Register result_reg = as_Register($result$$reg);
 3690 
 3691      Label miss;
 3692      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3693                                      nullptr, &miss,
 3694                                      /*set_cond_codes:*/ true);
 3695      if ($primary) {
 3696        __ mov(result_reg, zr);
 3697      }
 3698      __ bind(miss);
 3699   %}
 3700 
 3701   enc_class aarch64_enc_java_static_call(method meth) %{
 3702     address addr = (address)$meth$$method;
 3703     address call;
 3704     if (!_method) {
 3705       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3706       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
 3707       if (call == nullptr) {
 3708         ciEnv::current()->record_failure("CodeCache is full");
 3709         return;
 3710       }
 3711     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 3712       // The NOP here is purely to ensure that eliding a call to
 3713       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 3714       __ nop();
 3715       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 3716     } else {
 3717       int method_index = resolved_method_index(masm);
 3718       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3719                                                   : static_call_Relocation::spec(method_index);
 3720       call = __ trampoline_call(Address(addr, rspec));
 3721       if (call == nullptr) {
 3722         ciEnv::current()->record_failure("CodeCache is full");
 3723         return;
 3724       }
 3725       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 3726         // Calls of the same statically bound method can share
 3727         // a stub to the interpreter.
 3728         __ code()->shared_stub_to_interp_for(_method, call - __ begin());
 3729       } else {
 3730         // Emit stub for static call
 3731         address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
 3732         if (stub == nullptr) {
 3733           ciEnv::current()->record_failure("CodeCache is full");
 3734           return;
 3735         }
 3736       }
 3737     }
 3738 
 3739     __ post_call_nop();
 3740 
 3741     // Only non uncommon_trap calls need to reinitialize ptrue.
 3742     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3743       __ reinitialize_ptrue();
 3744     }
 3745   %}
 3746 
 3747   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3748     int method_index = resolved_method_index(masm);
 3749     address call = __ ic_call((address)$meth$$method, method_index);
 3750     if (call == nullptr) {
 3751       ciEnv::current()->record_failure("CodeCache is full");
 3752       return;
 3753     }
 3754     __ post_call_nop();
 3755     if (Compile::current()->max_vector_size() > 0) {
 3756       __ reinitialize_ptrue();
 3757     }
 3758   %}
 3759 
 3760   enc_class aarch64_enc_call_epilog() %{
 3761     if (VerifyStackAtCalls) {
 3762       // Check that stack depth is unchanged: find majik cookie on stack
 3763       __ call_Unimplemented();
 3764     }
 3765   %}
 3766 
 3767   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3768     // some calls to generated routines (arraycopy code) are scheduled
 3769     // by C2 as runtime calls. if so we can call them using a br (they
 3770     // will be in a reachable segment) otherwise we have to use a blr
 3771     // which loads the absolute address into a register.
 3772     address entry = (address)$meth$$method;
 3773     CodeBlob *cb = CodeCache::find_blob(entry);
 3774     if (cb) {
 3775       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3776       if (call == nullptr) {
 3777         ciEnv::current()->record_failure("CodeCache is full");
 3778         return;
 3779       }
 3780       __ post_call_nop();
 3781     } else {
 3782       Label retaddr;
 3783       // Make the anchor frame walkable
 3784       __ adr(rscratch2, retaddr);
 3785       __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
 3786       __ lea(rscratch1, RuntimeAddress(entry));
 3787       __ blr(rscratch1);
 3788       __ bind(retaddr);
 3789       __ post_call_nop();
 3790     }
 3791     if (Compile::current()->max_vector_size() > 0) {
 3792       __ reinitialize_ptrue();
 3793     }
 3794   %}
 3795 
 3796   enc_class aarch64_enc_rethrow() %{
 3797     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3798   %}
 3799 
 3800   enc_class aarch64_enc_ret() %{
 3801 #ifdef ASSERT
 3802     if (Compile::current()->max_vector_size() > 0) {
 3803       __ verify_ptrue();
 3804     }
 3805 #endif
 3806     __ ret(lr);
 3807   %}
 3808 
 3809   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3810     Register target_reg = as_Register($jump_target$$reg);
 3811     __ br(target_reg);
 3812   %}
 3813 
 3814   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3815     Register target_reg = as_Register($jump_target$$reg);
 3816     // exception oop should be in r0
 3817     // ret addr has been popped into lr
 3818     // callee expects it in r3
 3819     __ mov(r3, lr);
 3820     __ br(target_reg);
 3821   %}
 3822 
 3823 %}
 3824 
 3825 //----------FRAME--------------------------------------------------------------
 3826 // Definition of frame structure and management information.
 3827 //
 3828 //  S T A C K   L A Y O U T    Allocators stack-slot number
 3829 //                             |   (to get allocators register number
 3830 //  G  Owned by    |        |  v    add OptoReg::stack0())
 3831 //  r   CALLER     |        |
 3832 //  o     |        +--------+      pad to even-align allocators stack-slot
 3833 //  w     V        |  pad0  |        numbers; owned by CALLER
 3834 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 3835 //  h     ^        |   in   |  5
 3836 //        |        |  args  |  4   Holes in incoming args owned by SELF
 3837 //  |     |        |        |  3
 3838 //  |     |        +--------+
 3839 //  V     |        | old out|      Empty on Intel, window on Sparc
 3840 //        |    old |preserve|      Must be even aligned.
 3841 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 3842 //        |        |   in   |  3   area for Intel ret address
 3843 //     Owned by    |preserve|      Empty on Sparc.
 3844 //       SELF      +--------+
 3845 //        |        |  pad2  |  2   pad to align old SP
 3846 //        |        +--------+  1
 3847 //        |        | locks  |  0
 3848 //        |        +--------+----> OptoReg::stack0(), even aligned
 3849 //        |        |  pad1  | 11   pad to align new SP
 3850 //        |        +--------+
 3851 //        |        |        | 10
 3852 //        |        | spills |  9   spills
 3853 //        V        |        |  8   (pad0 slot for callee)
 3854 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 3855 //        ^        |  out   |  7
 3856 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 3857 //     Owned by    +--------+
 3858 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 3859 //        |    new |preserve|      Must be even-aligned.
 3860 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 3861 //        |        |        |
 3862 //
 3863 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 3864 //         known from SELF's arguments and the Java calling convention.
 3865 //         Region 6-7 is determined per call site.
 3866 // Note 2: If the calling convention leaves holes in the incoming argument
 3867 //         area, those holes are owned by SELF.  Holes in the outgoing area
 3868 //         are owned by the CALLEE.  Holes should not be necessary in the
 3869 //         incoming area, as the Java calling convention is completely under
 3870 //         the control of the AD file.  Doubles can be sorted and packed to
 3871 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 3872 //         varargs C calling conventions.
 3873 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 3874 //         even aligned with pad0 as needed.
 3875 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 3876 //           (the latter is true on Intel but is it false on AArch64?)
 3877 //         region 6-11 is even aligned; it may be padded out more so that
 3878 //         the region from SP to FP meets the minimum stack alignment.
 3879 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 3880 //         alignment.  Region 11, pad1, may be dynamically extended so that
 3881 //         SP meets the minimum alignment.
 3882 
 3883 frame %{
 3884   // These three registers define part of the calling convention
 3885   // between compiled code and the interpreter.
 3886 
 3887   // Inline Cache Register or Method for I2C.
 3888   inline_cache_reg(R12);
 3889 
 3890   // Number of stack slots consumed by locking an object
 3891   sync_stack_slots(2);
 3892 
 3893   // Compiled code's Frame Pointer
 3894   frame_pointer(R31);
 3895 
 3896   // Interpreter stores its frame pointer in a register which is
 3897   // stored to the stack by I2CAdaptors.
 3898   // I2CAdaptors convert from interpreted java to compiled java.
 3899   interpreter_frame_pointer(R29);
 3900 
 3901   // Stack alignment requirement
 3902   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 3903 
 3904   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 3905   // for calls to C.  Supports the var-args backing area for register parms.
 3906   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 3907 
 3908   // The after-PROLOG location of the return address.  Location of
 3909   // return address specifies a type (REG or STACK) and a number
 3910   // representing the register number (i.e. - use a register name) or
 3911   // stack slot.
 3912   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 3913   // Otherwise, it is above the locks and verification slot and alignment word
 3914   // TODO this may well be correct but need to check why that - 2 is there
 3915   // ppc port uses 0 but we definitely need to allow for fixed_slots
 3916   // which folds in the space used for monitors
 3917   return_addr(STACK - 2 +
 3918               align_up((Compile::current()->in_preserve_stack_slots() +
 3919                         Compile::current()->fixed_slots()),
 3920                        stack_alignment_in_slots()));
 3921 
 3922   // Location of compiled Java return values.  Same as C for now.
 3923   return_value
 3924   %{
 3925     // TODO do we allow ideal_reg == Op_RegN???
 3926     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 3927            "only return normal values");
 3928 
 3929     static const int lo[Op_RegL + 1] = { // enum name
 3930       0,                                 // Op_Node
 3931       0,                                 // Op_Set
 3932       R0_num,                            // Op_RegN
 3933       R0_num,                            // Op_RegI
 3934       R0_num,                            // Op_RegP
 3935       V0_num,                            // Op_RegF
 3936       V0_num,                            // Op_RegD
 3937       R0_num                             // Op_RegL
 3938     };
 3939 
 3940     static const int hi[Op_RegL + 1] = { // enum name
 3941       0,                                 // Op_Node
 3942       0,                                 // Op_Set
 3943       OptoReg::Bad,                      // Op_RegN
 3944       OptoReg::Bad,                      // Op_RegI
 3945       R0_H_num,                          // Op_RegP
 3946       OptoReg::Bad,                      // Op_RegF
 3947       V0_H_num,                          // Op_RegD
 3948       R0_H_num                           // Op_RegL
 3949     };
 3950 
 3951     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 3952   %}
 3953 %}
 3954 
 3955 //----------ATTRIBUTES---------------------------------------------------------
 3956 //----------Operand Attributes-------------------------------------------------
 3957 op_attrib op_cost(1);        // Required cost attribute
 3958 
 3959 //----------Instruction Attributes---------------------------------------------
 3960 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 3961 ins_attrib ins_size(32);        // Required size attribute (in bits)
 3962 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 3963                                 // a non-matching short branch variant
 3964                                 // of some long branch?
 3965 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 3966                                 // be a power of 2) specifies the
 3967                                 // alignment that some part of the
 3968                                 // instruction (not necessarily the
 3969                                 // start) requires.  If > 1, a
 3970                                 // compute_padding() function must be
 3971                                 // provided for the instruction
 3972 
 3973 // Whether this node is expanded during code emission into a sequence of
 3974 // instructions and the first instruction can perform an implicit null check.
 3975 ins_attrib ins_is_late_expanded_null_check_candidate(false);
 3976 
 3977 //----------OPERANDS-----------------------------------------------------------
 3978 // Operand definitions must precede instruction definitions for correct parsing
 3979 // in the ADLC because operands constitute user defined types which are used in
 3980 // instruction definitions.
 3981 
 3982 //----------Simple Operands----------------------------------------------------
 3983 
 3984 // Integer operands 32 bit
 3985 // 32 bit immediate
 3986 operand immI()
 3987 %{
 3988   match(ConI);
 3989 
 3990   op_cost(0);
 3991   format %{ %}
 3992   interface(CONST_INTER);
 3993 %}
 3994 
 3995 // 32 bit zero
 3996 operand immI0()
 3997 %{
 3998   predicate(n->get_int() == 0);
 3999   match(ConI);
 4000 
 4001   op_cost(0);
 4002   format %{ %}
 4003   interface(CONST_INTER);
 4004 %}
 4005 
 4006 // 32 bit unit increment
 4007 operand immI_1()
 4008 %{
 4009   predicate(n->get_int() == 1);
 4010   match(ConI);
 4011 
 4012   op_cost(0);
 4013   format %{ %}
 4014   interface(CONST_INTER);
 4015 %}
 4016 
 4017 // 32 bit unit decrement
 4018 operand immI_M1()
 4019 %{
 4020   predicate(n->get_int() == -1);
 4021   match(ConI);
 4022 
 4023   op_cost(0);
 4024   format %{ %}
 4025   interface(CONST_INTER);
 4026 %}
 4027 
 4028 // Shift values for add/sub extension shift
 4029 operand immIExt()
 4030 %{
 4031   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4032   match(ConI);
 4033 
 4034   op_cost(0);
 4035   format %{ %}
 4036   interface(CONST_INTER);
 4037 %}
 4038 
 4039 operand immI_gt_1()
 4040 %{
 4041   predicate(n->get_int() > 1);
 4042   match(ConI);
 4043 
 4044   op_cost(0);
 4045   format %{ %}
 4046   interface(CONST_INTER);
 4047 %}
 4048 
 4049 operand immI_le_4()
 4050 %{
 4051   predicate(n->get_int() <= 4);
 4052   match(ConI);
 4053 
 4054   op_cost(0);
 4055   format %{ %}
 4056   interface(CONST_INTER);
 4057 %}
 4058 
 4059 operand immI_16()
 4060 %{
 4061   predicate(n->get_int() == 16);
 4062   match(ConI);
 4063 
 4064   op_cost(0);
 4065   format %{ %}
 4066   interface(CONST_INTER);
 4067 %}
 4068 
 4069 operand immI_24()
 4070 %{
 4071   predicate(n->get_int() == 24);
 4072   match(ConI);
 4073 
 4074   op_cost(0);
 4075   format %{ %}
 4076   interface(CONST_INTER);
 4077 %}
 4078 
 4079 operand immI_32()
 4080 %{
 4081   predicate(n->get_int() == 32);
 4082   match(ConI);
 4083 
 4084   op_cost(0);
 4085   format %{ %}
 4086   interface(CONST_INTER);
 4087 %}
 4088 
 4089 operand immI_48()
 4090 %{
 4091   predicate(n->get_int() == 48);
 4092   match(ConI);
 4093 
 4094   op_cost(0);
 4095   format %{ %}
 4096   interface(CONST_INTER);
 4097 %}
 4098 
 4099 operand immI_56()
 4100 %{
 4101   predicate(n->get_int() == 56);
 4102   match(ConI);
 4103 
 4104   op_cost(0);
 4105   format %{ %}
 4106   interface(CONST_INTER);
 4107 %}
 4108 
 4109 operand immI_255()
 4110 %{
 4111   predicate(n->get_int() == 255);
 4112   match(ConI);
 4113 
 4114   op_cost(0);
 4115   format %{ %}
 4116   interface(CONST_INTER);
 4117 %}
 4118 
 4119 operand immI_65535()
 4120 %{
 4121   predicate(n->get_int() == 65535);
 4122   match(ConI);
 4123 
 4124   op_cost(0);
 4125   format %{ %}
 4126   interface(CONST_INTER);
 4127 %}
 4128 
 4129 operand immI_positive()
 4130 %{
 4131   predicate(n->get_int() > 0);
 4132   match(ConI);
 4133 
 4134   op_cost(0);
 4135   format %{ %}
 4136   interface(CONST_INTER);
 4137 %}
 4138 
 4139 // BoolTest condition for signed compare
 4140 operand immI_cmp_cond()
 4141 %{
 4142   predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
 4143   match(ConI);
 4144 
 4145   op_cost(0);
 4146   format %{ %}
 4147   interface(CONST_INTER);
 4148 %}
 4149 
 4150 // BoolTest condition for unsigned compare
 4151 operand immI_cmpU_cond()
 4152 %{
 4153   predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
 4154   match(ConI);
 4155 
 4156   op_cost(0);
 4157   format %{ %}
 4158   interface(CONST_INTER);
 4159 %}
 4160 
 4161 operand immL_255()
 4162 %{
 4163   predicate(n->get_long() == 255L);
 4164   match(ConL);
 4165 
 4166   op_cost(0);
 4167   format %{ %}
 4168   interface(CONST_INTER);
 4169 %}
 4170 
 4171 operand immL_65535()
 4172 %{
 4173   predicate(n->get_long() == 65535L);
 4174   match(ConL);
 4175 
 4176   op_cost(0);
 4177   format %{ %}
 4178   interface(CONST_INTER);
 4179 %}
 4180 
 4181 operand immL_4294967295()
 4182 %{
 4183   predicate(n->get_long() == 4294967295L);
 4184   match(ConL);
 4185 
 4186   op_cost(0);
 4187   format %{ %}
 4188   interface(CONST_INTER);
 4189 %}
 4190 
 4191 operand immL_bitmask()
 4192 %{
 4193   predicate((n->get_long() != 0)
 4194             && ((n->get_long() & 0xc000000000000000l) == 0)
 4195             && is_power_of_2(n->get_long() + 1));
 4196   match(ConL);
 4197 
 4198   op_cost(0);
 4199   format %{ %}
 4200   interface(CONST_INTER);
 4201 %}
 4202 
 4203 operand immI_bitmask()
 4204 %{
 4205   predicate((n->get_int() != 0)
 4206             && ((n->get_int() & 0xc0000000) == 0)
 4207             && is_power_of_2(n->get_int() + 1));
 4208   match(ConI);
 4209 
 4210   op_cost(0);
 4211   format %{ %}
 4212   interface(CONST_INTER);
 4213 %}
 4214 
 4215 operand immL_positive_bitmaskI()
 4216 %{
 4217   predicate((n->get_long() != 0)
 4218             && ((julong)n->get_long() < 0x80000000ULL)
 4219             && is_power_of_2(n->get_long() + 1));
 4220   match(ConL);
 4221 
 4222   op_cost(0);
 4223   format %{ %}
 4224   interface(CONST_INTER);
 4225 %}
 4226 
 4227 // Scale values for scaled offset addressing modes (up to long but not quad)
 4228 operand immIScale()
 4229 %{
 4230   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4231   match(ConI);
 4232 
 4233   op_cost(0);
 4234   format %{ %}
 4235   interface(CONST_INTER);
 4236 %}
 4237 
 4238 // 5 bit signed integer
 4239 operand immI5()
 4240 %{
 4241   predicate(Assembler::is_simm(n->get_int(), 5));
 4242   match(ConI);
 4243 
 4244   op_cost(0);
 4245   format %{ %}
 4246   interface(CONST_INTER);
 4247 %}
 4248 
 4249 // 7 bit unsigned integer
 4250 operand immIU7()
 4251 %{
 4252   predicate(Assembler::is_uimm(n->get_int(), 7));
 4253   match(ConI);
 4254 
 4255   op_cost(0);
 4256   format %{ %}
 4257   interface(CONST_INTER);
 4258 %}
 4259 
 4260 // Offset for scaled or unscaled immediate loads and stores
 4261 operand immIOffset()
 4262 %{
 4263   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4264   match(ConI);
 4265 
 4266   op_cost(0);
 4267   format %{ %}
 4268   interface(CONST_INTER);
 4269 %}
 4270 
 4271 operand immIOffset1()
 4272 %{
 4273   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4274   match(ConI);
 4275 
 4276   op_cost(0);
 4277   format %{ %}
 4278   interface(CONST_INTER);
 4279 %}
 4280 
 4281 operand immIOffset2()
 4282 %{
 4283   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4284   match(ConI);
 4285 
 4286   op_cost(0);
 4287   format %{ %}
 4288   interface(CONST_INTER);
 4289 %}
 4290 
 4291 operand immIOffset4()
 4292 %{
 4293   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4294   match(ConI);
 4295 
 4296   op_cost(0);
 4297   format %{ %}
 4298   interface(CONST_INTER);
 4299 %}
 4300 
 4301 operand immIOffset8()
 4302 %{
 4303   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4304   match(ConI);
 4305 
 4306   op_cost(0);
 4307   format %{ %}
 4308   interface(CONST_INTER);
 4309 %}
 4310 
 4311 operand immIOffset16()
 4312 %{
 4313   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4314   match(ConI);
 4315 
 4316   op_cost(0);
 4317   format %{ %}
 4318   interface(CONST_INTER);
 4319 %}
 4320 
 4321 operand immLOffset()
 4322 %{
 4323   predicate(n->get_long() >= -256 && n->get_long() <= 65520);
 4324   match(ConL);
 4325 
 4326   op_cost(0);
 4327   format %{ %}
 4328   interface(CONST_INTER);
 4329 %}
 4330 
 4331 operand immLoffset1()
 4332 %{
 4333   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4334   match(ConL);
 4335 
 4336   op_cost(0);
 4337   format %{ %}
 4338   interface(CONST_INTER);
 4339 %}
 4340 
 4341 operand immLoffset2()
 4342 %{
 4343   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4344   match(ConL);
 4345 
 4346   op_cost(0);
 4347   format %{ %}
 4348   interface(CONST_INTER);
 4349 %}
 4350 
 4351 operand immLoffset4()
 4352 %{
 4353   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4354   match(ConL);
 4355 
 4356   op_cost(0);
 4357   format %{ %}
 4358   interface(CONST_INTER);
 4359 %}
 4360 
 4361 operand immLoffset8()
 4362 %{
 4363   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4364   match(ConL);
 4365 
 4366   op_cost(0);
 4367   format %{ %}
 4368   interface(CONST_INTER);
 4369 %}
 4370 
 4371 operand immLoffset16()
 4372 %{
 4373   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4374   match(ConL);
 4375 
 4376   op_cost(0);
 4377   format %{ %}
 4378   interface(CONST_INTER);
 4379 %}
 4380 
 4381 // 5 bit signed long integer
 4382 operand immL5()
 4383 %{
 4384   predicate(Assembler::is_simm(n->get_long(), 5));
 4385   match(ConL);
 4386 
 4387   op_cost(0);
 4388   format %{ %}
 4389   interface(CONST_INTER);
 4390 %}
 4391 
 4392 // 7 bit unsigned long integer
 4393 operand immLU7()
 4394 %{
 4395   predicate(Assembler::is_uimm(n->get_long(), 7));
 4396   match(ConL);
 4397 
 4398   op_cost(0);
 4399   format %{ %}
 4400   interface(CONST_INTER);
 4401 %}
 4402 
 4403 // 8 bit signed value.
 4404 operand immI8()
 4405 %{
 4406   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4407   match(ConI);
 4408 
 4409   op_cost(0);
 4410   format %{ %}
 4411   interface(CONST_INTER);
 4412 %}
 4413 
 4414 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4415 operand immIDupV()
 4416 %{
 4417   predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
 4418   match(ConI);
 4419 
 4420   op_cost(0);
 4421   format %{ %}
 4422   interface(CONST_INTER);
 4423 %}
 4424 
 4425 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4426 operand immLDupV()
 4427 %{
 4428   predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
 4429   match(ConL);
 4430 
 4431   op_cost(0);
 4432   format %{ %}
 4433   interface(CONST_INTER);
 4434 %}
 4435 
 4436 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4437 operand immHDupV()
 4438 %{
 4439   predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
 4440   match(ConH);
 4441 
 4442   op_cost(0);
 4443   format %{ %}
 4444   interface(CONST_INTER);
 4445 %}
 4446 
 4447 // 8 bit integer valid for vector add sub immediate
 4448 operand immBAddSubV()
 4449 %{
 4450   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4451   match(ConI);
 4452 
 4453   op_cost(0);
 4454   format %{ %}
 4455   interface(CONST_INTER);
 4456 %}
 4457 
 4458 // 32 bit integer valid for add sub immediate
 4459 operand immIAddSub()
 4460 %{
 4461   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4462   match(ConI);
 4463   op_cost(0);
 4464   format %{ %}
 4465   interface(CONST_INTER);
 4466 %}
 4467 
 4468 // 32 bit integer valid for vector add sub immediate
 4469 operand immIAddSubV()
 4470 %{
 4471   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4472   match(ConI);
 4473 
 4474   op_cost(0);
 4475   format %{ %}
 4476   interface(CONST_INTER);
 4477 %}
 4478 
 4479 // 32 bit unsigned integer valid for logical immediate
 4480 
 4481 operand immBLog()
 4482 %{
 4483   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4484   match(ConI);
 4485 
 4486   op_cost(0);
 4487   format %{ %}
 4488   interface(CONST_INTER);
 4489 %}
 4490 
 4491 operand immSLog()
 4492 %{
 4493   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4494   match(ConI);
 4495 
 4496   op_cost(0);
 4497   format %{ %}
 4498   interface(CONST_INTER);
 4499 %}
 4500 
 4501 operand immILog()
 4502 %{
 4503   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4504   match(ConI);
 4505 
 4506   op_cost(0);
 4507   format %{ %}
 4508   interface(CONST_INTER);
 4509 %}
 4510 
 4511 // Integer operands 64 bit
 4512 // 64 bit immediate
 4513 operand immL()
 4514 %{
 4515   match(ConL);
 4516 
 4517   op_cost(0);
 4518   format %{ %}
 4519   interface(CONST_INTER);
 4520 %}
 4521 
 4522 // 64 bit zero
 4523 operand immL0()
 4524 %{
 4525   predicate(n->get_long() == 0);
 4526   match(ConL);
 4527 
 4528   op_cost(0);
 4529   format %{ %}
 4530   interface(CONST_INTER);
 4531 %}
 4532 
 4533 // 64 bit unit decrement
 4534 operand immL_M1()
 4535 %{
 4536   predicate(n->get_long() == -1);
 4537   match(ConL);
 4538 
 4539   op_cost(0);
 4540   format %{ %}
 4541   interface(CONST_INTER);
 4542 %}
 4543 
 4544 // 64 bit integer valid for add sub immediate
 4545 operand immLAddSub()
 4546 %{
 4547   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4548   match(ConL);
 4549   op_cost(0);
 4550   format %{ %}
 4551   interface(CONST_INTER);
 4552 %}
 4553 
 4554 // 64 bit integer valid for addv subv immediate
 4555 operand immLAddSubV()
 4556 %{
 4557   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4558   match(ConL);
 4559 
 4560   op_cost(0);
 4561   format %{ %}
 4562   interface(CONST_INTER);
 4563 %}
 4564 
 4565 // 64 bit integer valid for logical immediate
 4566 operand immLLog()
 4567 %{
 4568   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4569   match(ConL);
 4570   op_cost(0);
 4571   format %{ %}
 4572   interface(CONST_INTER);
 4573 %}
 4574 
 4575 // Long Immediate: low 32-bit mask
 4576 operand immL_32bits()
 4577 %{
 4578   predicate(n->get_long() == 0xFFFFFFFFL);
 4579   match(ConL);
 4580   op_cost(0);
 4581   format %{ %}
 4582   interface(CONST_INTER);
 4583 %}
 4584 
 4585 // Pointer operands
 4586 // Pointer Immediate
 4587 operand immP()
 4588 %{
 4589   match(ConP);
 4590 
 4591   op_cost(0);
 4592   format %{ %}
 4593   interface(CONST_INTER);
 4594 %}
 4595 
 4596 // nullptr Pointer Immediate
 4597 operand immP0()
 4598 %{
 4599   predicate(n->get_ptr() == 0);
 4600   match(ConP);
 4601 
 4602   op_cost(0);
 4603   format %{ %}
 4604   interface(CONST_INTER);
 4605 %}
 4606 
 4607 // Pointer Immediate One
 4608 // this is used in object initialization (initial object header)
 4609 operand immP_1()
 4610 %{
 4611   predicate(n->get_ptr() == 1);
 4612   match(ConP);
 4613 
 4614   op_cost(0);
 4615   format %{ %}
 4616   interface(CONST_INTER);
 4617 %}
 4618 
 4619 // Card Table Byte Map Base
 4620 operand immByteMapBase()
 4621 %{
 4622   // Get base of card map
 4623   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4624             is_card_table_address((address)(n->get_ptr())));
 4625   match(ConP);
 4626 
 4627   op_cost(0);
 4628   format %{ %}
 4629   interface(CONST_INTER);
 4630 %}
 4631 
 4632 // AOT Runtime Constants Address
 4633 operand immAOTRuntimeConstantsAddress()
 4634 %{
 4635   // Check if the address is in the range of AOT Runtime Constants
 4636   predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
 4637   match(ConP);
 4638 
 4639   op_cost(0);
 4640   format %{ %}
 4641   interface(CONST_INTER);
 4642 %}
 4643 
 4644 // Float and Double operands
 4645 // Double Immediate
 4646 operand immD()
 4647 %{
 4648   match(ConD);
 4649   op_cost(0);
 4650   format %{ %}
 4651   interface(CONST_INTER);
 4652 %}
 4653 
 4654 // Double Immediate: +0.0d
 4655 operand immD0()
 4656 %{
 4657   predicate(jlong_cast(n->getd()) == 0);
 4658   match(ConD);
 4659 
 4660   op_cost(0);
 4661   format %{ %}
 4662   interface(CONST_INTER);
 4663 %}
 4664 
 4665 // constant 'double +0.0'.
 4666 operand immDPacked()
 4667 %{
 4668   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4669   match(ConD);
 4670   op_cost(0);
 4671   format %{ %}
 4672   interface(CONST_INTER);
 4673 %}
 4674 
 4675 // Float Immediate
 4676 operand immF()
 4677 %{
 4678   match(ConF);
 4679   op_cost(0);
 4680   format %{ %}
 4681   interface(CONST_INTER);
 4682 %}
 4683 
 4684 // Float Immediate: +0.0f.
 4685 operand immF0()
 4686 %{
 4687   predicate(jint_cast(n->getf()) == 0);
 4688   match(ConF);
 4689 
 4690   op_cost(0);
 4691   format %{ %}
 4692   interface(CONST_INTER);
 4693 %}
 4694 
 4695 // Half Float (FP16) Immediate
 4696 operand immH()
 4697 %{
 4698   match(ConH);
 4699   op_cost(0);
 4700   format %{ %}
 4701   interface(CONST_INTER);
 4702 %}
 4703 
 4704 //
 4705 operand immFPacked()
 4706 %{
 4707   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4708   match(ConF);
 4709   op_cost(0);
 4710   format %{ %}
 4711   interface(CONST_INTER);
 4712 %}
 4713 
 4714 // Narrow pointer operands
 4715 // Narrow Pointer Immediate
 4716 operand immN()
 4717 %{
 4718   match(ConN);
 4719 
 4720   op_cost(0);
 4721   format %{ %}
 4722   interface(CONST_INTER);
 4723 %}
 4724 
 4725 // Narrow nullptr Pointer Immediate
 4726 operand immN0()
 4727 %{
 4728   predicate(n->get_narrowcon() == 0);
 4729   match(ConN);
 4730 
 4731   op_cost(0);
 4732   format %{ %}
 4733   interface(CONST_INTER);
 4734 %}
 4735 
 4736 operand immNKlass()
 4737 %{
 4738   match(ConNKlass);
 4739 
 4740   op_cost(0);
 4741   format %{ %}
 4742   interface(CONST_INTER);
 4743 %}
 4744 
 4745 // Integer 32 bit Register Operands
 4746 // Integer 32 bitRegister (excludes SP)
 4747 operand iRegI()
 4748 %{
 4749   constraint(ALLOC_IN_RC(any_reg32));
 4750   match(RegI);
 4751   match(iRegINoSp);
 4752   op_cost(0);
 4753   format %{ %}
 4754   interface(REG_INTER);
 4755 %}
 4756 
 4757 // Integer 32 bit Register not Special
 4758 operand iRegINoSp()
 4759 %{
 4760   constraint(ALLOC_IN_RC(no_special_reg32));
 4761   match(RegI);
 4762   op_cost(0);
 4763   format %{ %}
 4764   interface(REG_INTER);
 4765 %}
 4766 
 4767 // Integer 64 bit Register Operands
 4768 // Integer 64 bit Register (includes SP)
 4769 operand iRegL()
 4770 %{
 4771   constraint(ALLOC_IN_RC(any_reg));
 4772   match(RegL);
 4773   match(iRegLNoSp);
 4774   op_cost(0);
 4775   format %{ %}
 4776   interface(REG_INTER);
 4777 %}
 4778 
 4779 // Integer 64 bit Register not Special
 4780 operand iRegLNoSp()
 4781 %{
 4782   constraint(ALLOC_IN_RC(no_special_reg));
 4783   match(RegL);
 4784   match(iRegL_R0);
 4785   format %{ %}
 4786   interface(REG_INTER);
 4787 %}
 4788 
 4789 // Pointer Register Operands
 4790 // Pointer Register
 4791 operand iRegP()
 4792 %{
 4793   constraint(ALLOC_IN_RC(ptr_reg));
 4794   match(RegP);
 4795   match(iRegPNoSp);
 4796   match(iRegP_R0);
 4797   //match(iRegP_R2);
 4798   //match(iRegP_R4);
 4799   match(iRegP_R5);
 4800   match(thread_RegP);
 4801   op_cost(0);
 4802   format %{ %}
 4803   interface(REG_INTER);
 4804 %}
 4805 
 4806 // Pointer 64 bit Register not Special
 4807 operand iRegPNoSp()
 4808 %{
 4809   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4810   match(RegP);
 4811   // match(iRegP);
 4812   // match(iRegP_R0);
 4813   // match(iRegP_R2);
 4814   // match(iRegP_R4);
 4815   // match(iRegP_R5);
 4816   // match(thread_RegP);
 4817   op_cost(0);
 4818   format %{ %}
 4819   interface(REG_INTER);
 4820 %}
 4821 
 4822 // This operand is not allowed to use rfp even if
 4823 // rfp is not used to hold the frame pointer.
 4824 operand iRegPNoSpNoRfp()
 4825 %{
 4826   constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
 4827   match(RegP);
 4828   match(iRegPNoSp);
 4829   op_cost(0);
 4830   format %{ %}
 4831   interface(REG_INTER);
 4832 %}
 4833 
 4834 // Pointer 64 bit Register R0 only
 4835 operand iRegP_R0()
 4836 %{
 4837   constraint(ALLOC_IN_RC(r0_reg));
 4838   match(RegP);
 4839   // match(iRegP);
 4840   match(iRegPNoSp);
 4841   op_cost(0);
 4842   format %{ %}
 4843   interface(REG_INTER);
 4844 %}
 4845 
 4846 // Pointer 64 bit Register R1 only
 4847 operand iRegP_R1()
 4848 %{
 4849   constraint(ALLOC_IN_RC(r1_reg));
 4850   match(RegP);
 4851   // match(iRegP);
 4852   match(iRegPNoSp);
 4853   op_cost(0);
 4854   format %{ %}
 4855   interface(REG_INTER);
 4856 %}
 4857 
 4858 // Pointer 64 bit Register R2 only
 4859 operand iRegP_R2()
 4860 %{
 4861   constraint(ALLOC_IN_RC(r2_reg));
 4862   match(RegP);
 4863   // match(iRegP);
 4864   match(iRegPNoSp);
 4865   op_cost(0);
 4866   format %{ %}
 4867   interface(REG_INTER);
 4868 %}
 4869 
 4870 // Pointer 64 bit Register R3 only
 4871 operand iRegP_R3()
 4872 %{
 4873   constraint(ALLOC_IN_RC(r3_reg));
 4874   match(RegP);
 4875   // match(iRegP);
 4876   match(iRegPNoSp);
 4877   op_cost(0);
 4878   format %{ %}
 4879   interface(REG_INTER);
 4880 %}
 4881 
 4882 // Pointer 64 bit Register R4 only
 4883 operand iRegP_R4()
 4884 %{
 4885   constraint(ALLOC_IN_RC(r4_reg));
 4886   match(RegP);
 4887   // match(iRegP);
 4888   match(iRegPNoSp);
 4889   op_cost(0);
 4890   format %{ %}
 4891   interface(REG_INTER);
 4892 %}
 4893 
 4894 // Pointer 64 bit Register R5 only
 4895 operand iRegP_R5()
 4896 %{
 4897   constraint(ALLOC_IN_RC(r5_reg));
 4898   match(RegP);
 4899   // match(iRegP);
 4900   match(iRegPNoSp);
 4901   op_cost(0);
 4902   format %{ %}
 4903   interface(REG_INTER);
 4904 %}
 4905 
 4906 // Pointer 64 bit Register R10 only
 4907 operand iRegP_R10()
 4908 %{
 4909   constraint(ALLOC_IN_RC(r10_reg));
 4910   match(RegP);
 4911   // match(iRegP);
 4912   match(iRegPNoSp);
 4913   op_cost(0);
 4914   format %{ %}
 4915   interface(REG_INTER);
 4916 %}
 4917 
 4918 // Long 64 bit Register R0 only
 4919 operand iRegL_R0()
 4920 %{
 4921   constraint(ALLOC_IN_RC(r0_reg));
 4922   match(RegL);
 4923   match(iRegLNoSp);
 4924   op_cost(0);
 4925   format %{ %}
 4926   interface(REG_INTER);
 4927 %}
 4928 
 4929 // Long 64 bit Register R11 only
 4930 operand iRegL_R11()
 4931 %{
 4932   constraint(ALLOC_IN_RC(r11_reg));
 4933   match(RegL);
 4934   match(iRegLNoSp);
 4935   op_cost(0);
 4936   format %{ %}
 4937   interface(REG_INTER);
 4938 %}
 4939 
 4940 // Register R0 only
 4941 operand iRegI_R0()
 4942 %{
 4943   constraint(ALLOC_IN_RC(int_r0_reg));
 4944   match(RegI);
 4945   match(iRegINoSp);
 4946   op_cost(0);
 4947   format %{ %}
 4948   interface(REG_INTER);
 4949 %}
 4950 
 4951 // Register R2 only
 4952 operand iRegI_R2()
 4953 %{
 4954   constraint(ALLOC_IN_RC(int_r2_reg));
 4955   match(RegI);
 4956   match(iRegINoSp);
 4957   op_cost(0);
 4958   format %{ %}
 4959   interface(REG_INTER);
 4960 %}
 4961 
 4962 // Register R3 only
 4963 operand iRegI_R3()
 4964 %{
 4965   constraint(ALLOC_IN_RC(int_r3_reg));
 4966   match(RegI);
 4967   match(iRegINoSp);
 4968   op_cost(0);
 4969   format %{ %}
 4970   interface(REG_INTER);
 4971 %}
 4972 
 4973 
 4974 // Register R4 only
 4975 operand iRegI_R4()
 4976 %{
 4977   constraint(ALLOC_IN_RC(int_r4_reg));
 4978   match(RegI);
 4979   match(iRegINoSp);
 4980   op_cost(0);
 4981   format %{ %}
 4982   interface(REG_INTER);
 4983 %}
 4984 
 4985 
 4986 // Pointer Register Operands
 4987 // Narrow Pointer Register
 4988 operand iRegN()
 4989 %{
 4990   constraint(ALLOC_IN_RC(any_reg32));
 4991   match(RegN);
 4992   match(iRegNNoSp);
 4993   op_cost(0);
 4994   format %{ %}
 4995   interface(REG_INTER);
 4996 %}
 4997 
 4998 // Integer 64 bit Register not Special
 4999 operand iRegNNoSp()
 5000 %{
 5001   constraint(ALLOC_IN_RC(no_special_reg32));
 5002   match(RegN);
 5003   op_cost(0);
 5004   format %{ %}
 5005   interface(REG_INTER);
 5006 %}
 5007 
 5008 // Float Register
 5009 // Float register operands
 5010 operand vRegF()
 5011 %{
 5012   constraint(ALLOC_IN_RC(float_reg));
 5013   match(RegF);
 5014 
 5015   op_cost(0);
 5016   format %{ %}
 5017   interface(REG_INTER);
 5018 %}
 5019 
 5020 // Double Register
 5021 // Double register operands
 5022 operand vRegD()
 5023 %{
 5024   constraint(ALLOC_IN_RC(double_reg));
 5025   match(RegD);
 5026 
 5027   op_cost(0);
 5028   format %{ %}
 5029   interface(REG_INTER);
 5030 %}
 5031 
 5032 // Generic vector class. This will be used for
 5033 // all vector operands, including NEON and SVE.
 5034 operand vReg()
 5035 %{
 5036   constraint(ALLOC_IN_RC(dynamic));
 5037   match(VecA);
 5038   match(VecD);
 5039   match(VecX);
 5040 
 5041   op_cost(0);
 5042   format %{ %}
 5043   interface(REG_INTER);
 5044 %}
 5045 
 5046 operand vReg_V10()
 5047 %{
 5048   constraint(ALLOC_IN_RC(v10_veca_reg));
 5049   match(vReg);
 5050 
 5051   op_cost(0);
 5052   format %{ %}
 5053   interface(REG_INTER);
 5054 %}
 5055 
 5056 operand vReg_V11()
 5057 %{
 5058   constraint(ALLOC_IN_RC(v11_veca_reg));
 5059   match(vReg);
 5060 
 5061   op_cost(0);
 5062   format %{ %}
 5063   interface(REG_INTER);
 5064 %}
 5065 
 5066 operand vReg_V12()
 5067 %{
 5068   constraint(ALLOC_IN_RC(v12_veca_reg));
 5069   match(vReg);
 5070 
 5071   op_cost(0);
 5072   format %{ %}
 5073   interface(REG_INTER);
 5074 %}
 5075 
 5076 operand vReg_V13()
 5077 %{
 5078   constraint(ALLOC_IN_RC(v13_veca_reg));
 5079   match(vReg);
 5080 
 5081   op_cost(0);
 5082   format %{ %}
 5083   interface(REG_INTER);
 5084 %}
 5085 
 5086 operand vReg_V17()
 5087 %{
 5088   constraint(ALLOC_IN_RC(v17_veca_reg));
 5089   match(vReg);
 5090 
 5091   op_cost(0);
 5092   format %{ %}
 5093   interface(REG_INTER);
 5094 %}
 5095 
 5096 operand vReg_V18()
 5097 %{
 5098   constraint(ALLOC_IN_RC(v18_veca_reg));
 5099   match(vReg);
 5100 
 5101   op_cost(0);
 5102   format %{ %}
 5103   interface(REG_INTER);
 5104 %}
 5105 
 5106 operand vReg_V23()
 5107 %{
 5108   constraint(ALLOC_IN_RC(v23_veca_reg));
 5109   match(vReg);
 5110 
 5111   op_cost(0);
 5112   format %{ %}
 5113   interface(REG_INTER);
 5114 %}
 5115 
 5116 operand vReg_V24()
 5117 %{
 5118   constraint(ALLOC_IN_RC(v24_veca_reg));
 5119   match(vReg);
 5120 
 5121   op_cost(0);
 5122   format %{ %}
 5123   interface(REG_INTER);
 5124 %}
 5125 
 5126 operand vecA()
 5127 %{
 5128   constraint(ALLOC_IN_RC(vectora_reg));
 5129   match(VecA);
 5130 
 5131   op_cost(0);
 5132   format %{ %}
 5133   interface(REG_INTER);
 5134 %}
 5135 
 5136 operand vecD()
 5137 %{
 5138   constraint(ALLOC_IN_RC(vectord_reg));
 5139   match(VecD);
 5140 
 5141   op_cost(0);
 5142   format %{ %}
 5143   interface(REG_INTER);
 5144 %}
 5145 
 5146 operand vecX()
 5147 %{
 5148   constraint(ALLOC_IN_RC(vectorx_reg));
 5149   match(VecX);
 5150 
 5151   op_cost(0);
 5152   format %{ %}
 5153   interface(REG_INTER);
 5154 %}
 5155 
 5156 operand vRegD_V0()
 5157 %{
 5158   constraint(ALLOC_IN_RC(v0_reg));
 5159   match(RegD);
 5160   op_cost(0);
 5161   format %{ %}
 5162   interface(REG_INTER);
 5163 %}
 5164 
 5165 operand vRegD_V1()
 5166 %{
 5167   constraint(ALLOC_IN_RC(v1_reg));
 5168   match(RegD);
 5169   op_cost(0);
 5170   format %{ %}
 5171   interface(REG_INTER);
 5172 %}
 5173 
 5174 operand vRegD_V2()
 5175 %{
 5176   constraint(ALLOC_IN_RC(v2_reg));
 5177   match(RegD);
 5178   op_cost(0);
 5179   format %{ %}
 5180   interface(REG_INTER);
 5181 %}
 5182 
 5183 operand vRegD_V3()
 5184 %{
 5185   constraint(ALLOC_IN_RC(v3_reg));
 5186   match(RegD);
 5187   op_cost(0);
 5188   format %{ %}
 5189   interface(REG_INTER);
 5190 %}
 5191 
 5192 operand vRegD_V4()
 5193 %{
 5194   constraint(ALLOC_IN_RC(v4_reg));
 5195   match(RegD);
 5196   op_cost(0);
 5197   format %{ %}
 5198   interface(REG_INTER);
 5199 %}
 5200 
 5201 operand vRegD_V5()
 5202 %{
 5203   constraint(ALLOC_IN_RC(v5_reg));
 5204   match(RegD);
 5205   op_cost(0);
 5206   format %{ %}
 5207   interface(REG_INTER);
 5208 %}
 5209 
 5210 operand vRegD_V6()
 5211 %{
 5212   constraint(ALLOC_IN_RC(v6_reg));
 5213   match(RegD);
 5214   op_cost(0);
 5215   format %{ %}
 5216   interface(REG_INTER);
 5217 %}
 5218 
 5219 operand vRegD_V7()
 5220 %{
 5221   constraint(ALLOC_IN_RC(v7_reg));
 5222   match(RegD);
 5223   op_cost(0);
 5224   format %{ %}
 5225   interface(REG_INTER);
 5226 %}
 5227 
 5228 operand vRegD_V12()
 5229 %{
 5230   constraint(ALLOC_IN_RC(v12_reg));
 5231   match(RegD);
 5232   op_cost(0);
 5233   format %{ %}
 5234   interface(REG_INTER);
 5235 %}
 5236 
 5237 operand vRegD_V13()
 5238 %{
 5239   constraint(ALLOC_IN_RC(v13_reg));
 5240   match(RegD);
 5241   op_cost(0);
 5242   format %{ %}
 5243   interface(REG_INTER);
 5244 %}
 5245 
 5246 operand pReg()
 5247 %{
 5248   constraint(ALLOC_IN_RC(pr_reg));
 5249   match(RegVectMask);
 5250   match(pRegGov);
 5251   op_cost(0);
 5252   format %{ %}
 5253   interface(REG_INTER);
 5254 %}
 5255 
 5256 operand pRegGov()
 5257 %{
 5258   constraint(ALLOC_IN_RC(gov_pr));
 5259   match(RegVectMask);
 5260   match(pReg);
 5261   op_cost(0);
 5262   format %{ %}
 5263   interface(REG_INTER);
 5264 %}
 5265 
 5266 operand pRegGov_P0()
 5267 %{
 5268   constraint(ALLOC_IN_RC(p0_reg));
 5269   match(RegVectMask);
 5270   op_cost(0);
 5271   format %{ %}
 5272   interface(REG_INTER);
 5273 %}
 5274 
 5275 operand pRegGov_P1()
 5276 %{
 5277   constraint(ALLOC_IN_RC(p1_reg));
 5278   match(RegVectMask);
 5279   op_cost(0);
 5280   format %{ %}
 5281   interface(REG_INTER);
 5282 %}
 5283 
 5284 // Flags register, used as output of signed compare instructions
 5285 
 5286 // note that on AArch64 we also use this register as the output for
 5287 // for floating point compare instructions (CmpF CmpD). this ensures
 5288 // that ordered inequality tests use GT, GE, LT or LE none of which
 5289 // pass through cases where the result is unordered i.e. one or both
 5290 // inputs to the compare is a NaN. this means that the ideal code can
 5291 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5292 // (where the comparison should always fail). EQ and NE tests are
 5293 // always generated in ideal code so that unordered folds into the NE
 5294 // case, matching the behaviour of AArch64 NE.
 5295 //
 5296 // This differs from x86 where the outputs of FP compares use a
 5297 // special FP flags registers and where compares based on this
 5298 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5299 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5300 // to explicitly handle the unordered case in branches. x86 also has
 5301 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5302 
 5303 operand rFlagsReg()
 5304 %{
 5305   constraint(ALLOC_IN_RC(int_flags));
 5306   match(RegFlags);
 5307 
 5308   op_cost(0);
 5309   format %{ "RFLAGS" %}
 5310   interface(REG_INTER);
 5311 %}
 5312 
 5313 // Flags register, used as output of unsigned compare instructions
 5314 operand rFlagsRegU()
 5315 %{
 5316   constraint(ALLOC_IN_RC(int_flags));
 5317   match(RegFlags);
 5318 
 5319   op_cost(0);
 5320   format %{ "RFLAGSU" %}
 5321   interface(REG_INTER);
 5322 %}
 5323 
 5324 // Special Registers
 5325 
 5326 // Method Register
 5327 operand inline_cache_RegP(iRegP reg)
 5328 %{
 5329   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5330   match(reg);
 5331   match(iRegPNoSp);
 5332   op_cost(0);
 5333   format %{ %}
 5334   interface(REG_INTER);
 5335 %}
 5336 
 5337 // Thread Register
 5338 operand thread_RegP(iRegP reg)
 5339 %{
 5340   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5341   match(reg);
 5342   op_cost(0);
 5343   format %{ %}
 5344   interface(REG_INTER);
 5345 %}
 5346 
 5347 //----------Memory Operands----------------------------------------------------
 5348 
 5349 operand indirect(iRegP reg)
 5350 %{
 5351   constraint(ALLOC_IN_RC(ptr_reg));
 5352   match(reg);
 5353   op_cost(0);
 5354   format %{ "[$reg]" %}
 5355   interface(MEMORY_INTER) %{
 5356     base($reg);
 5357     index(0xffffffff);
 5358     scale(0x0);
 5359     disp(0x0);
 5360   %}
 5361 %}
 5362 
 5363 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5364 %{
 5365   constraint(ALLOC_IN_RC(ptr_reg));
 5366   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5367   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5368   op_cost(0);
 5369   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5370   interface(MEMORY_INTER) %{
 5371     base($reg);
 5372     index($ireg);
 5373     scale($scale);
 5374     disp(0x0);
 5375   %}
 5376 %}
 5377 
 5378 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5379 %{
 5380   constraint(ALLOC_IN_RC(ptr_reg));
 5381   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5382   match(AddP reg (LShiftL lreg scale));
 5383   op_cost(0);
 5384   format %{ "$reg, $lreg lsl($scale)" %}
 5385   interface(MEMORY_INTER) %{
 5386     base($reg);
 5387     index($lreg);
 5388     scale($scale);
 5389     disp(0x0);
 5390   %}
 5391 %}
 5392 
 5393 operand indIndexI2L(iRegP reg, iRegI ireg)
 5394 %{
 5395   constraint(ALLOC_IN_RC(ptr_reg));
 5396   match(AddP reg (ConvI2L ireg));
 5397   op_cost(0);
 5398   format %{ "$reg, $ireg, 0, I2L" %}
 5399   interface(MEMORY_INTER) %{
 5400     base($reg);
 5401     index($ireg);
 5402     scale(0x0);
 5403     disp(0x0);
 5404   %}
 5405 %}
 5406 
 5407 operand indIndex(iRegP reg, iRegL lreg)
 5408 %{
 5409   constraint(ALLOC_IN_RC(ptr_reg));
 5410   match(AddP reg lreg);
 5411   op_cost(0);
 5412   format %{ "$reg, $lreg" %}
 5413   interface(MEMORY_INTER) %{
 5414     base($reg);
 5415     index($lreg);
 5416     scale(0x0);
 5417     disp(0x0);
 5418   %}
 5419 %}
 5420 
 5421 operand indOffI1(iRegP reg, immIOffset1 off)
 5422 %{
 5423   constraint(ALLOC_IN_RC(ptr_reg));
 5424   match(AddP reg off);
 5425   op_cost(0);
 5426   format %{ "[$reg, $off]" %}
 5427   interface(MEMORY_INTER) %{
 5428     base($reg);
 5429     index(0xffffffff);
 5430     scale(0x0);
 5431     disp($off);
 5432   %}
 5433 %}
 5434 
 5435 operand indOffI2(iRegP reg, immIOffset2 off)
 5436 %{
 5437   constraint(ALLOC_IN_RC(ptr_reg));
 5438   match(AddP reg off);
 5439   op_cost(0);
 5440   format %{ "[$reg, $off]" %}
 5441   interface(MEMORY_INTER) %{
 5442     base($reg);
 5443     index(0xffffffff);
 5444     scale(0x0);
 5445     disp($off);
 5446   %}
 5447 %}
 5448 
 5449 operand indOffI4(iRegP reg, immIOffset4 off)
 5450 %{
 5451   constraint(ALLOC_IN_RC(ptr_reg));
 5452   match(AddP reg off);
 5453   op_cost(0);
 5454   format %{ "[$reg, $off]" %}
 5455   interface(MEMORY_INTER) %{
 5456     base($reg);
 5457     index(0xffffffff);
 5458     scale(0x0);
 5459     disp($off);
 5460   %}
 5461 %}
 5462 
 5463 operand indOffI8(iRegP reg, immIOffset8 off)
 5464 %{
 5465   constraint(ALLOC_IN_RC(ptr_reg));
 5466   match(AddP reg off);
 5467   op_cost(0);
 5468   format %{ "[$reg, $off]" %}
 5469   interface(MEMORY_INTER) %{
 5470     base($reg);
 5471     index(0xffffffff);
 5472     scale(0x0);
 5473     disp($off);
 5474   %}
 5475 %}
 5476 
 5477 operand indOffI16(iRegP reg, immIOffset16 off)
 5478 %{
 5479   constraint(ALLOC_IN_RC(ptr_reg));
 5480   match(AddP reg off);
 5481   op_cost(0);
 5482   format %{ "[$reg, $off]" %}
 5483   interface(MEMORY_INTER) %{
 5484     base($reg);
 5485     index(0xffffffff);
 5486     scale(0x0);
 5487     disp($off);
 5488   %}
 5489 %}
 5490 
 5491 operand indOffL1(iRegP reg, immLoffset1 off)
 5492 %{
 5493   constraint(ALLOC_IN_RC(ptr_reg));
 5494   match(AddP reg off);
 5495   op_cost(0);
 5496   format %{ "[$reg, $off]" %}
 5497   interface(MEMORY_INTER) %{
 5498     base($reg);
 5499     index(0xffffffff);
 5500     scale(0x0);
 5501     disp($off);
 5502   %}
 5503 %}
 5504 
 5505 operand indOffL2(iRegP reg, immLoffset2 off)
 5506 %{
 5507   constraint(ALLOC_IN_RC(ptr_reg));
 5508   match(AddP reg off);
 5509   op_cost(0);
 5510   format %{ "[$reg, $off]" %}
 5511   interface(MEMORY_INTER) %{
 5512     base($reg);
 5513     index(0xffffffff);
 5514     scale(0x0);
 5515     disp($off);
 5516   %}
 5517 %}
 5518 
 5519 operand indOffL4(iRegP reg, immLoffset4 off)
 5520 %{
 5521   constraint(ALLOC_IN_RC(ptr_reg));
 5522   match(AddP reg off);
 5523   op_cost(0);
 5524   format %{ "[$reg, $off]" %}
 5525   interface(MEMORY_INTER) %{
 5526     base($reg);
 5527     index(0xffffffff);
 5528     scale(0x0);
 5529     disp($off);
 5530   %}
 5531 %}
 5532 
 5533 operand indOffL8(iRegP reg, immLoffset8 off)
 5534 %{
 5535   constraint(ALLOC_IN_RC(ptr_reg));
 5536   match(AddP reg off);
 5537   op_cost(0);
 5538   format %{ "[$reg, $off]" %}
 5539   interface(MEMORY_INTER) %{
 5540     base($reg);
 5541     index(0xffffffff);
 5542     scale(0x0);
 5543     disp($off);
 5544   %}
 5545 %}
 5546 
 5547 operand indOffL16(iRegP reg, immLoffset16 off)
 5548 %{
 5549   constraint(ALLOC_IN_RC(ptr_reg));
 5550   match(AddP reg off);
 5551   op_cost(0);
 5552   format %{ "[$reg, $off]" %}
 5553   interface(MEMORY_INTER) %{
 5554     base($reg);
 5555     index(0xffffffff);
 5556     scale(0x0);
 5557     disp($off);
 5558   %}
 5559 %}
 5560 
 5561 operand indirectX2P(iRegL reg)
 5562 %{
 5563   constraint(ALLOC_IN_RC(ptr_reg));
 5564   match(CastX2P reg);
 5565   op_cost(0);
 5566   format %{ "[$reg]\t# long -> ptr" %}
 5567   interface(MEMORY_INTER) %{
 5568     base($reg);
 5569     index(0xffffffff);
 5570     scale(0x0);
 5571     disp(0x0);
 5572   %}
 5573 %}
 5574 
 5575 operand indOffX2P(iRegL reg, immLOffset off)
 5576 %{
 5577   constraint(ALLOC_IN_RC(ptr_reg));
 5578   match(AddP (CastX2P reg) off);
 5579   op_cost(0);
 5580   format %{ "[$reg, $off]\t# long -> ptr" %}
 5581   interface(MEMORY_INTER) %{
 5582     base($reg);
 5583     index(0xffffffff);
 5584     scale(0x0);
 5585     disp($off);
 5586   %}
 5587 %}
 5588 
 5589 operand indirectN(iRegN reg)
 5590 %{
 5591   predicate(CompressedOops::shift() == 0);
 5592   constraint(ALLOC_IN_RC(ptr_reg));
 5593   match(DecodeN reg);
 5594   op_cost(0);
 5595   format %{ "[$reg]\t# narrow" %}
 5596   interface(MEMORY_INTER) %{
 5597     base($reg);
 5598     index(0xffffffff);
 5599     scale(0x0);
 5600     disp(0x0);
 5601   %}
 5602 %}
 5603 
 5604 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5605 %{
 5606   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5607   constraint(ALLOC_IN_RC(ptr_reg));
 5608   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5609   op_cost(0);
 5610   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5611   interface(MEMORY_INTER) %{
 5612     base($reg);
 5613     index($ireg);
 5614     scale($scale);
 5615     disp(0x0);
 5616   %}
 5617 %}
 5618 
 5619 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5620 %{
 5621   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5622   constraint(ALLOC_IN_RC(ptr_reg));
 5623   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5624   op_cost(0);
 5625   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5626   interface(MEMORY_INTER) %{
 5627     base($reg);
 5628     index($lreg);
 5629     scale($scale);
 5630     disp(0x0);
 5631   %}
 5632 %}
 5633 
 5634 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5635 %{
 5636   predicate(CompressedOops::shift() == 0);
 5637   constraint(ALLOC_IN_RC(ptr_reg));
 5638   match(AddP (DecodeN reg) (ConvI2L ireg));
 5639   op_cost(0);
 5640   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5641   interface(MEMORY_INTER) %{
 5642     base($reg);
 5643     index($ireg);
 5644     scale(0x0);
 5645     disp(0x0);
 5646   %}
 5647 %}
 5648 
 5649 operand indIndexN(iRegN reg, iRegL lreg)
 5650 %{
 5651   predicate(CompressedOops::shift() == 0);
 5652   constraint(ALLOC_IN_RC(ptr_reg));
 5653   match(AddP (DecodeN reg) lreg);
 5654   op_cost(0);
 5655   format %{ "$reg, $lreg\t# narrow" %}
 5656   interface(MEMORY_INTER) %{
 5657     base($reg);
 5658     index($lreg);
 5659     scale(0x0);
 5660     disp(0x0);
 5661   %}
 5662 %}
 5663 
 5664 operand indOffIN(iRegN reg, immIOffset off)
 5665 %{
 5666   predicate(CompressedOops::shift() == 0);
 5667   constraint(ALLOC_IN_RC(ptr_reg));
 5668   match(AddP (DecodeN reg) off);
 5669   op_cost(0);
 5670   format %{ "[$reg, $off]\t# narrow" %}
 5671   interface(MEMORY_INTER) %{
 5672     base($reg);
 5673     index(0xffffffff);
 5674     scale(0x0);
 5675     disp($off);
 5676   %}
 5677 %}
 5678 
 5679 operand indOffLN(iRegN reg, immLOffset off)
 5680 %{
 5681   predicate(CompressedOops::shift() == 0);
 5682   constraint(ALLOC_IN_RC(ptr_reg));
 5683   match(AddP (DecodeN reg) off);
 5684   op_cost(0);
 5685   format %{ "[$reg, $off]\t# narrow" %}
 5686   interface(MEMORY_INTER) %{
 5687     base($reg);
 5688     index(0xffffffff);
 5689     scale(0x0);
 5690     disp($off);
 5691   %}
 5692 %}
 5693 
 5694 
 5695 //----------Special Memory Operands--------------------------------------------
 5696 // Stack Slot Operand - This operand is used for loading and storing temporary
 5697 //                      values on the stack where a match requires a value to
 5698 //                      flow through memory.
 5699 operand stackSlotP(sRegP reg)
 5700 %{
 5701   constraint(ALLOC_IN_RC(stack_slots));
 5702   op_cost(100);
 5703   // No match rule because this operand is only generated in matching
 5704   // match(RegP);
 5705   format %{ "[$reg]" %}
 5706   interface(MEMORY_INTER) %{
 5707     base(0x1e);  // RSP
 5708     index(0x0);  // No Index
 5709     scale(0x0);  // No Scale
 5710     disp($reg);  // Stack Offset
 5711   %}
 5712 %}
 5713 
 5714 operand stackSlotI(sRegI reg)
 5715 %{
 5716   constraint(ALLOC_IN_RC(stack_slots));
 5717   // No match rule because this operand is only generated in matching
 5718   // match(RegI);
 5719   format %{ "[$reg]" %}
 5720   interface(MEMORY_INTER) %{
 5721     base(0x1e);  // RSP
 5722     index(0x0);  // No Index
 5723     scale(0x0);  // No Scale
 5724     disp($reg);  // Stack Offset
 5725   %}
 5726 %}
 5727 
 5728 operand stackSlotF(sRegF reg)
 5729 %{
 5730   constraint(ALLOC_IN_RC(stack_slots));
 5731   // No match rule because this operand is only generated in matching
 5732   // match(RegF);
 5733   format %{ "[$reg]" %}
 5734   interface(MEMORY_INTER) %{
 5735     base(0x1e);  // RSP
 5736     index(0x0);  // No Index
 5737     scale(0x0);  // No Scale
 5738     disp($reg);  // Stack Offset
 5739   %}
 5740 %}
 5741 
 5742 operand stackSlotD(sRegD reg)
 5743 %{
 5744   constraint(ALLOC_IN_RC(stack_slots));
 5745   // No match rule because this operand is only generated in matching
 5746   // match(RegD);
 5747   format %{ "[$reg]" %}
 5748   interface(MEMORY_INTER) %{
 5749     base(0x1e);  // RSP
 5750     index(0x0);  // No Index
 5751     scale(0x0);  // No Scale
 5752     disp($reg);  // Stack Offset
 5753   %}
 5754 %}
 5755 
 5756 operand stackSlotL(sRegL reg)
 5757 %{
 5758   constraint(ALLOC_IN_RC(stack_slots));
 5759   // No match rule because this operand is only generated in matching
 5760   // match(RegL);
 5761   format %{ "[$reg]" %}
 5762   interface(MEMORY_INTER) %{
 5763     base(0x1e);  // RSP
 5764     index(0x0);  // No Index
 5765     scale(0x0);  // No Scale
 5766     disp($reg);  // Stack Offset
 5767   %}
 5768 %}
 5769 
 5770 // Operands for expressing Control Flow
 5771 // NOTE: Label is a predefined operand which should not be redefined in
 5772 //       the AD file. It is generically handled within the ADLC.
 5773 
 5774 //----------Conditional Branch Operands----------------------------------------
 5775 // Comparison Op  - This is the operation of the comparison, and is limited to
 5776 //                  the following set of codes:
 5777 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 5778 //
 5779 // Other attributes of the comparison, such as unsignedness, are specified
 5780 // by the comparison instruction that sets a condition code flags register.
 5781 // That result is represented by a flags operand whose subtype is appropriate
 5782 // to the unsignedness (etc.) of the comparison.
 5783 //
 5784 // Later, the instruction which matches both the Comparison Op (a Bool) and
 5785 // the flags (produced by the Cmp) specifies the coding of the comparison op
 5786 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 5787 
 5788 // used for signed integral comparisons and fp comparisons
 5789 
 5790 operand cmpOp()
 5791 %{
 5792   match(Bool);
 5793 
 5794   format %{ "" %}
 5795   interface(COND_INTER) %{
 5796     equal(0x0, "eq");
 5797     not_equal(0x1, "ne");
 5798     less(0xb, "lt");
 5799     greater_equal(0xa, "ge");
 5800     less_equal(0xd, "le");
 5801     greater(0xc, "gt");
 5802     overflow(0x6, "vs");
 5803     no_overflow(0x7, "vc");
 5804   %}
 5805 %}
 5806 
 5807 // used for unsigned integral comparisons
 5808 
 5809 operand cmpOpU()
 5810 %{
 5811   match(Bool);
 5812 
 5813   format %{ "" %}
 5814   interface(COND_INTER) %{
 5815     equal(0x0, "eq");
 5816     not_equal(0x1, "ne");
 5817     less(0x3, "lo");
 5818     greater_equal(0x2, "hs");
 5819     less_equal(0x9, "ls");
 5820     greater(0x8, "hi");
 5821     overflow(0x6, "vs");
 5822     no_overflow(0x7, "vc");
 5823   %}
 5824 %}
 5825 
 5826 // used for certain integral comparisons which can be
 5827 // converted to cbxx or tbxx instructions
 5828 
 5829 operand cmpOpEqNe()
 5830 %{
 5831   match(Bool);
 5832   op_cost(0);
 5833   predicate(n->as_Bool()->_test._test == BoolTest::ne
 5834             || n->as_Bool()->_test._test == BoolTest::eq);
 5835 
 5836   format %{ "" %}
 5837   interface(COND_INTER) %{
 5838     equal(0x0, "eq");
 5839     not_equal(0x1, "ne");
 5840     less(0xb, "lt");
 5841     greater_equal(0xa, "ge");
 5842     less_equal(0xd, "le");
 5843     greater(0xc, "gt");
 5844     overflow(0x6, "vs");
 5845     no_overflow(0x7, "vc");
 5846   %}
 5847 %}
 5848 
 5849 // used for certain integral comparisons which can be
 5850 // converted to cbxx or tbxx instructions
 5851 
 5852 operand cmpOpLtGe()
 5853 %{
 5854   match(Bool);
 5855   op_cost(0);
 5856 
 5857   predicate(n->as_Bool()->_test._test == BoolTest::lt
 5858             || n->as_Bool()->_test._test == BoolTest::ge);
 5859 
 5860   format %{ "" %}
 5861   interface(COND_INTER) %{
 5862     equal(0x0, "eq");
 5863     not_equal(0x1, "ne");
 5864     less(0xb, "lt");
 5865     greater_equal(0xa, "ge");
 5866     less_equal(0xd, "le");
 5867     greater(0xc, "gt");
 5868     overflow(0x6, "vs");
 5869     no_overflow(0x7, "vc");
 5870   %}
 5871 %}
 5872 
 5873 // used for certain unsigned integral comparisons which can be
 5874 // converted to cbxx or tbxx instructions
 5875 
 5876 operand cmpOpUEqNeLeGt()
 5877 %{
 5878   match(Bool);
 5879   op_cost(0);
 5880 
 5881   predicate(n->as_Bool()->_test._test == BoolTest::eq ||
 5882             n->as_Bool()->_test._test == BoolTest::ne ||
 5883             n->as_Bool()->_test._test == BoolTest::le ||
 5884             n->as_Bool()->_test._test == BoolTest::gt);
 5885 
 5886   format %{ "" %}
 5887   interface(COND_INTER) %{
 5888     equal(0x0, "eq");
 5889     not_equal(0x1, "ne");
 5890     less(0x3, "lo");
 5891     greater_equal(0x2, "hs");
 5892     less_equal(0x9, "ls");
 5893     greater(0x8, "hi");
 5894     overflow(0x6, "vs");
 5895     no_overflow(0x7, "vc");
 5896   %}
 5897 %}
 5898 
 5899 // Special operand allowing long args to int ops to be truncated for free
 5900 
 5901 operand iRegL2I(iRegL reg) %{
 5902 
 5903   op_cost(0);
 5904 
 5905   match(ConvL2I reg);
 5906 
 5907   format %{ "l2i($reg)" %}
 5908 
 5909   interface(REG_INTER)
 5910 %}
 5911 
 5912 operand iRegL2P(iRegL reg) %{
 5913 
 5914   op_cost(0);
 5915 
 5916   match(CastX2P reg);
 5917 
 5918   format %{ "l2p($reg)" %}
 5919 
 5920   interface(REG_INTER)
 5921 %}
 5922 
 5923 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 5924 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 5925 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 5926 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 5927 
 5928 //----------OPERAND CLASSES----------------------------------------------------
 5929 // Operand Classes are groups of operands that are used as to simplify
 5930 // instruction definitions by not requiring the AD writer to specify
 5931 // separate instructions for every form of operand when the
 5932 // instruction accepts multiple operand types with the same basic
 5933 // encoding and format. The classic case of this is memory operands.
 5934 
 5935 // memory is used to define read/write location for load/store
 5936 // instruction defs. we can turn a memory op into an Address
 5937 
 5938 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 5939                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
 5940 
 5941 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 5942                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
 5943 
 5944 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 5945                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
 5946 
 5947 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 5948                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
 5949 
 5950 // All of the memory operands. For the pipeline description.
 5951 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 5952                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 5953                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
 5954 
 5955 
 5956 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 5957 // operations. it allows the src to be either an iRegI or a (ConvL2I
 5958 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 5959 // can be elided because the 32-bit instruction will just employ the
 5960 // lower 32 bits anyway.
 5961 //
 5962 // n.b. this does not elide all L2I conversions. if the truncated
 5963 // value is consumed by more than one operation then the ConvL2I
 5964 // cannot be bundled into the consuming nodes so an l2i gets planted
 5965 // (actually a movw $dst $src) and the downstream instructions consume
 5966 // the result of the l2i as an iRegI input. That's a shame since the
 5967 // movw is actually redundant but its not too costly.
 5968 
 5969 opclass iRegIorL2I(iRegI, iRegL2I);
 5970 opclass iRegPorL2P(iRegP, iRegL2P);
 5971 
 5972 //----------PIPELINE-----------------------------------------------------------
 5973 // Rules which define the behavior of the target architectures pipeline.
 5974 
 5975 // For specific pipelines, eg A53, define the stages of that pipeline
 5976 //pipe_desc(ISS, EX1, EX2, WR);
 5977 #define ISS S0
 5978 #define EX1 S1
 5979 #define EX2 S2
 5980 #define WR  S3
 5981 
 5982 // Integer ALU reg operation
 5983 pipeline %{
 5984 
 5985 attributes %{
 5986   // ARM instructions are of fixed length
 5987   fixed_size_instructions;        // Fixed size instructions TODO does
 5988   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 5989   // ARM instructions come in 32-bit word units
 5990   instruction_unit_size = 4;         // An instruction is 4 bytes long
 5991   instruction_fetch_unit_size = 64;  // The processor fetches one line
 5992   instruction_fetch_units = 1;       // of 64 bytes
 5993 
 5994   // List of nop instructions
 5995   nops( MachNop );
 5996 %}
 5997 
 5998 // We don't use an actual pipeline model so don't care about resources
 5999 // or description. we do use pipeline classes to introduce fixed
 6000 // latencies
 6001 
 6002 //----------RESOURCES----------------------------------------------------------
 6003 // Resources are the functional units available to the machine
 6004 
 6005 resources( INS0, INS1, INS01 = INS0 | INS1,
 6006            ALU0, ALU1, ALU = ALU0 | ALU1,
 6007            MAC,
 6008            DIV,
 6009            BRANCH,
 6010            LDST,
 6011            NEON_FP);
 6012 
 6013 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6014 // Pipeline Description specifies the stages in the machine's pipeline
 6015 
 6016 // Define the pipeline as a generic 6 stage pipeline
 6017 pipe_desc(S0, S1, S2, S3, S4, S5);
 6018 
 6019 //----------PIPELINE CLASSES---------------------------------------------------
 6020 // Pipeline Classes describe the stages in which input and output are
 6021 // referenced by the hardware pipeline.
 6022 
 6023 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6024 %{
 6025   single_instruction;
 6026   src1   : S1(read);
 6027   src2   : S2(read);
 6028   dst    : S5(write);
 6029   INS01  : ISS;
 6030   NEON_FP : S5;
 6031 %}
 6032 
 6033 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6034 %{
 6035   single_instruction;
 6036   src1   : S1(read);
 6037   src2   : S2(read);
 6038   dst    : S5(write);
 6039   INS01  : ISS;
 6040   NEON_FP : S5;
 6041 %}
 6042 
 6043 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6044 %{
 6045   single_instruction;
 6046   src    : S1(read);
 6047   dst    : S5(write);
 6048   INS01  : ISS;
 6049   NEON_FP : S5;
 6050 %}
 6051 
 6052 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6053 %{
 6054   single_instruction;
 6055   src    : S1(read);
 6056   dst    : S5(write);
 6057   INS01  : ISS;
 6058   NEON_FP : S5;
 6059 %}
 6060 
 6061 pipe_class fp_d2f(vRegF dst, vRegD src)
 6062 %{
 6063   single_instruction;
 6064   src    : S1(read);
 6065   dst    : S5(write);
 6066   INS01  : ISS;
 6067   NEON_FP : S5;
 6068 %}
 6069 
 6070 pipe_class fp_f2d(vRegD dst, vRegF src)
 6071 %{
 6072   single_instruction;
 6073   src    : S1(read);
 6074   dst    : S5(write);
 6075   INS01  : ISS;
 6076   NEON_FP : S5;
 6077 %}
 6078 
 6079 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6080 %{
 6081   single_instruction;
 6082   src    : S1(read);
 6083   dst    : S5(write);
 6084   INS01  : ISS;
 6085   NEON_FP : S5;
 6086 %}
 6087 
 6088 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6089 %{
 6090   single_instruction;
 6091   src    : S1(read);
 6092   dst    : S5(write);
 6093   INS01  : ISS;
 6094   NEON_FP : S5;
 6095 %}
 6096 
 6097 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6098 %{
 6099   single_instruction;
 6100   src    : S1(read);
 6101   dst    : S5(write);
 6102   INS01  : ISS;
 6103   NEON_FP : S5;
 6104 %}
 6105 
 6106 pipe_class fp_l2f(vRegF dst, iRegL src)
 6107 %{
 6108   single_instruction;
 6109   src    : S1(read);
 6110   dst    : S5(write);
 6111   INS01  : ISS;
 6112   NEON_FP : S5;
 6113 %}
 6114 
 6115 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6116 %{
 6117   single_instruction;
 6118   src    : S1(read);
 6119   dst    : S5(write);
 6120   INS01  : ISS;
 6121   NEON_FP : S5;
 6122 %}
 6123 
 6124 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6125 %{
 6126   single_instruction;
 6127   src    : S1(read);
 6128   dst    : S5(write);
 6129   INS01  : ISS;
 6130   NEON_FP : S5;
 6131 %}
 6132 
 6133 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6134 %{
 6135   single_instruction;
 6136   src    : S1(read);
 6137   dst    : S5(write);
 6138   INS01  : ISS;
 6139   NEON_FP : S5;
 6140 %}
 6141 
 6142 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6143 %{
 6144   single_instruction;
 6145   src    : S1(read);
 6146   dst    : S5(write);
 6147   INS01  : ISS;
 6148   NEON_FP : S5;
 6149 %}
 6150 
 6151 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6152 %{
 6153   single_instruction;
 6154   src1   : S1(read);
 6155   src2   : S2(read);
 6156   dst    : S5(write);
 6157   INS0   : ISS;
 6158   NEON_FP : S5;
 6159 %}
 6160 
 6161 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6162 %{
 6163   single_instruction;
 6164   src1   : S1(read);
 6165   src2   : S2(read);
 6166   dst    : S5(write);
 6167   INS0   : ISS;
 6168   NEON_FP : S5;
 6169 %}
 6170 
 6171 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6172 %{
 6173   single_instruction;
 6174   cr     : S1(read);
 6175   src1   : S1(read);
 6176   src2   : S1(read);
 6177   dst    : S3(write);
 6178   INS01  : ISS;
 6179   NEON_FP : S3;
 6180 %}
 6181 
 6182 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6183 %{
 6184   single_instruction;
 6185   cr     : S1(read);
 6186   src1   : S1(read);
 6187   src2   : S1(read);
 6188   dst    : S3(write);
 6189   INS01  : ISS;
 6190   NEON_FP : S3;
 6191 %}
 6192 
 6193 pipe_class fp_imm_s(vRegF dst)
 6194 %{
 6195   single_instruction;
 6196   dst    : S3(write);
 6197   INS01  : ISS;
 6198   NEON_FP : S3;
 6199 %}
 6200 
 6201 pipe_class fp_imm_d(vRegD dst)
 6202 %{
 6203   single_instruction;
 6204   dst    : S3(write);
 6205   INS01  : ISS;
 6206   NEON_FP : S3;
 6207 %}
 6208 
 6209 pipe_class fp_load_constant_s(vRegF dst)
 6210 %{
 6211   single_instruction;
 6212   dst    : S4(write);
 6213   INS01  : ISS;
 6214   NEON_FP : S4;
 6215 %}
 6216 
 6217 pipe_class fp_load_constant_d(vRegD dst)
 6218 %{
 6219   single_instruction;
 6220   dst    : S4(write);
 6221   INS01  : ISS;
 6222   NEON_FP : S4;
 6223 %}
 6224 
 6225 //------- Integer ALU operations --------------------------
 6226 
 6227 // Integer ALU reg-reg operation
 6228 // Operands needed in EX1, result generated in EX2
 6229 // Eg.  ADD     x0, x1, x2
 6230 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6231 %{
 6232   single_instruction;
 6233   dst    : EX2(write);
 6234   src1   : EX1(read);
 6235   src2   : EX1(read);
 6236   INS01  : ISS; // Dual issue as instruction 0 or 1
 6237   ALU    : EX2;
 6238 %}
 6239 
 6240 // Integer ALU reg-reg operation with constant shift
 6241 // Shifted register must be available in LATE_ISS instead of EX1
 6242 // Eg.  ADD     x0, x1, x2, LSL #2
 6243 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6244 %{
 6245   single_instruction;
 6246   dst    : EX2(write);
 6247   src1   : EX1(read);
 6248   src2   : ISS(read);
 6249   INS01  : ISS;
 6250   ALU    : EX2;
 6251 %}
 6252 
 6253 // Integer ALU reg operation with constant shift
 6254 // Eg.  LSL     x0, x1, #shift
 6255 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6256 %{
 6257   single_instruction;
 6258   dst    : EX2(write);
 6259   src1   : ISS(read);
 6260   INS01  : ISS;
 6261   ALU    : EX2;
 6262 %}
 6263 
 6264 // Integer ALU reg-reg operation with variable shift
 6265 // Both operands must be available in LATE_ISS instead of EX1
 6266 // Result is available in EX1 instead of EX2
 6267 // Eg.  LSLV    x0, x1, x2
 6268 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6269 %{
 6270   single_instruction;
 6271   dst    : EX1(write);
 6272   src1   : ISS(read);
 6273   src2   : ISS(read);
 6274   INS01  : ISS;
 6275   ALU    : EX1;
 6276 %}
 6277 
 6278 // Integer ALU reg-reg operation with extract
 6279 // As for _vshift above, but result generated in EX2
 6280 // Eg.  EXTR    x0, x1, x2, #N
 6281 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6282 %{
 6283   single_instruction;
 6284   dst    : EX2(write);
 6285   src1   : ISS(read);
 6286   src2   : ISS(read);
 6287   INS1   : ISS; // Can only dual issue as Instruction 1
 6288   ALU    : EX1;
 6289 %}
 6290 
 6291 // Integer ALU reg operation
 6292 // Eg.  NEG     x0, x1
 6293 pipe_class ialu_reg(iRegI dst, iRegI src)
 6294 %{
 6295   single_instruction;
 6296   dst    : EX2(write);
 6297   src    : EX1(read);
 6298   INS01  : ISS;
 6299   ALU    : EX2;
 6300 %}
 6301 
 6302 // Integer ALU reg mmediate operation
 6303 // Eg.  ADD     x0, x1, #N
 6304 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6305 %{
 6306   single_instruction;
 6307   dst    : EX2(write);
 6308   src1   : EX1(read);
 6309   INS01  : ISS;
 6310   ALU    : EX2;
 6311 %}
 6312 
 6313 // Integer ALU immediate operation (no source operands)
 6314 // Eg.  MOV     x0, #N
 6315 pipe_class ialu_imm(iRegI dst)
 6316 %{
 6317   single_instruction;
 6318   dst    : EX1(write);
 6319   INS01  : ISS;
 6320   ALU    : EX1;
 6321 %}
 6322 
 6323 //------- Compare operation -------------------------------
 6324 
 6325 // Compare reg-reg
 6326 // Eg.  CMP     x0, x1
 6327 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6328 %{
 6329   single_instruction;
 6330 //  fixed_latency(16);
 6331   cr     : EX2(write);
 6332   op1    : EX1(read);
 6333   op2    : EX1(read);
 6334   INS01  : ISS;
 6335   ALU    : EX2;
 6336 %}
 6337 
 6338 // Compare reg-reg
 6339 // Eg.  CMP     x0, #N
 6340 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6341 %{
 6342   single_instruction;
 6343 //  fixed_latency(16);
 6344   cr     : EX2(write);
 6345   op1    : EX1(read);
 6346   INS01  : ISS;
 6347   ALU    : EX2;
 6348 %}
 6349 
 6350 //------- Conditional instructions ------------------------
 6351 
 6352 // Conditional no operands
 6353 // Eg.  CSINC   x0, zr, zr, <cond>
 6354 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6355 %{
 6356   single_instruction;
 6357   cr     : EX1(read);
 6358   dst    : EX2(write);
 6359   INS01  : ISS;
 6360   ALU    : EX2;
 6361 %}
 6362 
 6363 // Conditional 2 operand
 6364 // EG.  CSEL    X0, X1, X2, <cond>
 6365 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6366 %{
 6367   single_instruction;
 6368   cr     : EX1(read);
 6369   src1   : EX1(read);
 6370   src2   : EX1(read);
 6371   dst    : EX2(write);
 6372   INS01  : ISS;
 6373   ALU    : EX2;
 6374 %}
 6375 
 6376 // Conditional 2 operand
 6377 // EG.  CSEL    X0, X1, X2, <cond>
 6378 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6379 %{
 6380   single_instruction;
 6381   cr     : EX1(read);
 6382   src    : EX1(read);
 6383   dst    : EX2(write);
 6384   INS01  : ISS;
 6385   ALU    : EX2;
 6386 %}
 6387 
 6388 //------- Multiply pipeline operations --------------------
 6389 
 6390 // Multiply reg-reg
 6391 // Eg.  MUL     w0, w1, w2
 6392 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6393 %{
 6394   single_instruction;
 6395   dst    : WR(write);
 6396   src1   : ISS(read);
 6397   src2   : ISS(read);
 6398   INS01  : ISS;
 6399   MAC    : WR;
 6400 %}
 6401 
 6402 // Multiply accumulate
 6403 // Eg.  MADD    w0, w1, w2, w3
 6404 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6405 %{
 6406   single_instruction;
 6407   dst    : WR(write);
 6408   src1   : ISS(read);
 6409   src2   : ISS(read);
 6410   src3   : ISS(read);
 6411   INS01  : ISS;
 6412   MAC    : WR;
 6413 %}
 6414 
 6415 // Eg.  MUL     w0, w1, w2
 6416 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6417 %{
 6418   single_instruction;
 6419   fixed_latency(3); // Maximum latency for 64 bit mul
 6420   dst    : WR(write);
 6421   src1   : ISS(read);
 6422   src2   : ISS(read);
 6423   INS01  : ISS;
 6424   MAC    : WR;
 6425 %}
 6426 
 6427 // Multiply accumulate
 6428 // Eg.  MADD    w0, w1, w2, w3
 6429 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6430 %{
 6431   single_instruction;
 6432   fixed_latency(3); // Maximum latency for 64 bit mul
 6433   dst    : WR(write);
 6434   src1   : ISS(read);
 6435   src2   : ISS(read);
 6436   src3   : ISS(read);
 6437   INS01  : ISS;
 6438   MAC    : WR;
 6439 %}
 6440 
 6441 //------- Divide pipeline operations --------------------
 6442 
 6443 // Eg.  SDIV    w0, w1, w2
 6444 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6445 %{
 6446   single_instruction;
 6447   fixed_latency(8); // Maximum latency for 32 bit divide
 6448   dst    : WR(write);
 6449   src1   : ISS(read);
 6450   src2   : ISS(read);
 6451   INS0   : ISS; // Can only dual issue as instruction 0
 6452   DIV    : WR;
 6453 %}
 6454 
 6455 // Eg.  SDIV    x0, x1, x2
 6456 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6457 %{
 6458   single_instruction;
 6459   fixed_latency(16); // Maximum latency for 64 bit divide
 6460   dst    : WR(write);
 6461   src1   : ISS(read);
 6462   src2   : ISS(read);
 6463   INS0   : ISS; // Can only dual issue as instruction 0
 6464   DIV    : WR;
 6465 %}
 6466 
 6467 //------- Load pipeline operations ------------------------
 6468 
 6469 // Load - prefetch
 6470 // Eg.  PFRM    <mem>
 6471 pipe_class iload_prefetch(memory mem)
 6472 %{
 6473   single_instruction;
 6474   mem    : ISS(read);
 6475   INS01  : ISS;
 6476   LDST   : WR;
 6477 %}
 6478 
 6479 // Load - reg, mem
 6480 // Eg.  LDR     x0, <mem>
 6481 pipe_class iload_reg_mem(iRegI dst, memory mem)
 6482 %{
 6483   single_instruction;
 6484   dst    : WR(write);
 6485   mem    : ISS(read);
 6486   INS01  : ISS;
 6487   LDST   : WR;
 6488 %}
 6489 
 6490 // Load - reg, reg
 6491 // Eg.  LDR     x0, [sp, x1]
 6492 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 6493 %{
 6494   single_instruction;
 6495   dst    : WR(write);
 6496   src    : ISS(read);
 6497   INS01  : ISS;
 6498   LDST   : WR;
 6499 %}
 6500 
 6501 //------- Store pipeline operations -----------------------
 6502 
 6503 // Store - zr, mem
 6504 // Eg.  STR     zr, <mem>
 6505 pipe_class istore_mem(memory mem)
 6506 %{
 6507   single_instruction;
 6508   mem    : ISS(read);
 6509   INS01  : ISS;
 6510   LDST   : WR;
 6511 %}
 6512 
 6513 // Store - reg, mem
 6514 // Eg.  STR     x0, <mem>
 6515 pipe_class istore_reg_mem(iRegI src, memory mem)
 6516 %{
 6517   single_instruction;
 6518   mem    : ISS(read);
 6519   src    : EX2(read);
 6520   INS01  : ISS;
 6521   LDST   : WR;
 6522 %}
 6523 
 6524 // Store - reg, reg
 6525 // Eg. STR      x0, [sp, x1]
 6526 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 6527 %{
 6528   single_instruction;
 6529   dst    : ISS(read);
 6530   src    : EX2(read);
 6531   INS01  : ISS;
 6532   LDST   : WR;
 6533 %}
 6534 
 6535 //------- Store pipeline operations -----------------------
 6536 
 6537 // Branch
 6538 pipe_class pipe_branch()
 6539 %{
 6540   single_instruction;
 6541   INS01  : ISS;
 6542   BRANCH : EX1;
 6543 %}
 6544 
 6545 // Conditional branch
 6546 pipe_class pipe_branch_cond(rFlagsReg cr)
 6547 %{
 6548   single_instruction;
 6549   cr     : EX1(read);
 6550   INS01  : ISS;
 6551   BRANCH : EX1;
 6552 %}
 6553 
 6554 // Compare & Branch
 6555 // EG.  CBZ/CBNZ
 6556 pipe_class pipe_cmp_branch(iRegI op1)
 6557 %{
 6558   single_instruction;
 6559   op1    : EX1(read);
 6560   INS01  : ISS;
 6561   BRANCH : EX1;
 6562 %}
 6563 
 6564 //------- Synchronisation operations ----------------------
 6565 
 6566 // Any operation requiring serialization.
 6567 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 6568 pipe_class pipe_serial()
 6569 %{
 6570   single_instruction;
 6571   force_serialization;
 6572   fixed_latency(16);
 6573   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6574   LDST   : WR;
 6575 %}
 6576 
 6577 // Generic big/slow expanded idiom - also serialized
 6578 pipe_class pipe_slow()
 6579 %{
 6580   instruction_count(10);
 6581   multiple_bundles;
 6582   force_serialization;
 6583   fixed_latency(16);
 6584   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6585   LDST   : WR;
 6586 %}
 6587 
 6588 // Empty pipeline class
 6589 pipe_class pipe_class_empty()
 6590 %{
 6591   single_instruction;
 6592   fixed_latency(0);
 6593 %}
 6594 
 6595 // Default pipeline class.
 6596 pipe_class pipe_class_default()
 6597 %{
 6598   single_instruction;
 6599   fixed_latency(2);
 6600 %}
 6601 
 6602 // Pipeline class for compares.
 6603 pipe_class pipe_class_compare()
 6604 %{
 6605   single_instruction;
 6606   fixed_latency(16);
 6607 %}
 6608 
 6609 // Pipeline class for memory operations.
 6610 pipe_class pipe_class_memory()
 6611 %{
 6612   single_instruction;
 6613   fixed_latency(16);
 6614 %}
 6615 
 6616 // Pipeline class for call.
 6617 pipe_class pipe_class_call()
 6618 %{
 6619   single_instruction;
 6620   fixed_latency(100);
 6621 %}
 6622 
 6623 // Define the class for the Nop node.
 6624 define %{
 6625    MachNop = pipe_class_empty;
 6626 %}
 6627 
 6628 %}
 6629 //----------INSTRUCTIONS-------------------------------------------------------
 6630 //
 6631 // match      -- States which machine-independent subtree may be replaced
 6632 //               by this instruction.
 6633 // ins_cost   -- The estimated cost of this instruction is used by instruction
 6634 //               selection to identify a minimum cost tree of machine
 6635 //               instructions that matches a tree of machine-independent
 6636 //               instructions.
 6637 // format     -- A string providing the disassembly for this instruction.
 6638 //               The value of an instruction's operand may be inserted
 6639 //               by referring to it with a '$' prefix.
 6640 // opcode     -- Three instruction opcodes may be provided.  These are referred
 6641 //               to within an encode class as $primary, $secondary, and $tertiary
 6642 //               rrspectively.  The primary opcode is commonly used to
 6643 //               indicate the type of machine instruction, while secondary
 6644 //               and tertiary are often used for prefix options or addressing
 6645 //               modes.
 6646 // ins_encode -- A list of encode classes with parameters. The encode class
 6647 //               name must have been defined in an 'enc_class' specification
 6648 //               in the encode section of the architecture description.
 6649 
 6650 // ============================================================================
 6651 // Memory (Load/Store) Instructions
 6652 
 6653 // Load Instructions
 6654 
 6655 // Load Byte (8 bit signed)
 6656 instruct loadB(iRegINoSp dst, memory1 mem)
 6657 %{
 6658   match(Set dst (LoadB mem));
 6659   predicate(!needs_acquiring_load(n));
 6660 
 6661   ins_cost(4 * INSN_COST);
 6662   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 6663 
 6664   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 6665 
 6666   ins_pipe(iload_reg_mem);
 6667 %}
 6668 
 6669 // Load Byte (8 bit signed) into long
 6670 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 6671 %{
 6672   match(Set dst (ConvI2L (LoadB mem)));
 6673   predicate(!needs_acquiring_load(n->in(1)));
 6674 
 6675   ins_cost(4 * INSN_COST);
 6676   format %{ "ldrsb  $dst, $mem\t# byte" %}
 6677 
 6678   ins_encode(aarch64_enc_ldrsb(dst, mem));
 6679 
 6680   ins_pipe(iload_reg_mem);
 6681 %}
 6682 
 6683 // Load Byte (8 bit unsigned)
 6684 instruct loadUB(iRegINoSp dst, memory1 mem)
 6685 %{
 6686   match(Set dst (LoadUB mem));
 6687   predicate(!needs_acquiring_load(n));
 6688 
 6689   ins_cost(4 * INSN_COST);
 6690   format %{ "ldrbw  $dst, $mem\t# byte" %}
 6691 
 6692   ins_encode(aarch64_enc_ldrb(dst, mem));
 6693 
 6694   ins_pipe(iload_reg_mem);
 6695 %}
 6696 
 6697 // Load Byte (8 bit unsigned) into long
 6698 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 6699 %{
 6700   match(Set dst (ConvI2L (LoadUB mem)));
 6701   predicate(!needs_acquiring_load(n->in(1)));
 6702 
 6703   ins_cost(4 * INSN_COST);
 6704   format %{ "ldrb  $dst, $mem\t# byte" %}
 6705 
 6706   ins_encode(aarch64_enc_ldrb(dst, mem));
 6707 
 6708   ins_pipe(iload_reg_mem);
 6709 %}
 6710 
 6711 // Load Short (16 bit signed)
 6712 instruct loadS(iRegINoSp dst, memory2 mem)
 6713 %{
 6714   match(Set dst (LoadS mem));
 6715   predicate(!needs_acquiring_load(n));
 6716 
 6717   ins_cost(4 * INSN_COST);
 6718   format %{ "ldrshw  $dst, $mem\t# short" %}
 6719 
 6720   ins_encode(aarch64_enc_ldrshw(dst, mem));
 6721 
 6722   ins_pipe(iload_reg_mem);
 6723 %}
 6724 
 6725 // Load Short (16 bit signed) into long
 6726 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 6727 %{
 6728   match(Set dst (ConvI2L (LoadS mem)));
 6729   predicate(!needs_acquiring_load(n->in(1)));
 6730 
 6731   ins_cost(4 * INSN_COST);
 6732   format %{ "ldrsh  $dst, $mem\t# short" %}
 6733 
 6734   ins_encode(aarch64_enc_ldrsh(dst, mem));
 6735 
 6736   ins_pipe(iload_reg_mem);
 6737 %}
 6738 
 6739 // Load Char (16 bit unsigned)
 6740 instruct loadUS(iRegINoSp dst, memory2 mem)
 6741 %{
 6742   match(Set dst (LoadUS mem));
 6743   predicate(!needs_acquiring_load(n));
 6744 
 6745   ins_cost(4 * INSN_COST);
 6746   format %{ "ldrh  $dst, $mem\t# short" %}
 6747 
 6748   ins_encode(aarch64_enc_ldrh(dst, mem));
 6749 
 6750   ins_pipe(iload_reg_mem);
 6751 %}
 6752 
 6753 // Load Short/Char (16 bit unsigned) into long
 6754 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 6755 %{
 6756   match(Set dst (ConvI2L (LoadUS mem)));
 6757   predicate(!needs_acquiring_load(n->in(1)));
 6758 
 6759   ins_cost(4 * INSN_COST);
 6760   format %{ "ldrh  $dst, $mem\t# short" %}
 6761 
 6762   ins_encode(aarch64_enc_ldrh(dst, mem));
 6763 
 6764   ins_pipe(iload_reg_mem);
 6765 %}
 6766 
 6767 // Load Integer (32 bit signed)
 6768 instruct loadI(iRegINoSp dst, memory4 mem)
 6769 %{
 6770   match(Set dst (LoadI mem));
 6771   predicate(!needs_acquiring_load(n));
 6772 
 6773   ins_cost(4 * INSN_COST);
 6774   format %{ "ldrw  $dst, $mem\t# int" %}
 6775 
 6776   ins_encode(aarch64_enc_ldrw(dst, mem));
 6777 
 6778   ins_pipe(iload_reg_mem);
 6779 %}
 6780 
 6781 // Load Integer (32 bit signed) into long
 6782 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 6783 %{
 6784   match(Set dst (ConvI2L (LoadI mem)));
 6785   predicate(!needs_acquiring_load(n->in(1)));
 6786 
 6787   ins_cost(4 * INSN_COST);
 6788   format %{ "ldrsw  $dst, $mem\t# int" %}
 6789 
 6790   ins_encode(aarch64_enc_ldrsw(dst, mem));
 6791 
 6792   ins_pipe(iload_reg_mem);
 6793 %}
 6794 
 6795 // Load Integer (32 bit unsigned) into long
 6796 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 6797 %{
 6798   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 6799   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 6800 
 6801   ins_cost(4 * INSN_COST);
 6802   format %{ "ldrw  $dst, $mem\t# int" %}
 6803 
 6804   ins_encode(aarch64_enc_ldrw(dst, mem));
 6805 
 6806   ins_pipe(iload_reg_mem);
 6807 %}
 6808 
 6809 // Load Long (64 bit signed)
 6810 instruct loadL(iRegLNoSp dst, memory8 mem)
 6811 %{
 6812   match(Set dst (LoadL mem));
 6813   predicate(!needs_acquiring_load(n));
 6814 
 6815   ins_cost(4 * INSN_COST);
 6816   format %{ "ldr  $dst, $mem\t# int" %}
 6817 
 6818   ins_encode(aarch64_enc_ldr(dst, mem));
 6819 
 6820   ins_pipe(iload_reg_mem);
 6821 %}
 6822 
 6823 // Load Range
 6824 instruct loadRange(iRegINoSp dst, memory4 mem)
 6825 %{
 6826   match(Set dst (LoadRange mem));
 6827 
 6828   ins_cost(4 * INSN_COST);
 6829   format %{ "ldrw  $dst, $mem\t# range" %}
 6830 
 6831   ins_encode(aarch64_enc_ldrw(dst, mem));
 6832 
 6833   ins_pipe(iload_reg_mem);
 6834 %}
 6835 
 6836 // Load Pointer
 6837 instruct loadP(iRegPNoSp dst, memory8 mem)
 6838 %{
 6839   match(Set dst (LoadP mem));
 6840   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 6841 
 6842   ins_cost(4 * INSN_COST);
 6843   format %{ "ldr  $dst, $mem\t# ptr" %}
 6844 
 6845   ins_encode(aarch64_enc_ldr(dst, mem));
 6846 
 6847   ins_pipe(iload_reg_mem);
 6848 %}
 6849 
 6850 // Load Compressed Pointer
 6851 instruct loadN(iRegNNoSp dst, memory4 mem)
 6852 %{
 6853   match(Set dst (LoadN mem));
 6854   predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
 6855 
 6856   ins_cost(4 * INSN_COST);
 6857   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 6858 
 6859   ins_encode(aarch64_enc_ldrw(dst, mem));
 6860 
 6861   ins_pipe(iload_reg_mem);
 6862 %}
 6863 
 6864 // Load Klass Pointer
 6865 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 6866 %{
 6867   match(Set dst (LoadKlass mem));
 6868   predicate(!needs_acquiring_load(n));
 6869 
 6870   ins_cost(4 * INSN_COST);
 6871   format %{ "ldr  $dst, $mem\t# class" %}
 6872 
 6873   ins_encode(aarch64_enc_ldr(dst, mem));
 6874 
 6875   ins_pipe(iload_reg_mem);
 6876 %}
 6877 
 6878 // Load Narrow Klass Pointer
 6879 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 6880 %{
 6881   match(Set dst (LoadNKlass mem));
 6882   predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
 6883 
 6884   ins_cost(4 * INSN_COST);
 6885   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 6886 
 6887   ins_encode(aarch64_enc_ldrw(dst, mem));
 6888 
 6889   ins_pipe(iload_reg_mem);
 6890 %}
 6891 
 6892 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
 6893 %{
 6894   match(Set dst (LoadNKlass mem));
 6895   predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
 6896 
 6897   ins_cost(4 * INSN_COST);
 6898   format %{
 6899     "ldrw  $dst, $mem\t# compressed class ptr, shifted\n\t"
 6900     "lsrw  $dst, $dst, markWord::klass_shift_at_offset"
 6901   %}
 6902   ins_encode %{
 6903     // inlined aarch64_enc_ldrw
 6904     loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
 6905               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 6906     __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
 6907   %}
 6908   ins_pipe(iload_reg_mem);
 6909 %}
 6910 
 6911 // Load Float
 6912 instruct loadF(vRegF dst, memory4 mem)
 6913 %{
 6914   match(Set dst (LoadF mem));
 6915   predicate(!needs_acquiring_load(n));
 6916 
 6917   ins_cost(4 * INSN_COST);
 6918   format %{ "ldrs  $dst, $mem\t# float" %}
 6919 
 6920   ins_encode( aarch64_enc_ldrs(dst, mem) );
 6921 
 6922   ins_pipe(pipe_class_memory);
 6923 %}
 6924 
 6925 // Load Double
 6926 instruct loadD(vRegD dst, memory8 mem)
 6927 %{
 6928   match(Set dst (LoadD mem));
 6929   predicate(!needs_acquiring_load(n));
 6930 
 6931   ins_cost(4 * INSN_COST);
 6932   format %{ "ldrd  $dst, $mem\t# double" %}
 6933 
 6934   ins_encode( aarch64_enc_ldrd(dst, mem) );
 6935 
 6936   ins_pipe(pipe_class_memory);
 6937 %}
 6938 
 6939 
 6940 // Load Int Constant
 6941 instruct loadConI(iRegINoSp dst, immI src)
 6942 %{
 6943   match(Set dst src);
 6944 
 6945   ins_cost(INSN_COST);
 6946   format %{ "mov $dst, $src\t# int" %}
 6947 
 6948   ins_encode( aarch64_enc_movw_imm(dst, src) );
 6949 
 6950   ins_pipe(ialu_imm);
 6951 %}
 6952 
 6953 // Load Long Constant
 6954 instruct loadConL(iRegLNoSp dst, immL src)
 6955 %{
 6956   match(Set dst src);
 6957 
 6958   ins_cost(INSN_COST);
 6959   format %{ "mov $dst, $src\t# long" %}
 6960 
 6961   ins_encode( aarch64_enc_mov_imm(dst, src) );
 6962 
 6963   ins_pipe(ialu_imm);
 6964 %}
 6965 
 6966 // Load Pointer Constant
 6967 
 6968 instruct loadConP(iRegPNoSp dst, immP con)
 6969 %{
 6970   match(Set dst con);
 6971 
 6972   ins_cost(INSN_COST * 4);
 6973   format %{
 6974     "mov  $dst, $con\t# ptr\n\t"
 6975   %}
 6976 
 6977   ins_encode(aarch64_enc_mov_p(dst, con));
 6978 
 6979   ins_pipe(ialu_imm);
 6980 %}
 6981 
 6982 // Load Null Pointer Constant
 6983 
 6984 instruct loadConP0(iRegPNoSp dst, immP0 con)
 6985 %{
 6986   match(Set dst con);
 6987 
 6988   ins_cost(INSN_COST);
 6989   format %{ "mov  $dst, $con\t# nullptr ptr" %}
 6990 
 6991   ins_encode(aarch64_enc_mov_p0(dst, con));
 6992 
 6993   ins_pipe(ialu_imm);
 6994 %}
 6995 
 6996 // Load Pointer Constant One
 6997 
 6998 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 6999 %{
 7000   match(Set dst con);
 7001 
 7002   ins_cost(INSN_COST);
 7003   format %{ "mov  $dst, $con\t# nullptr ptr" %}
 7004 
 7005   ins_encode(aarch64_enc_mov_p1(dst, con));
 7006 
 7007   ins_pipe(ialu_imm);
 7008 %}
 7009 
 7010 // Load Byte Map Base Constant
 7011 
 7012 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7013 %{
 7014   match(Set dst con);
 7015 
 7016   ins_cost(INSN_COST);
 7017   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7018 
 7019   ins_encode %{
 7020     __ load_byte_map_base($dst$$Register);
 7021   %}
 7022 
 7023   ins_pipe(ialu_imm);
 7024 %}
 7025 
 7026 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
 7027 %{
 7028   match(Set dst con);
 7029 
 7030   ins_cost(INSN_COST);
 7031   format %{ "adr  $dst, $con\t# AOT Runtime Constants Address" %}
 7032 
 7033   ins_encode %{
 7034     __ load_aotrc_address($dst$$Register, (address)$con$$constant);
 7035   %}
 7036 
 7037   ins_pipe(ialu_imm);
 7038 %}
 7039 
 7040 // Load Narrow Pointer Constant
 7041 
 7042 instruct loadConN(iRegNNoSp dst, immN con)
 7043 %{
 7044   match(Set dst con);
 7045 
 7046   ins_cost(INSN_COST * 4);
 7047   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7048 
 7049   ins_encode(aarch64_enc_mov_n(dst, con));
 7050 
 7051   ins_pipe(ialu_imm);
 7052 %}
 7053 
 7054 // Load Narrow Null Pointer Constant
 7055 
 7056 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7057 %{
 7058   match(Set dst con);
 7059 
 7060   ins_cost(INSN_COST);
 7061   format %{ "mov  $dst, $con\t# compressed nullptr ptr" %}
 7062 
 7063   ins_encode(aarch64_enc_mov_n0(dst, con));
 7064 
 7065   ins_pipe(ialu_imm);
 7066 %}
 7067 
 7068 // Load Narrow Klass Constant
 7069 
 7070 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7071 %{
 7072   match(Set dst con);
 7073 
 7074   ins_cost(INSN_COST);
 7075   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7076 
 7077   ins_encode(aarch64_enc_mov_nk(dst, con));
 7078 
 7079   ins_pipe(ialu_imm);
 7080 %}
 7081 
 7082 // Load Packed Float Constant
 7083 
 7084 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7085   match(Set dst con);
 7086   ins_cost(INSN_COST * 4);
 7087   format %{ "fmovs  $dst, $con"%}
 7088   ins_encode %{
 7089     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7090   %}
 7091 
 7092   ins_pipe(fp_imm_s);
 7093 %}
 7094 
 7095 // Load Float Constant
 7096 
 7097 instruct loadConF(vRegF dst, immF con) %{
 7098   match(Set dst con);
 7099 
 7100   ins_cost(INSN_COST * 4);
 7101 
 7102   format %{
 7103     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7104   %}
 7105 
 7106   ins_encode %{
 7107     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7108   %}
 7109 
 7110   ins_pipe(fp_load_constant_s);
 7111 %}
 7112 
 7113 // Load Packed Double Constant
 7114 
 7115 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7116   match(Set dst con);
 7117   ins_cost(INSN_COST);
 7118   format %{ "fmovd  $dst, $con"%}
 7119   ins_encode %{
 7120     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7121   %}
 7122 
 7123   ins_pipe(fp_imm_d);
 7124 %}
 7125 
 7126 // Load Double Constant
 7127 
 7128 instruct loadConD(vRegD dst, immD con) %{
 7129   match(Set dst con);
 7130 
 7131   ins_cost(INSN_COST * 5);
 7132   format %{
 7133     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7134   %}
 7135 
 7136   ins_encode %{
 7137     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7138   %}
 7139 
 7140   ins_pipe(fp_load_constant_d);
 7141 %}
 7142 
 7143 // Load Half Float Constant
 7144 instruct loadConH(vRegF dst, immH con) %{
 7145   match(Set dst con);
 7146   format %{ "mov    rscratch1, $con\n\t"
 7147             "fmov   $dst, rscratch1"
 7148          %}
 7149   ins_encode %{
 7150     __ movw(rscratch1, (uint32_t)$con$$constant);
 7151     __ fmovs($dst$$FloatRegister, rscratch1);
 7152   %}
 7153   ins_pipe(pipe_class_default);
 7154 %}
 7155 
 7156 // Store Instructions
 7157 
 7158 // Store Byte
 7159 instruct storeB(iRegIorL2I src, memory1 mem)
 7160 %{
 7161   match(Set mem (StoreB mem src));
 7162   predicate(!needs_releasing_store(n));
 7163 
 7164   ins_cost(INSN_COST);
 7165   format %{ "strb  $src, $mem\t# byte" %}
 7166 
 7167   ins_encode(aarch64_enc_strb(src, mem));
 7168 
 7169   ins_pipe(istore_reg_mem);
 7170 %}
 7171 
 7172 
 7173 instruct storeimmB0(immI0 zero, memory1 mem)
 7174 %{
 7175   match(Set mem (StoreB mem zero));
 7176   predicate(!needs_releasing_store(n));
 7177 
 7178   ins_cost(INSN_COST);
 7179   format %{ "strb rscractch2, $mem\t# byte" %}
 7180 
 7181   ins_encode(aarch64_enc_strb0(mem));
 7182 
 7183   ins_pipe(istore_mem);
 7184 %}
 7185 
 7186 // Store Char/Short
 7187 instruct storeC(iRegIorL2I src, memory2 mem)
 7188 %{
 7189   match(Set mem (StoreC mem src));
 7190   predicate(!needs_releasing_store(n));
 7191 
 7192   ins_cost(INSN_COST);
 7193   format %{ "strh  $src, $mem\t# short" %}
 7194 
 7195   ins_encode(aarch64_enc_strh(src, mem));
 7196 
 7197   ins_pipe(istore_reg_mem);
 7198 %}
 7199 
 7200 instruct storeimmC0(immI0 zero, memory2 mem)
 7201 %{
 7202   match(Set mem (StoreC mem zero));
 7203   predicate(!needs_releasing_store(n));
 7204 
 7205   ins_cost(INSN_COST);
 7206   format %{ "strh  zr, $mem\t# short" %}
 7207 
 7208   ins_encode(aarch64_enc_strh0(mem));
 7209 
 7210   ins_pipe(istore_mem);
 7211 %}
 7212 
 7213 // Store Integer
 7214 
 7215 instruct storeI(iRegIorL2I src, memory4 mem)
 7216 %{
 7217   match(Set mem(StoreI mem src));
 7218   predicate(!needs_releasing_store(n));
 7219 
 7220   ins_cost(INSN_COST);
 7221   format %{ "strw  $src, $mem\t# int" %}
 7222 
 7223   ins_encode(aarch64_enc_strw(src, mem));
 7224 
 7225   ins_pipe(istore_reg_mem);
 7226 %}
 7227 
 7228 instruct storeimmI0(immI0 zero, memory4 mem)
 7229 %{
 7230   match(Set mem(StoreI mem zero));
 7231   predicate(!needs_releasing_store(n));
 7232 
 7233   ins_cost(INSN_COST);
 7234   format %{ "strw  zr, $mem\t# int" %}
 7235 
 7236   ins_encode(aarch64_enc_strw0(mem));
 7237 
 7238   ins_pipe(istore_mem);
 7239 %}
 7240 
 7241 // Store Long (64 bit signed)
 7242 instruct storeL(iRegL src, memory8 mem)
 7243 %{
 7244   match(Set mem (StoreL mem src));
 7245   predicate(!needs_releasing_store(n));
 7246 
 7247   ins_cost(INSN_COST);
 7248   format %{ "str  $src, $mem\t# int" %}
 7249 
 7250   ins_encode(aarch64_enc_str(src, mem));
 7251 
 7252   ins_pipe(istore_reg_mem);
 7253 %}
 7254 
 7255 // Store Long (64 bit signed)
 7256 instruct storeimmL0(immL0 zero, memory8 mem)
 7257 %{
 7258   match(Set mem (StoreL mem zero));
 7259   predicate(!needs_releasing_store(n));
 7260 
 7261   ins_cost(INSN_COST);
 7262   format %{ "str  zr, $mem\t# int" %}
 7263 
 7264   ins_encode(aarch64_enc_str0(mem));
 7265 
 7266   ins_pipe(istore_mem);
 7267 %}
 7268 
 7269 // Store Pointer
 7270 instruct storeP(iRegP src, memory8 mem)
 7271 %{
 7272   match(Set mem (StoreP mem src));
 7273   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7274 
 7275   ins_cost(INSN_COST);
 7276   format %{ "str  $src, $mem\t# ptr" %}
 7277 
 7278   ins_encode(aarch64_enc_str(src, mem));
 7279 
 7280   ins_pipe(istore_reg_mem);
 7281 %}
 7282 
 7283 // Store Pointer
 7284 instruct storeimmP0(immP0 zero, memory8 mem)
 7285 %{
 7286   match(Set mem (StoreP mem zero));
 7287   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7288 
 7289   ins_cost(INSN_COST);
 7290   format %{ "str zr, $mem\t# ptr" %}
 7291 
 7292   ins_encode(aarch64_enc_str0(mem));
 7293 
 7294   ins_pipe(istore_mem);
 7295 %}
 7296 
 7297 // Store Compressed Pointer
 7298 instruct storeN(iRegN src, memory4 mem)
 7299 %{
 7300   match(Set mem (StoreN mem src));
 7301   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7302 
 7303   ins_cost(INSN_COST);
 7304   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7305 
 7306   ins_encode(aarch64_enc_strw(src, mem));
 7307 
 7308   ins_pipe(istore_reg_mem);
 7309 %}
 7310 
 7311 instruct storeImmN0(immN0 zero, memory4 mem)
 7312 %{
 7313   match(Set mem (StoreN mem zero));
 7314   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7315 
 7316   ins_cost(INSN_COST);
 7317   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7318 
 7319   ins_encode(aarch64_enc_strw0(mem));
 7320 
 7321   ins_pipe(istore_mem);
 7322 %}
 7323 
 7324 // Store Float
 7325 instruct storeF(vRegF src, memory4 mem)
 7326 %{
 7327   match(Set mem (StoreF mem src));
 7328   predicate(!needs_releasing_store(n));
 7329 
 7330   ins_cost(INSN_COST);
 7331   format %{ "strs  $src, $mem\t# float" %}
 7332 
 7333   ins_encode( aarch64_enc_strs(src, mem) );
 7334 
 7335   ins_pipe(pipe_class_memory);
 7336 %}
 7337 
 7338 // TODO
 7339 // implement storeImmF0 and storeFImmPacked
 7340 
 7341 // Store Double
 7342 instruct storeD(vRegD src, memory8 mem)
 7343 %{
 7344   match(Set mem (StoreD mem src));
 7345   predicate(!needs_releasing_store(n));
 7346 
 7347   ins_cost(INSN_COST);
 7348   format %{ "strd  $src, $mem\t# double" %}
 7349 
 7350   ins_encode( aarch64_enc_strd(src, mem) );
 7351 
 7352   ins_pipe(pipe_class_memory);
 7353 %}
 7354 
 7355 // Store Compressed Klass Pointer
 7356 instruct storeNKlass(iRegN src, memory4 mem)
 7357 %{
 7358   predicate(!needs_releasing_store(n));
 7359   match(Set mem (StoreNKlass mem src));
 7360 
 7361   ins_cost(INSN_COST);
 7362   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7363 
 7364   ins_encode(aarch64_enc_strw(src, mem));
 7365 
 7366   ins_pipe(istore_reg_mem);
 7367 %}
 7368 
 7369 // TODO
 7370 // implement storeImmD0 and storeDImmPacked
 7371 
 7372 // prefetch instructions
 7373 // Must be safe to execute with invalid address (cannot fault).
 7374 
 7375 instruct prefetchalloc( memory8 mem ) %{
 7376   match(PrefetchAllocation mem);
 7377 
 7378   ins_cost(INSN_COST);
 7379   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7380 
 7381   ins_encode( aarch64_enc_prefetchw(mem) );
 7382 
 7383   ins_pipe(iload_prefetch);
 7384 %}
 7385 
 7386 //  ---------------- volatile loads and stores ----------------
 7387 
 7388 // Load Byte (8 bit signed)
 7389 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7390 %{
 7391   match(Set dst (LoadB mem));
 7392 
 7393   ins_cost(VOLATILE_REF_COST);
 7394   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7395 
 7396   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7397 
 7398   ins_pipe(pipe_serial);
 7399 %}
 7400 
 7401 // Load Byte (8 bit signed) into long
 7402 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7403 %{
 7404   match(Set dst (ConvI2L (LoadB mem)));
 7405 
 7406   ins_cost(VOLATILE_REF_COST);
 7407   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7408 
 7409   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7410 
 7411   ins_pipe(pipe_serial);
 7412 %}
 7413 
 7414 // Load Byte (8 bit unsigned)
 7415 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7416 %{
 7417   match(Set dst (LoadUB mem));
 7418 
 7419   ins_cost(VOLATILE_REF_COST);
 7420   format %{ "ldarb  $dst, $mem\t# byte" %}
 7421 
 7422   ins_encode(aarch64_enc_ldarb(dst, mem));
 7423 
 7424   ins_pipe(pipe_serial);
 7425 %}
 7426 
 7427 // Load Byte (8 bit unsigned) into long
 7428 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7429 %{
 7430   match(Set dst (ConvI2L (LoadUB mem)));
 7431 
 7432   ins_cost(VOLATILE_REF_COST);
 7433   format %{ "ldarb  $dst, $mem\t# byte" %}
 7434 
 7435   ins_encode(aarch64_enc_ldarb(dst, mem));
 7436 
 7437   ins_pipe(pipe_serial);
 7438 %}
 7439 
 7440 // Load Short (16 bit signed)
 7441 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7442 %{
 7443   match(Set dst (LoadS mem));
 7444 
 7445   ins_cost(VOLATILE_REF_COST);
 7446   format %{ "ldarshw  $dst, $mem\t# short" %}
 7447 
 7448   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7449 
 7450   ins_pipe(pipe_serial);
 7451 %}
 7452 
 7453 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7454 %{
 7455   match(Set dst (LoadUS mem));
 7456 
 7457   ins_cost(VOLATILE_REF_COST);
 7458   format %{ "ldarhw  $dst, $mem\t# short" %}
 7459 
 7460   ins_encode(aarch64_enc_ldarhw(dst, mem));
 7461 
 7462   ins_pipe(pipe_serial);
 7463 %}
 7464 
 7465 // Load Short/Char (16 bit unsigned) into long
 7466 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7467 %{
 7468   match(Set dst (ConvI2L (LoadUS mem)));
 7469 
 7470   ins_cost(VOLATILE_REF_COST);
 7471   format %{ "ldarh  $dst, $mem\t# short" %}
 7472 
 7473   ins_encode(aarch64_enc_ldarh(dst, mem));
 7474 
 7475   ins_pipe(pipe_serial);
 7476 %}
 7477 
 7478 // Load Short/Char (16 bit signed) into long
 7479 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7480 %{
 7481   match(Set dst (ConvI2L (LoadS mem)));
 7482 
 7483   ins_cost(VOLATILE_REF_COST);
 7484   format %{ "ldarh  $dst, $mem\t# short" %}
 7485 
 7486   ins_encode(aarch64_enc_ldarsh(dst, mem));
 7487 
 7488   ins_pipe(pipe_serial);
 7489 %}
 7490 
 7491 // Load Integer (32 bit signed)
 7492 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7493 %{
 7494   match(Set dst (LoadI mem));
 7495 
 7496   ins_cost(VOLATILE_REF_COST);
 7497   format %{ "ldarw  $dst, $mem\t# int" %}
 7498 
 7499   ins_encode(aarch64_enc_ldarw(dst, mem));
 7500 
 7501   ins_pipe(pipe_serial);
 7502 %}
 7503 
 7504 // Load Integer (32 bit unsigned) into long
 7505 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 7506 %{
 7507   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7508 
 7509   ins_cost(VOLATILE_REF_COST);
 7510   format %{ "ldarw  $dst, $mem\t# int" %}
 7511 
 7512   ins_encode(aarch64_enc_ldarw(dst, mem));
 7513 
 7514   ins_pipe(pipe_serial);
 7515 %}
 7516 
 7517 // Load Long (64 bit signed)
 7518 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7519 %{
 7520   match(Set dst (LoadL mem));
 7521 
 7522   ins_cost(VOLATILE_REF_COST);
 7523   format %{ "ldar  $dst, $mem\t# int" %}
 7524 
 7525   ins_encode(aarch64_enc_ldar(dst, mem));
 7526 
 7527   ins_pipe(pipe_serial);
 7528 %}
 7529 
 7530 // Load Pointer
 7531 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 7532 %{
 7533   match(Set dst (LoadP mem));
 7534   predicate(n->as_Load()->barrier_data() == 0);
 7535 
 7536   ins_cost(VOLATILE_REF_COST);
 7537   format %{ "ldar  $dst, $mem\t# ptr" %}
 7538 
 7539   ins_encode(aarch64_enc_ldar(dst, mem));
 7540 
 7541   ins_pipe(pipe_serial);
 7542 %}
 7543 
 7544 // Load Compressed Pointer
 7545 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 7546 %{
 7547   match(Set dst (LoadN mem));
 7548   predicate(n->as_Load()->barrier_data() == 0);
 7549 
 7550   ins_cost(VOLATILE_REF_COST);
 7551   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 7552 
 7553   ins_encode(aarch64_enc_ldarw(dst, mem));
 7554 
 7555   ins_pipe(pipe_serial);
 7556 %}
 7557 
 7558 // Load Float
 7559 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 7560 %{
 7561   match(Set dst (LoadF mem));
 7562 
 7563   ins_cost(VOLATILE_REF_COST);
 7564   format %{ "ldars  $dst, $mem\t# float" %}
 7565 
 7566   ins_encode( aarch64_enc_fldars(dst, mem) );
 7567 
 7568   ins_pipe(pipe_serial);
 7569 %}
 7570 
 7571 // Load Double
 7572 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 7573 %{
 7574   match(Set dst (LoadD mem));
 7575 
 7576   ins_cost(VOLATILE_REF_COST);
 7577   format %{ "ldard  $dst, $mem\t# double" %}
 7578 
 7579   ins_encode( aarch64_enc_fldard(dst, mem) );
 7580 
 7581   ins_pipe(pipe_serial);
 7582 %}
 7583 
 7584 // Store Byte
 7585 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7586 %{
 7587   match(Set mem (StoreB mem src));
 7588 
 7589   ins_cost(VOLATILE_REF_COST);
 7590   format %{ "stlrb  $src, $mem\t# byte" %}
 7591 
 7592   ins_encode(aarch64_enc_stlrb(src, mem));
 7593 
 7594   ins_pipe(pipe_class_memory);
 7595 %}
 7596 
 7597 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7598 %{
 7599   match(Set mem (StoreB mem zero));
 7600 
 7601   ins_cost(VOLATILE_REF_COST);
 7602   format %{ "stlrb  zr, $mem\t# byte" %}
 7603 
 7604   ins_encode(aarch64_enc_stlrb0(mem));
 7605 
 7606   ins_pipe(pipe_class_memory);
 7607 %}
 7608 
 7609 // Store Char/Short
 7610 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7611 %{
 7612   match(Set mem (StoreC mem src));
 7613 
 7614   ins_cost(VOLATILE_REF_COST);
 7615   format %{ "stlrh  $src, $mem\t# short" %}
 7616 
 7617   ins_encode(aarch64_enc_stlrh(src, mem));
 7618 
 7619   ins_pipe(pipe_class_memory);
 7620 %}
 7621 
 7622 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7623 %{
 7624   match(Set mem (StoreC mem zero));
 7625 
 7626   ins_cost(VOLATILE_REF_COST);
 7627   format %{ "stlrh  zr, $mem\t# short" %}
 7628 
 7629   ins_encode(aarch64_enc_stlrh0(mem));
 7630 
 7631   ins_pipe(pipe_class_memory);
 7632 %}
 7633 
 7634 // Store Integer
 7635 
 7636 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7637 %{
 7638   match(Set mem(StoreI mem src));
 7639 
 7640   ins_cost(VOLATILE_REF_COST);
 7641   format %{ "stlrw  $src, $mem\t# int" %}
 7642 
 7643   ins_encode(aarch64_enc_stlrw(src, mem));
 7644 
 7645   ins_pipe(pipe_class_memory);
 7646 %}
 7647 
 7648 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7649 %{
 7650   match(Set mem(StoreI mem zero));
 7651 
 7652   ins_cost(VOLATILE_REF_COST);
 7653   format %{ "stlrw  zr, $mem\t# int" %}
 7654 
 7655   ins_encode(aarch64_enc_stlrw0(mem));
 7656 
 7657   ins_pipe(pipe_class_memory);
 7658 %}
 7659 
 7660 // Store Long (64 bit signed)
 7661 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 7662 %{
 7663   match(Set mem (StoreL mem src));
 7664 
 7665   ins_cost(VOLATILE_REF_COST);
 7666   format %{ "stlr  $src, $mem\t# int" %}
 7667 
 7668   ins_encode(aarch64_enc_stlr(src, mem));
 7669 
 7670   ins_pipe(pipe_class_memory);
 7671 %}
 7672 
 7673 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 7674 %{
 7675   match(Set mem (StoreL mem zero));
 7676 
 7677   ins_cost(VOLATILE_REF_COST);
 7678   format %{ "stlr  zr, $mem\t# int" %}
 7679 
 7680   ins_encode(aarch64_enc_stlr0(mem));
 7681 
 7682   ins_pipe(pipe_class_memory);
 7683 %}
 7684 
 7685 // Store Pointer
 7686 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 7687 %{
 7688   match(Set mem (StoreP mem src));
 7689   predicate(n->as_Store()->barrier_data() == 0);
 7690 
 7691   ins_cost(VOLATILE_REF_COST);
 7692   format %{ "stlr  $src, $mem\t# ptr" %}
 7693 
 7694   ins_encode(aarch64_enc_stlr(src, mem));
 7695 
 7696   ins_pipe(pipe_class_memory);
 7697 %}
 7698 
 7699 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 7700 %{
 7701   match(Set mem (StoreP mem zero));
 7702   predicate(n->as_Store()->barrier_data() == 0);
 7703 
 7704   ins_cost(VOLATILE_REF_COST);
 7705   format %{ "stlr  zr, $mem\t# ptr" %}
 7706 
 7707   ins_encode(aarch64_enc_stlr0(mem));
 7708 
 7709   ins_pipe(pipe_class_memory);
 7710 %}
 7711 
 7712 // Store Compressed Pointer
 7713 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 7714 %{
 7715   match(Set mem (StoreN mem src));
 7716   predicate(n->as_Store()->barrier_data() == 0);
 7717 
 7718   ins_cost(VOLATILE_REF_COST);
 7719   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 7720 
 7721   ins_encode(aarch64_enc_stlrw(src, mem));
 7722 
 7723   ins_pipe(pipe_class_memory);
 7724 %}
 7725 
 7726 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 7727 %{
 7728   match(Set mem (StoreN mem zero));
 7729   predicate(n->as_Store()->barrier_data() == 0);
 7730 
 7731   ins_cost(VOLATILE_REF_COST);
 7732   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 7733 
 7734   ins_encode(aarch64_enc_stlrw0(mem));
 7735 
 7736   ins_pipe(pipe_class_memory);
 7737 %}
 7738 
 7739 // Store Float
 7740 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 7741 %{
 7742   match(Set mem (StoreF mem src));
 7743 
 7744   ins_cost(VOLATILE_REF_COST);
 7745   format %{ "stlrs  $src, $mem\t# float" %}
 7746 
 7747   ins_encode( aarch64_enc_fstlrs(src, mem) );
 7748 
 7749   ins_pipe(pipe_class_memory);
 7750 %}
 7751 
 7752 // TODO
 7753 // implement storeImmF0 and storeFImmPacked
 7754 
 7755 // Store Double
 7756 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 7757 %{
 7758   match(Set mem (StoreD mem src));
 7759 
 7760   ins_cost(VOLATILE_REF_COST);
 7761   format %{ "stlrd  $src, $mem\t# double" %}
 7762 
 7763   ins_encode( aarch64_enc_fstlrd(src, mem) );
 7764 
 7765   ins_pipe(pipe_class_memory);
 7766 %}
 7767 
 7768 //  ---------------- end of volatile loads and stores ----------------
 7769 
 7770 instruct cacheWB(indirect addr)
 7771 %{
 7772   predicate(VM_Version::supports_data_cache_line_flush());
 7773   match(CacheWB addr);
 7774 
 7775   ins_cost(100);
 7776   format %{"cache wb $addr" %}
 7777   ins_encode %{
 7778     assert($addr->index_position() < 0, "should be");
 7779     assert($addr$$disp == 0, "should be");
 7780     __ cache_wb(Address($addr$$base$$Register, 0));
 7781   %}
 7782   ins_pipe(pipe_slow); // XXX
 7783 %}
 7784 
 7785 instruct cacheWBPreSync()
 7786 %{
 7787   predicate(VM_Version::supports_data_cache_line_flush());
 7788   match(CacheWBPreSync);
 7789 
 7790   ins_cost(100);
 7791   format %{"cache wb presync" %}
 7792   ins_encode %{
 7793     __ cache_wbsync(true);
 7794   %}
 7795   ins_pipe(pipe_slow); // XXX
 7796 %}
 7797 
 7798 instruct cacheWBPostSync()
 7799 %{
 7800   predicate(VM_Version::supports_data_cache_line_flush());
 7801   match(CacheWBPostSync);
 7802 
 7803   ins_cost(100);
 7804   format %{"cache wb postsync" %}
 7805   ins_encode %{
 7806     __ cache_wbsync(false);
 7807   %}
 7808   ins_pipe(pipe_slow); // XXX
 7809 %}
 7810 
 7811 // ============================================================================
 7812 // BSWAP Instructions
 7813 
 7814 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 7815   match(Set dst (ReverseBytesI src));
 7816 
 7817   ins_cost(INSN_COST);
 7818   format %{ "revw  $dst, $src" %}
 7819 
 7820   ins_encode %{
 7821     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 7822   %}
 7823 
 7824   ins_pipe(ialu_reg);
 7825 %}
 7826 
 7827 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 7828   match(Set dst (ReverseBytesL src));
 7829 
 7830   ins_cost(INSN_COST);
 7831   format %{ "rev  $dst, $src" %}
 7832 
 7833   ins_encode %{
 7834     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 7835   %}
 7836 
 7837   ins_pipe(ialu_reg);
 7838 %}
 7839 
 7840 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 7841   match(Set dst (ReverseBytesUS src));
 7842 
 7843   ins_cost(INSN_COST);
 7844   format %{ "rev16w  $dst, $src" %}
 7845 
 7846   ins_encode %{
 7847     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 7848   %}
 7849 
 7850   ins_pipe(ialu_reg);
 7851 %}
 7852 
 7853 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 7854   match(Set dst (ReverseBytesS src));
 7855 
 7856   ins_cost(INSN_COST);
 7857   format %{ "rev16w  $dst, $src\n\t"
 7858             "sbfmw $dst, $dst, #0, #15" %}
 7859 
 7860   ins_encode %{
 7861     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 7862     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 7863   %}
 7864 
 7865   ins_pipe(ialu_reg);
 7866 %}
 7867 
 7868 // ============================================================================
 7869 // Zero Count Instructions
 7870 
 7871 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 7872   match(Set dst (CountLeadingZerosI src));
 7873 
 7874   ins_cost(INSN_COST);
 7875   format %{ "clzw  $dst, $src" %}
 7876   ins_encode %{
 7877     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 7878   %}
 7879 
 7880   ins_pipe(ialu_reg);
 7881 %}
 7882 
 7883 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 7884   match(Set dst (CountLeadingZerosL src));
 7885 
 7886   ins_cost(INSN_COST);
 7887   format %{ "clz   $dst, $src" %}
 7888   ins_encode %{
 7889     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 7890   %}
 7891 
 7892   ins_pipe(ialu_reg);
 7893 %}
 7894 
 7895 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 7896   match(Set dst (CountTrailingZerosI src));
 7897 
 7898   ins_cost(INSN_COST * 2);
 7899   format %{ "rbitw  $dst, $src\n\t"
 7900             "clzw   $dst, $dst" %}
 7901   ins_encode %{
 7902     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 7903     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 7904   %}
 7905 
 7906   ins_pipe(ialu_reg);
 7907 %}
 7908 
 7909 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 7910   match(Set dst (CountTrailingZerosL src));
 7911 
 7912   ins_cost(INSN_COST * 2);
 7913   format %{ "rbit   $dst, $src\n\t"
 7914             "clz    $dst, $dst" %}
 7915   ins_encode %{
 7916     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 7917     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 7918   %}
 7919 
 7920   ins_pipe(ialu_reg);
 7921 %}
 7922 
 7923 //---------- Population Count Instructions -------------------------------------
 7924 //
 7925 
 7926 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 7927   match(Set dst (PopCountI src));
 7928   effect(TEMP tmp);
 7929   ins_cost(INSN_COST * 13);
 7930 
 7931   format %{ "fmovs  $tmp, $src\t# vector (1S)\n\t"
 7932             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7933             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7934             "mov    $dst, $tmp\t# vector (1D)" %}
 7935   ins_encode %{
 7936     __ fmovs($tmp$$FloatRegister, $src$$Register);
 7937     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7938     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7939     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7940   %}
 7941 
 7942   ins_pipe(pipe_class_default);
 7943 %}
 7944 
 7945 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 7946   match(Set dst (PopCountI (LoadI mem)));
 7947   effect(TEMP tmp);
 7948   ins_cost(INSN_COST * 13);
 7949 
 7950   format %{ "ldrs   $tmp, $mem\n\t"
 7951             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7952             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7953             "mov    $dst, $tmp\t# vector (1D)" %}
 7954   ins_encode %{
 7955     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 7956     loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 7957               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 7958     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7959     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7960     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7961   %}
 7962 
 7963   ins_pipe(pipe_class_default);
 7964 %}
 7965 
 7966 // Note: Long.bitCount(long) returns an int.
 7967 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 7968   match(Set dst (PopCountL src));
 7969   effect(TEMP tmp);
 7970   ins_cost(INSN_COST * 13);
 7971 
 7972   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 7973             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7974             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7975             "mov    $dst, $tmp\t# vector (1D)" %}
 7976   ins_encode %{
 7977     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 7978     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7979     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7980     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7981   %}
 7982 
 7983   ins_pipe(pipe_class_default);
 7984 %}
 7985 
 7986 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 7987   match(Set dst (PopCountL (LoadL mem)));
 7988   effect(TEMP tmp);
 7989   ins_cost(INSN_COST * 13);
 7990 
 7991   format %{ "ldrd   $tmp, $mem\n\t"
 7992             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7993             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7994             "mov    $dst, $tmp\t# vector (1D)" %}
 7995   ins_encode %{
 7996     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 7997     loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 7998               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 7999     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8000     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8001     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8002   %}
 8003 
 8004   ins_pipe(pipe_class_default);
 8005 %}
 8006 
 8007 // ============================================================================
 8008 // VerifyVectorAlignment Instruction
 8009 
 8010 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
 8011   match(Set addr (VerifyVectorAlignment addr mask));
 8012   effect(KILL cr);
 8013   format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
 8014   ins_encode %{
 8015     Label Lskip;
 8016     // check if masked bits of addr are zero
 8017     __ tst($addr$$Register, $mask$$constant);
 8018     __ br(Assembler::EQ, Lskip);
 8019     __ stop("verify_vector_alignment found a misaligned vector memory access");
 8020     __ bind(Lskip);
 8021   %}
 8022   ins_pipe(pipe_slow);
 8023 %}
 8024 
 8025 // ============================================================================
 8026 // MemBar Instruction
 8027 
 8028 instruct load_fence() %{
 8029   match(LoadFence);
 8030   ins_cost(VOLATILE_REF_COST);
 8031 
 8032   format %{ "load_fence" %}
 8033 
 8034   ins_encode %{
 8035     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8036   %}
 8037   ins_pipe(pipe_serial);
 8038 %}
 8039 
 8040 instruct unnecessary_membar_acquire() %{
 8041   predicate(unnecessary_acquire(n));
 8042   match(MemBarAcquire);
 8043   ins_cost(0);
 8044 
 8045   format %{ "membar_acquire (elided)" %}
 8046 
 8047   ins_encode %{
 8048     __ block_comment("membar_acquire (elided)");
 8049   %}
 8050 
 8051   ins_pipe(pipe_class_empty);
 8052 %}
 8053 
 8054 instruct membar_acquire() %{
 8055   match(MemBarAcquire);
 8056   ins_cost(VOLATILE_REF_COST);
 8057 
 8058   format %{ "membar_acquire\n\t"
 8059             "dmb ishld" %}
 8060 
 8061   ins_encode %{
 8062     __ block_comment("membar_acquire");
 8063     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8064   %}
 8065 
 8066   ins_pipe(pipe_serial);
 8067 %}
 8068 
 8069 
 8070 instruct membar_acquire_lock() %{
 8071   match(MemBarAcquireLock);
 8072   ins_cost(VOLATILE_REF_COST);
 8073 
 8074   format %{ "membar_acquire_lock (elided)" %}
 8075 
 8076   ins_encode %{
 8077     __ block_comment("membar_acquire_lock (elided)");
 8078   %}
 8079 
 8080   ins_pipe(pipe_serial);
 8081 %}
 8082 
 8083 instruct store_fence() %{
 8084   match(StoreFence);
 8085   ins_cost(VOLATILE_REF_COST);
 8086 
 8087   format %{ "store_fence" %}
 8088 
 8089   ins_encode %{
 8090     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8091   %}
 8092   ins_pipe(pipe_serial);
 8093 %}
 8094 
 8095 instruct unnecessary_membar_release() %{
 8096   predicate(unnecessary_release(n));
 8097   match(MemBarRelease);
 8098   ins_cost(0);
 8099 
 8100   format %{ "membar_release (elided)" %}
 8101 
 8102   ins_encode %{
 8103     __ block_comment("membar_release (elided)");
 8104   %}
 8105   ins_pipe(pipe_serial);
 8106 %}
 8107 
 8108 instruct membar_release() %{
 8109   match(MemBarRelease);
 8110   ins_cost(VOLATILE_REF_COST);
 8111 
 8112   format %{ "membar_release\n\t"
 8113             "dmb ishst\n\tdmb ishld" %}
 8114 
 8115   ins_encode %{
 8116     __ block_comment("membar_release");
 8117     // These will be merged if AlwaysMergeDMB is enabled.
 8118     __ membar(Assembler::StoreStore);
 8119     __ membar(Assembler::LoadStore);
 8120   %}
 8121   ins_pipe(pipe_serial);
 8122 %}
 8123 
 8124 instruct membar_storestore() %{
 8125   match(MemBarStoreStore);
 8126   match(StoreStoreFence);
 8127   ins_cost(VOLATILE_REF_COST);
 8128 
 8129   format %{ "MEMBAR-store-store" %}
 8130 
 8131   ins_encode %{
 8132     __ membar(Assembler::StoreStore);
 8133   %}
 8134   ins_pipe(pipe_serial);
 8135 %}
 8136 
 8137 instruct membar_release_lock() %{
 8138   match(MemBarReleaseLock);
 8139   ins_cost(VOLATILE_REF_COST);
 8140 
 8141   format %{ "membar_release_lock (elided)" %}
 8142 
 8143   ins_encode %{
 8144     __ block_comment("membar_release_lock (elided)");
 8145   %}
 8146 
 8147   ins_pipe(pipe_serial);
 8148 %}
 8149 
 8150 instruct unnecessary_membar_volatile() %{
 8151   predicate(unnecessary_volatile(n));
 8152   match(MemBarVolatile);
 8153   ins_cost(0);
 8154 
 8155   format %{ "membar_volatile (elided)" %}
 8156 
 8157   ins_encode %{
 8158     __ block_comment("membar_volatile (elided)");
 8159   %}
 8160 
 8161   ins_pipe(pipe_serial);
 8162 %}
 8163 
 8164 instruct membar_volatile() %{
 8165   match(MemBarVolatile);
 8166   ins_cost(VOLATILE_REF_COST*100);
 8167 
 8168   format %{ "membar_volatile\n\t"
 8169              "dmb ish"%}
 8170 
 8171   ins_encode %{
 8172     __ block_comment("membar_volatile");
 8173     __ membar(Assembler::StoreLoad);
 8174   %}
 8175 
 8176   ins_pipe(pipe_serial);
 8177 %}
 8178 
 8179 // ============================================================================
 8180 // Cast/Convert Instructions
 8181 
 8182 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8183   match(Set dst (CastX2P src));
 8184 
 8185   ins_cost(INSN_COST);
 8186   format %{ "mov $dst, $src\t# long -> ptr" %}
 8187 
 8188   ins_encode %{
 8189     if ($dst$$reg != $src$$reg) {
 8190       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8191     }
 8192   %}
 8193 
 8194   ins_pipe(ialu_reg);
 8195 %}
 8196 
 8197 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8198   match(Set dst (CastP2X src));
 8199 
 8200   ins_cost(INSN_COST);
 8201   format %{ "mov $dst, $src\t# ptr -> long" %}
 8202 
 8203   ins_encode %{
 8204     if ($dst$$reg != $src$$reg) {
 8205       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8206     }
 8207   %}
 8208 
 8209   ins_pipe(ialu_reg);
 8210 %}
 8211 
 8212 // Convert oop into int for vectors alignment masking
 8213 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8214   match(Set dst (ConvL2I (CastP2X src)));
 8215 
 8216   ins_cost(INSN_COST);
 8217   format %{ "movw $dst, $src\t# ptr -> int" %}
 8218   ins_encode %{
 8219     __ movw($dst$$Register, $src$$Register);
 8220   %}
 8221 
 8222   ins_pipe(ialu_reg);
 8223 %}
 8224 
 8225 // Convert compressed oop into int for vectors alignment masking
 8226 // in case of 32bit oops (heap < 4Gb).
 8227 instruct convN2I(iRegINoSp dst, iRegN src)
 8228 %{
 8229   predicate(CompressedOops::shift() == 0);
 8230   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8231 
 8232   ins_cost(INSN_COST);
 8233   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8234   ins_encode %{
 8235     __ movw($dst$$Register, $src$$Register);
 8236   %}
 8237 
 8238   ins_pipe(ialu_reg);
 8239 %}
 8240 
 8241 
 8242 // Convert oop pointer into compressed form
 8243 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8244   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8245   match(Set dst (EncodeP src));
 8246   effect(KILL cr);
 8247   ins_cost(INSN_COST * 3);
 8248   format %{ "encode_heap_oop $dst, $src" %}
 8249   ins_encode %{
 8250     Register s = $src$$Register;
 8251     Register d = $dst$$Register;
 8252     __ encode_heap_oop(d, s);
 8253   %}
 8254   ins_pipe(ialu_reg);
 8255 %}
 8256 
 8257 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8258   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8259   match(Set dst (EncodeP src));
 8260   ins_cost(INSN_COST * 3);
 8261   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8262   ins_encode %{
 8263     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8264   %}
 8265   ins_pipe(ialu_reg);
 8266 %}
 8267 
 8268 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8269   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8270             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8271   match(Set dst (DecodeN src));
 8272   ins_cost(INSN_COST * 3);
 8273   format %{ "decode_heap_oop $dst, $src" %}
 8274   ins_encode %{
 8275     Register s = $src$$Register;
 8276     Register d = $dst$$Register;
 8277     __ decode_heap_oop(d, s);
 8278   %}
 8279   ins_pipe(ialu_reg);
 8280 %}
 8281 
 8282 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8283   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8284             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8285   match(Set dst (DecodeN src));
 8286   ins_cost(INSN_COST * 3);
 8287   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8288   ins_encode %{
 8289     Register s = $src$$Register;
 8290     Register d = $dst$$Register;
 8291     __ decode_heap_oop_not_null(d, s);
 8292   %}
 8293   ins_pipe(ialu_reg);
 8294 %}
 8295 
 8296 // n.b. AArch64 implementations of encode_klass_not_null and
 8297 // decode_klass_not_null do not modify the flags register so, unlike
 8298 // Intel, we don't kill CR as a side effect here
 8299 
 8300 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8301   match(Set dst (EncodePKlass src));
 8302 
 8303   ins_cost(INSN_COST * 3);
 8304   format %{ "encode_klass_not_null $dst,$src" %}
 8305 
 8306   ins_encode %{
 8307     Register src_reg = as_Register($src$$reg);
 8308     Register dst_reg = as_Register($dst$$reg);
 8309     __ encode_klass_not_null(dst_reg, src_reg);
 8310   %}
 8311 
 8312    ins_pipe(ialu_reg);
 8313 %}
 8314 
 8315 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8316   match(Set dst (DecodeNKlass src));
 8317 
 8318   ins_cost(INSN_COST * 3);
 8319   format %{ "decode_klass_not_null $dst,$src" %}
 8320 
 8321   ins_encode %{
 8322     Register src_reg = as_Register($src$$reg);
 8323     Register dst_reg = as_Register($dst$$reg);
 8324     if (dst_reg != src_reg) {
 8325       __ decode_klass_not_null(dst_reg, src_reg);
 8326     } else {
 8327       __ decode_klass_not_null(dst_reg);
 8328     }
 8329   %}
 8330 
 8331    ins_pipe(ialu_reg);
 8332 %}
 8333 
 8334 instruct checkCastPP(iRegPNoSp dst)
 8335 %{
 8336   match(Set dst (CheckCastPP dst));
 8337 
 8338   size(0);
 8339   format %{ "# checkcastPP of $dst" %}
 8340   ins_encode(/* empty encoding */);
 8341   ins_pipe(pipe_class_empty);
 8342 %}
 8343 
 8344 instruct castPP(iRegPNoSp dst)
 8345 %{
 8346   match(Set dst (CastPP dst));
 8347 
 8348   size(0);
 8349   format %{ "# castPP of $dst" %}
 8350   ins_encode(/* empty encoding */);
 8351   ins_pipe(pipe_class_empty);
 8352 %}
 8353 
 8354 instruct castII(iRegI dst)
 8355 %{
 8356   predicate(VerifyConstraintCasts == 0);
 8357   match(Set dst (CastII dst));
 8358 
 8359   size(0);
 8360   format %{ "# castII of $dst" %}
 8361   ins_encode(/* empty encoding */);
 8362   ins_cost(0);
 8363   ins_pipe(pipe_class_empty);
 8364 %}
 8365 
 8366 instruct castII_checked(iRegI dst, rFlagsReg cr)
 8367 %{
 8368   predicate(VerifyConstraintCasts > 0);
 8369   match(Set dst (CastII dst));
 8370   effect(KILL cr);
 8371 
 8372   format %{ "# castII_checked of $dst" %}
 8373   ins_encode %{
 8374     __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
 8375   %}
 8376   ins_pipe(pipe_slow);
 8377 %}
 8378 
 8379 instruct castLL(iRegL dst)
 8380 %{
 8381   predicate(VerifyConstraintCasts == 0);
 8382   match(Set dst (CastLL dst));
 8383 
 8384   size(0);
 8385   format %{ "# castLL of $dst" %}
 8386   ins_encode(/* empty encoding */);
 8387   ins_cost(0);
 8388   ins_pipe(pipe_class_empty);
 8389 %}
 8390 
 8391 instruct castLL_checked(iRegL dst, rFlagsReg cr)
 8392 %{
 8393   predicate(VerifyConstraintCasts > 0);
 8394   match(Set dst (CastLL dst));
 8395   effect(KILL cr);
 8396 
 8397   format %{ "# castLL_checked of $dst" %}
 8398   ins_encode %{
 8399     __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
 8400   %}
 8401   ins_pipe(pipe_slow);
 8402 %}
 8403 
 8404 instruct castHH(vRegF dst)
 8405 %{
 8406   match(Set dst (CastHH dst));
 8407   size(0);
 8408   format %{ "# castHH of $dst" %}
 8409   ins_encode(/* empty encoding */);
 8410   ins_cost(0);
 8411   ins_pipe(pipe_class_empty);
 8412 %}
 8413 
 8414 instruct castFF(vRegF dst)
 8415 %{
 8416   match(Set dst (CastFF dst));
 8417 
 8418   size(0);
 8419   format %{ "# castFF of $dst" %}
 8420   ins_encode(/* empty encoding */);
 8421   ins_cost(0);
 8422   ins_pipe(pipe_class_empty);
 8423 %}
 8424 
 8425 instruct castDD(vRegD dst)
 8426 %{
 8427   match(Set dst (CastDD dst));
 8428 
 8429   size(0);
 8430   format %{ "# castDD of $dst" %}
 8431   ins_encode(/* empty encoding */);
 8432   ins_cost(0);
 8433   ins_pipe(pipe_class_empty);
 8434 %}
 8435 
 8436 instruct castVV(vReg dst)
 8437 %{
 8438   match(Set dst (CastVV dst));
 8439 
 8440   size(0);
 8441   format %{ "# castVV of $dst" %}
 8442   ins_encode(/* empty encoding */);
 8443   ins_cost(0);
 8444   ins_pipe(pipe_class_empty);
 8445 %}
 8446 
 8447 instruct castVVMask(pRegGov dst)
 8448 %{
 8449   match(Set dst (CastVV dst));
 8450 
 8451   size(0);
 8452   format %{ "# castVV of $dst" %}
 8453   ins_encode(/* empty encoding */);
 8454   ins_cost(0);
 8455   ins_pipe(pipe_class_empty);
 8456 %}
 8457 
 8458 // ============================================================================
 8459 // Atomic operation instructions
 8460 //
 8461 
 8462 // standard CompareAndSwapX when we are using barriers
 8463 // these have higher priority than the rules selected by a predicate
 8464 
 8465 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 8466 // can't match them
 8467 
 8468 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8469 
 8470   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8471   ins_cost(2 * VOLATILE_REF_COST);
 8472 
 8473   effect(KILL cr);
 8474 
 8475   format %{
 8476     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8477     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8478   %}
 8479 
 8480   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 8481             aarch64_enc_cset_eq(res));
 8482 
 8483   ins_pipe(pipe_slow);
 8484 %}
 8485 
 8486 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8487 
 8488   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8489   ins_cost(2 * VOLATILE_REF_COST);
 8490 
 8491   effect(KILL cr);
 8492 
 8493   format %{
 8494     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8495     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8496   %}
 8497 
 8498   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 8499             aarch64_enc_cset_eq(res));
 8500 
 8501   ins_pipe(pipe_slow);
 8502 %}
 8503 
 8504 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8505 
 8506   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8507   ins_cost(2 * VOLATILE_REF_COST);
 8508 
 8509   effect(KILL cr);
 8510 
 8511  format %{
 8512     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8513     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8514  %}
 8515 
 8516  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8517             aarch64_enc_cset_eq(res));
 8518 
 8519   ins_pipe(pipe_slow);
 8520 %}
 8521 
 8522 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8523 
 8524   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8525   ins_cost(2 * VOLATILE_REF_COST);
 8526 
 8527   effect(KILL cr);
 8528 
 8529  format %{
 8530     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8531     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8532  %}
 8533 
 8534  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8535             aarch64_enc_cset_eq(res));
 8536 
 8537   ins_pipe(pipe_slow);
 8538 %}
 8539 
 8540 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8541 
 8542   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8543   predicate(n->as_LoadStore()->barrier_data() == 0);
 8544   ins_cost(2 * VOLATILE_REF_COST);
 8545 
 8546   effect(KILL cr);
 8547 
 8548  format %{
 8549     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8550     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8551  %}
 8552 
 8553  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8554             aarch64_enc_cset_eq(res));
 8555 
 8556   ins_pipe(pipe_slow);
 8557 %}
 8558 
 8559 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8560 
 8561   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8562   predicate(n->as_LoadStore()->barrier_data() == 0);
 8563   ins_cost(2 * VOLATILE_REF_COST);
 8564 
 8565   effect(KILL cr);
 8566 
 8567  format %{
 8568     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8569     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8570  %}
 8571 
 8572  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8573             aarch64_enc_cset_eq(res));
 8574 
 8575   ins_pipe(pipe_slow);
 8576 %}
 8577 
 8578 // alternative CompareAndSwapX when we are eliding barriers
 8579 
 8580 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8581 
 8582   predicate(needs_acquiring_load_exclusive(n));
 8583   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8584   ins_cost(VOLATILE_REF_COST);
 8585 
 8586   effect(KILL cr);
 8587 
 8588   format %{
 8589     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8590     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8591   %}
 8592 
 8593   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 8594             aarch64_enc_cset_eq(res));
 8595 
 8596   ins_pipe(pipe_slow);
 8597 %}
 8598 
 8599 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8600 
 8601   predicate(needs_acquiring_load_exclusive(n));
 8602   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8603   ins_cost(VOLATILE_REF_COST);
 8604 
 8605   effect(KILL cr);
 8606 
 8607   format %{
 8608     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8609     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8610   %}
 8611 
 8612   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 8613             aarch64_enc_cset_eq(res));
 8614 
 8615   ins_pipe(pipe_slow);
 8616 %}
 8617 
 8618 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8619 
 8620   predicate(needs_acquiring_load_exclusive(n));
 8621   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8622   ins_cost(VOLATILE_REF_COST);
 8623 
 8624   effect(KILL cr);
 8625 
 8626  format %{
 8627     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8628     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8629  %}
 8630 
 8631  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8632             aarch64_enc_cset_eq(res));
 8633 
 8634   ins_pipe(pipe_slow);
 8635 %}
 8636 
 8637 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8638 
 8639   predicate(needs_acquiring_load_exclusive(n));
 8640   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8641   ins_cost(VOLATILE_REF_COST);
 8642 
 8643   effect(KILL cr);
 8644 
 8645  format %{
 8646     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8647     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8648  %}
 8649 
 8650  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8651             aarch64_enc_cset_eq(res));
 8652 
 8653   ins_pipe(pipe_slow);
 8654 %}
 8655 
 8656 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8657 
 8658   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8659   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8660   ins_cost(VOLATILE_REF_COST);
 8661 
 8662   effect(KILL cr);
 8663 
 8664  format %{
 8665     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8666     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8667  %}
 8668 
 8669  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8670             aarch64_enc_cset_eq(res));
 8671 
 8672   ins_pipe(pipe_slow);
 8673 %}
 8674 
 8675 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8676 
 8677   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 8678   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8679   ins_cost(VOLATILE_REF_COST);
 8680 
 8681   effect(KILL cr);
 8682 
 8683  format %{
 8684     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8685     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8686  %}
 8687 
 8688  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8689             aarch64_enc_cset_eq(res));
 8690 
 8691   ins_pipe(pipe_slow);
 8692 %}
 8693 
 8694 
 8695 // ---------------------------------------------------------------------
 8696 
 8697 // BEGIN This section of the file is automatically generated. Do not edit --------------
 8698 
 8699 // Sundry CAS operations.  Note that release is always true,
 8700 // regardless of the memory ordering of the CAS.  This is because we
 8701 // need the volatile case to be sequentially consistent but there is
 8702 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 8703 // can't check the type of memory ordering here, so we always emit a
 8704 // STLXR.
 8705 
 8706 // This section is generated from cas.m4
 8707 
 8708 
 8709 // This pattern is generated automatically from cas.m4.
 8710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8711 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8712   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8713   ins_cost(2 * VOLATILE_REF_COST);
 8714   effect(TEMP_DEF res, KILL cr);
 8715   format %{
 8716     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8717   %}
 8718   ins_encode %{
 8719     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8720                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8721                /*weak*/ false, $res$$Register);
 8722     __ sxtbw($res$$Register, $res$$Register);
 8723   %}
 8724   ins_pipe(pipe_slow);
 8725 %}
 8726 
 8727 // This pattern is generated automatically from cas.m4.
 8728 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8729 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8730   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 8731   ins_cost(2 * VOLATILE_REF_COST);
 8732   effect(TEMP_DEF res, KILL cr);
 8733   format %{
 8734     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8735   %}
 8736   ins_encode %{
 8737     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8738                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 8739                /*weak*/ false, $res$$Register);
 8740     __ sxthw($res$$Register, $res$$Register);
 8741   %}
 8742   ins_pipe(pipe_slow);
 8743 %}
 8744 
 8745 // This pattern is generated automatically from cas.m4.
 8746 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8747 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8748   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 8749   ins_cost(2 * VOLATILE_REF_COST);
 8750   effect(TEMP_DEF res, KILL cr);
 8751   format %{
 8752     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8753   %}
 8754   ins_encode %{
 8755     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8756                Assembler::word, /*acquire*/ false, /*release*/ true,
 8757                /*weak*/ false, $res$$Register);
 8758   %}
 8759   ins_pipe(pipe_slow);
 8760 %}
 8761 
 8762 // This pattern is generated automatically from cas.m4.
 8763 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8764 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8765   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 8766   ins_cost(2 * VOLATILE_REF_COST);
 8767   effect(TEMP_DEF res, KILL cr);
 8768   format %{
 8769     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8770   %}
 8771   ins_encode %{
 8772     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8773                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8774                /*weak*/ false, $res$$Register);
 8775   %}
 8776   ins_pipe(pipe_slow);
 8777 %}
 8778 
 8779 // This pattern is generated automatically from cas.m4.
 8780 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8781 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8782   predicate(n->as_LoadStore()->barrier_data() == 0);
 8783   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 8784   ins_cost(2 * VOLATILE_REF_COST);
 8785   effect(TEMP_DEF res, KILL cr);
 8786   format %{
 8787     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8788   %}
 8789   ins_encode %{
 8790     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8791                Assembler::word, /*acquire*/ false, /*release*/ true,
 8792                /*weak*/ false, $res$$Register);
 8793   %}
 8794   ins_pipe(pipe_slow);
 8795 %}
 8796 
 8797 // This pattern is generated automatically from cas.m4.
 8798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8799 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8800   predicate(n->as_LoadStore()->barrier_data() == 0);
 8801   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 8802   ins_cost(2 * VOLATILE_REF_COST);
 8803   effect(TEMP_DEF res, KILL cr);
 8804   format %{
 8805     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8806   %}
 8807   ins_encode %{
 8808     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8809                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8810                /*weak*/ false, $res$$Register);
 8811   %}
 8812   ins_pipe(pipe_slow);
 8813 %}
 8814 
 8815 // This pattern is generated automatically from cas.m4.
 8816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8817 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8818   predicate(needs_acquiring_load_exclusive(n));
 8819   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8820   ins_cost(VOLATILE_REF_COST);
 8821   effect(TEMP_DEF res, KILL cr);
 8822   format %{
 8823     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8824   %}
 8825   ins_encode %{
 8826     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8827                Assembler::byte, /*acquire*/ true, /*release*/ true,
 8828                /*weak*/ false, $res$$Register);
 8829     __ sxtbw($res$$Register, $res$$Register);
 8830   %}
 8831   ins_pipe(pipe_slow);
 8832 %}
 8833 
 8834 // This pattern is generated automatically from cas.m4.
 8835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8836 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8837   predicate(needs_acquiring_load_exclusive(n));
 8838   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 8839   ins_cost(VOLATILE_REF_COST);
 8840   effect(TEMP_DEF res, KILL cr);
 8841   format %{
 8842     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8843   %}
 8844   ins_encode %{
 8845     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8846                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 8847                /*weak*/ false, $res$$Register);
 8848     __ sxthw($res$$Register, $res$$Register);
 8849   %}
 8850   ins_pipe(pipe_slow);
 8851 %}
 8852 
 8853 // This pattern is generated automatically from cas.m4.
 8854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8855 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8856   predicate(needs_acquiring_load_exclusive(n));
 8857   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 8858   ins_cost(VOLATILE_REF_COST);
 8859   effect(TEMP_DEF res, KILL cr);
 8860   format %{
 8861     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8862   %}
 8863   ins_encode %{
 8864     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8865                Assembler::word, /*acquire*/ true, /*release*/ true,
 8866                /*weak*/ false, $res$$Register);
 8867   %}
 8868   ins_pipe(pipe_slow);
 8869 %}
 8870 
 8871 // This pattern is generated automatically from cas.m4.
 8872 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8873 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8874   predicate(needs_acquiring_load_exclusive(n));
 8875   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 8876   ins_cost(VOLATILE_REF_COST);
 8877   effect(TEMP_DEF res, KILL cr);
 8878   format %{
 8879     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8880   %}
 8881   ins_encode %{
 8882     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8883                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8884                /*weak*/ false, $res$$Register);
 8885   %}
 8886   ins_pipe(pipe_slow);
 8887 %}
 8888 
 8889 // This pattern is generated automatically from cas.m4.
 8890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8891 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8892   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 8893   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 8894   ins_cost(VOLATILE_REF_COST);
 8895   effect(TEMP_DEF res, KILL cr);
 8896   format %{
 8897     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8898   %}
 8899   ins_encode %{
 8900     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8901                Assembler::word, /*acquire*/ true, /*release*/ true,
 8902                /*weak*/ false, $res$$Register);
 8903   %}
 8904   ins_pipe(pipe_slow);
 8905 %}
 8906 
 8907 // This pattern is generated automatically from cas.m4.
 8908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8909 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8910   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8911   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 8912   ins_cost(VOLATILE_REF_COST);
 8913   effect(TEMP_DEF res, KILL cr);
 8914   format %{
 8915     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8916   %}
 8917   ins_encode %{
 8918     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8919                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8920                /*weak*/ false, $res$$Register);
 8921   %}
 8922   ins_pipe(pipe_slow);
 8923 %}
 8924 
 8925 // This pattern is generated automatically from cas.m4.
 8926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8927 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8928   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 8929   ins_cost(2 * VOLATILE_REF_COST);
 8930   effect(KILL cr);
 8931   format %{
 8932     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8933     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8934   %}
 8935   ins_encode %{
 8936     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8937                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8938                /*weak*/ true, noreg);
 8939     __ csetw($res$$Register, Assembler::EQ);
 8940   %}
 8941   ins_pipe(pipe_slow);
 8942 %}
 8943 
 8944 // This pattern is generated automatically from cas.m4.
 8945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8946 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8947   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 8948   ins_cost(2 * VOLATILE_REF_COST);
 8949   effect(KILL cr);
 8950   format %{
 8951     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8952     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8953   %}
 8954   ins_encode %{
 8955     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8956                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 8957                /*weak*/ true, noreg);
 8958     __ csetw($res$$Register, Assembler::EQ);
 8959   %}
 8960   ins_pipe(pipe_slow);
 8961 %}
 8962 
 8963 // This pattern is generated automatically from cas.m4.
 8964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8965 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8966   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 8967   ins_cost(2 * VOLATILE_REF_COST);
 8968   effect(KILL cr);
 8969   format %{
 8970     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8971     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8972   %}
 8973   ins_encode %{
 8974     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8975                Assembler::word, /*acquire*/ false, /*release*/ true,
 8976                /*weak*/ true, noreg);
 8977     __ csetw($res$$Register, Assembler::EQ);
 8978   %}
 8979   ins_pipe(pipe_slow);
 8980 %}
 8981 
 8982 // This pattern is generated automatically from cas.m4.
 8983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8984 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8985   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 8986   ins_cost(2 * VOLATILE_REF_COST);
 8987   effect(KILL cr);
 8988   format %{
 8989     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8990     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8991   %}
 8992   ins_encode %{
 8993     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8994                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8995                /*weak*/ true, noreg);
 8996     __ csetw($res$$Register, Assembler::EQ);
 8997   %}
 8998   ins_pipe(pipe_slow);
 8999 %}
 9000 
 9001 // This pattern is generated automatically from cas.m4.
 9002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9003 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9004   predicate(n->as_LoadStore()->barrier_data() == 0);
 9005   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9006   ins_cost(2 * VOLATILE_REF_COST);
 9007   effect(KILL cr);
 9008   format %{
 9009     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9010     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9011   %}
 9012   ins_encode %{
 9013     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9014                Assembler::word, /*acquire*/ false, /*release*/ true,
 9015                /*weak*/ true, noreg);
 9016     __ csetw($res$$Register, Assembler::EQ);
 9017   %}
 9018   ins_pipe(pipe_slow);
 9019 %}
 9020 
 9021 // This pattern is generated automatically from cas.m4.
 9022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9023 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9024   predicate(n->as_LoadStore()->barrier_data() == 0);
 9025   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9026   ins_cost(2 * VOLATILE_REF_COST);
 9027   effect(KILL cr);
 9028   format %{
 9029     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9030     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9031   %}
 9032   ins_encode %{
 9033     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9034                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9035                /*weak*/ true, noreg);
 9036     __ csetw($res$$Register, Assembler::EQ);
 9037   %}
 9038   ins_pipe(pipe_slow);
 9039 %}
 9040 
 9041 // This pattern is generated automatically from cas.m4.
 9042 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9043 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9044   predicate(needs_acquiring_load_exclusive(n));
 9045   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9046   ins_cost(VOLATILE_REF_COST);
 9047   effect(KILL cr);
 9048   format %{
 9049     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9050     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9051   %}
 9052   ins_encode %{
 9053     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9054                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9055                /*weak*/ true, noreg);
 9056     __ csetw($res$$Register, Assembler::EQ);
 9057   %}
 9058   ins_pipe(pipe_slow);
 9059 %}
 9060 
 9061 // This pattern is generated automatically from cas.m4.
 9062 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9063 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9064   predicate(needs_acquiring_load_exclusive(n));
 9065   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9066   ins_cost(VOLATILE_REF_COST);
 9067   effect(KILL cr);
 9068   format %{
 9069     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9070     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9071   %}
 9072   ins_encode %{
 9073     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9074                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9075                /*weak*/ true, noreg);
 9076     __ csetw($res$$Register, Assembler::EQ);
 9077   %}
 9078   ins_pipe(pipe_slow);
 9079 %}
 9080 
 9081 // This pattern is generated automatically from cas.m4.
 9082 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9083 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9084   predicate(needs_acquiring_load_exclusive(n));
 9085   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9086   ins_cost(VOLATILE_REF_COST);
 9087   effect(KILL cr);
 9088   format %{
 9089     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9090     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9091   %}
 9092   ins_encode %{
 9093     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9094                Assembler::word, /*acquire*/ true, /*release*/ true,
 9095                /*weak*/ true, noreg);
 9096     __ csetw($res$$Register, Assembler::EQ);
 9097   %}
 9098   ins_pipe(pipe_slow);
 9099 %}
 9100 
 9101 // This pattern is generated automatically from cas.m4.
 9102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9103 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9104   predicate(needs_acquiring_load_exclusive(n));
 9105   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9106   ins_cost(VOLATILE_REF_COST);
 9107   effect(KILL cr);
 9108   format %{
 9109     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9110     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9111   %}
 9112   ins_encode %{
 9113     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9114                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9115                /*weak*/ true, noreg);
 9116     __ csetw($res$$Register, Assembler::EQ);
 9117   %}
 9118   ins_pipe(pipe_slow);
 9119 %}
 9120 
 9121 // This pattern is generated automatically from cas.m4.
 9122 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9123 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9124   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 9125   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9126   ins_cost(VOLATILE_REF_COST);
 9127   effect(KILL cr);
 9128   format %{
 9129     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9130     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9131   %}
 9132   ins_encode %{
 9133     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9134                Assembler::word, /*acquire*/ true, /*release*/ true,
 9135                /*weak*/ true, noreg);
 9136     __ csetw($res$$Register, Assembler::EQ);
 9137   %}
 9138   ins_pipe(pipe_slow);
 9139 %}
 9140 
 9141 // This pattern is generated automatically from cas.m4.
 9142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9143 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9144   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9145   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9146   ins_cost(VOLATILE_REF_COST);
 9147   effect(KILL cr);
 9148   format %{
 9149     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9150     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9151   %}
 9152   ins_encode %{
 9153     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9154                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9155                /*weak*/ true, noreg);
 9156     __ csetw($res$$Register, Assembler::EQ);
 9157   %}
 9158   ins_pipe(pipe_slow);
 9159 %}
 9160 
 9161 // END This section of the file is automatically generated. Do not edit --------------
 9162 // ---------------------------------------------------------------------
 9163 
 9164 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9165   match(Set prev (GetAndSetI mem newv));
 9166   ins_cost(2 * VOLATILE_REF_COST);
 9167   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9168   ins_encode %{
 9169     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9170   %}
 9171   ins_pipe(pipe_serial);
 9172 %}
 9173 
 9174 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9175   match(Set prev (GetAndSetL mem newv));
 9176   ins_cost(2 * VOLATILE_REF_COST);
 9177   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9178   ins_encode %{
 9179     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9180   %}
 9181   ins_pipe(pipe_serial);
 9182 %}
 9183 
 9184 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9185   predicate(n->as_LoadStore()->barrier_data() == 0);
 9186   match(Set prev (GetAndSetN mem newv));
 9187   ins_cost(2 * VOLATILE_REF_COST);
 9188   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9189   ins_encode %{
 9190     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9191   %}
 9192   ins_pipe(pipe_serial);
 9193 %}
 9194 
 9195 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9196   predicate(n->as_LoadStore()->barrier_data() == 0);
 9197   match(Set prev (GetAndSetP mem newv));
 9198   ins_cost(2 * VOLATILE_REF_COST);
 9199   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9200   ins_encode %{
 9201     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9202   %}
 9203   ins_pipe(pipe_serial);
 9204 %}
 9205 
 9206 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9207   predicate(needs_acquiring_load_exclusive(n));
 9208   match(Set prev (GetAndSetI mem newv));
 9209   ins_cost(VOLATILE_REF_COST);
 9210   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9211   ins_encode %{
 9212     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9213   %}
 9214   ins_pipe(pipe_serial);
 9215 %}
 9216 
 9217 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9218   predicate(needs_acquiring_load_exclusive(n));
 9219   match(Set prev (GetAndSetL mem newv));
 9220   ins_cost(VOLATILE_REF_COST);
 9221   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9222   ins_encode %{
 9223     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9224   %}
 9225   ins_pipe(pipe_serial);
 9226 %}
 9227 
 9228 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9229   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 9230   match(Set prev (GetAndSetN mem newv));
 9231   ins_cost(VOLATILE_REF_COST);
 9232   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9233   ins_encode %{
 9234     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9235   %}
 9236   ins_pipe(pipe_serial);
 9237 %}
 9238 
 9239 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9240   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9241   match(Set prev (GetAndSetP mem newv));
 9242   ins_cost(VOLATILE_REF_COST);
 9243   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9244   ins_encode %{
 9245     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9246   %}
 9247   ins_pipe(pipe_serial);
 9248 %}
 9249 
 9250 
 9251 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9252   match(Set newval (GetAndAddL mem incr));
 9253   ins_cost(2 * VOLATILE_REF_COST + 1);
 9254   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9255   ins_encode %{
 9256     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9257   %}
 9258   ins_pipe(pipe_serial);
 9259 %}
 9260 
 9261 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9262   predicate(n->as_LoadStore()->result_not_used());
 9263   match(Set dummy (GetAndAddL mem incr));
 9264   ins_cost(2 * VOLATILE_REF_COST);
 9265   format %{ "get_and_addL [$mem], $incr" %}
 9266   ins_encode %{
 9267     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9268   %}
 9269   ins_pipe(pipe_serial);
 9270 %}
 9271 
 9272 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9273   match(Set newval (GetAndAddL mem incr));
 9274   ins_cost(2 * VOLATILE_REF_COST + 1);
 9275   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9276   ins_encode %{
 9277     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9278   %}
 9279   ins_pipe(pipe_serial);
 9280 %}
 9281 
 9282 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9283   predicate(n->as_LoadStore()->result_not_used());
 9284   match(Set dummy (GetAndAddL mem incr));
 9285   ins_cost(2 * VOLATILE_REF_COST);
 9286   format %{ "get_and_addL [$mem], $incr" %}
 9287   ins_encode %{
 9288     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9289   %}
 9290   ins_pipe(pipe_serial);
 9291 %}
 9292 
 9293 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9294   match(Set newval (GetAndAddI mem incr));
 9295   ins_cost(2 * VOLATILE_REF_COST + 1);
 9296   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9297   ins_encode %{
 9298     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9299   %}
 9300   ins_pipe(pipe_serial);
 9301 %}
 9302 
 9303 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9304   predicate(n->as_LoadStore()->result_not_used());
 9305   match(Set dummy (GetAndAddI mem incr));
 9306   ins_cost(2 * VOLATILE_REF_COST);
 9307   format %{ "get_and_addI [$mem], $incr" %}
 9308   ins_encode %{
 9309     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9310   %}
 9311   ins_pipe(pipe_serial);
 9312 %}
 9313 
 9314 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9315   match(Set newval (GetAndAddI mem incr));
 9316   ins_cost(2 * VOLATILE_REF_COST + 1);
 9317   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9318   ins_encode %{
 9319     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9320   %}
 9321   ins_pipe(pipe_serial);
 9322 %}
 9323 
 9324 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9325   predicate(n->as_LoadStore()->result_not_used());
 9326   match(Set dummy (GetAndAddI mem incr));
 9327   ins_cost(2 * VOLATILE_REF_COST);
 9328   format %{ "get_and_addI [$mem], $incr" %}
 9329   ins_encode %{
 9330     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9331   %}
 9332   ins_pipe(pipe_serial);
 9333 %}
 9334 
 9335 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9336   predicate(needs_acquiring_load_exclusive(n));
 9337   match(Set newval (GetAndAddL mem incr));
 9338   ins_cost(VOLATILE_REF_COST + 1);
 9339   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9340   ins_encode %{
 9341     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9342   %}
 9343   ins_pipe(pipe_serial);
 9344 %}
 9345 
 9346 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9347   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9348   match(Set dummy (GetAndAddL mem incr));
 9349   ins_cost(VOLATILE_REF_COST);
 9350   format %{ "get_and_addL_acq [$mem], $incr" %}
 9351   ins_encode %{
 9352     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9353   %}
 9354   ins_pipe(pipe_serial);
 9355 %}
 9356 
 9357 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9358   predicate(needs_acquiring_load_exclusive(n));
 9359   match(Set newval (GetAndAddL mem incr));
 9360   ins_cost(VOLATILE_REF_COST + 1);
 9361   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9362   ins_encode %{
 9363     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9364   %}
 9365   ins_pipe(pipe_serial);
 9366 %}
 9367 
 9368 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9369   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9370   match(Set dummy (GetAndAddL mem incr));
 9371   ins_cost(VOLATILE_REF_COST);
 9372   format %{ "get_and_addL_acq [$mem], $incr" %}
 9373   ins_encode %{
 9374     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9375   %}
 9376   ins_pipe(pipe_serial);
 9377 %}
 9378 
 9379 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9380   predicate(needs_acquiring_load_exclusive(n));
 9381   match(Set newval (GetAndAddI mem incr));
 9382   ins_cost(VOLATILE_REF_COST + 1);
 9383   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9384   ins_encode %{
 9385     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9386   %}
 9387   ins_pipe(pipe_serial);
 9388 %}
 9389 
 9390 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9391   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9392   match(Set dummy (GetAndAddI mem incr));
 9393   ins_cost(VOLATILE_REF_COST);
 9394   format %{ "get_and_addI_acq [$mem], $incr" %}
 9395   ins_encode %{
 9396     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9397   %}
 9398   ins_pipe(pipe_serial);
 9399 %}
 9400 
 9401 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9402   predicate(needs_acquiring_load_exclusive(n));
 9403   match(Set newval (GetAndAddI mem incr));
 9404   ins_cost(VOLATILE_REF_COST + 1);
 9405   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9406   ins_encode %{
 9407     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9408   %}
 9409   ins_pipe(pipe_serial);
 9410 %}
 9411 
 9412 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9413   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9414   match(Set dummy (GetAndAddI mem incr));
 9415   ins_cost(VOLATILE_REF_COST);
 9416   format %{ "get_and_addI_acq [$mem], $incr" %}
 9417   ins_encode %{
 9418     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9419   %}
 9420   ins_pipe(pipe_serial);
 9421 %}
 9422 
 9423 // Manifest a CmpU result in an integer register.
 9424 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9425 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
 9426 %{
 9427   match(Set dst (CmpU3 src1 src2));
 9428   effect(KILL flags);
 9429 
 9430   ins_cost(INSN_COST * 3);
 9431   format %{
 9432       "cmpw $src1, $src2\n\t"
 9433       "csetw $dst, ne\n\t"
 9434       "cnegw $dst, lo\t# CmpU3(reg)"
 9435   %}
 9436   ins_encode %{
 9437     __ cmpw($src1$$Register, $src2$$Register);
 9438     __ csetw($dst$$Register, Assembler::NE);
 9439     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9440   %}
 9441 
 9442   ins_pipe(pipe_class_default);
 9443 %}
 9444 
 9445 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
 9446 %{
 9447   match(Set dst (CmpU3 src1 src2));
 9448   effect(KILL flags);
 9449 
 9450   ins_cost(INSN_COST * 3);
 9451   format %{
 9452       "subsw zr, $src1, $src2\n\t"
 9453       "csetw $dst, ne\n\t"
 9454       "cnegw $dst, lo\t# CmpU3(imm)"
 9455   %}
 9456   ins_encode %{
 9457     __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
 9458     __ csetw($dst$$Register, Assembler::NE);
 9459     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9460   %}
 9461 
 9462   ins_pipe(pipe_class_default);
 9463 %}
 9464 
 9465 // Manifest a CmpUL result in an integer register.
 9466 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9467 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9468 %{
 9469   match(Set dst (CmpUL3 src1 src2));
 9470   effect(KILL flags);
 9471 
 9472   ins_cost(INSN_COST * 3);
 9473   format %{
 9474       "cmp $src1, $src2\n\t"
 9475       "csetw $dst, ne\n\t"
 9476       "cnegw $dst, lo\t# CmpUL3(reg)"
 9477   %}
 9478   ins_encode %{
 9479     __ cmp($src1$$Register, $src2$$Register);
 9480     __ csetw($dst$$Register, Assembler::NE);
 9481     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9482   %}
 9483 
 9484   ins_pipe(pipe_class_default);
 9485 %}
 9486 
 9487 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9488 %{
 9489   match(Set dst (CmpUL3 src1 src2));
 9490   effect(KILL flags);
 9491 
 9492   ins_cost(INSN_COST * 3);
 9493   format %{
 9494       "subs zr, $src1, $src2\n\t"
 9495       "csetw $dst, ne\n\t"
 9496       "cnegw $dst, lo\t# CmpUL3(imm)"
 9497   %}
 9498   ins_encode %{
 9499     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9500     __ csetw($dst$$Register, Assembler::NE);
 9501     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9502   %}
 9503 
 9504   ins_pipe(pipe_class_default);
 9505 %}
 9506 
 9507 // Manifest a CmpL result in an integer register.
 9508 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9509 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9510 %{
 9511   match(Set dst (CmpL3 src1 src2));
 9512   effect(KILL flags);
 9513 
 9514   ins_cost(INSN_COST * 3);
 9515   format %{
 9516       "cmp $src1, $src2\n\t"
 9517       "csetw $dst, ne\n\t"
 9518       "cnegw $dst, lt\t# CmpL3(reg)"
 9519   %}
 9520   ins_encode %{
 9521     __ cmp($src1$$Register, $src2$$Register);
 9522     __ csetw($dst$$Register, Assembler::NE);
 9523     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9524   %}
 9525 
 9526   ins_pipe(pipe_class_default);
 9527 %}
 9528 
 9529 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9530 %{
 9531   match(Set dst (CmpL3 src1 src2));
 9532   effect(KILL flags);
 9533 
 9534   ins_cost(INSN_COST * 3);
 9535   format %{
 9536       "subs zr, $src1, $src2\n\t"
 9537       "csetw $dst, ne\n\t"
 9538       "cnegw $dst, lt\t# CmpL3(imm)"
 9539   %}
 9540   ins_encode %{
 9541     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9542     __ csetw($dst$$Register, Assembler::NE);
 9543     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9544   %}
 9545 
 9546   ins_pipe(pipe_class_default);
 9547 %}
 9548 
 9549 // ============================================================================
 9550 // Conditional Move Instructions
 9551 
 9552 // n.b. we have identical rules for both a signed compare op (cmpOp)
 9553 // and an unsigned compare op (cmpOpU). it would be nice if we could
 9554 // define an op class which merged both inputs and use it to type the
 9555 // argument to a single rule. unfortunatelyt his fails because the
 9556 // opclass does not live up to the COND_INTER interface of its
 9557 // component operands. When the generic code tries to negate the
 9558 // operand it ends up running the generci Machoper::negate method
 9559 // which throws a ShouldNotHappen. So, we have to provide two flavours
 9560 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
 9561 
 9562 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9563   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9564 
 9565   ins_cost(INSN_COST * 2);
 9566   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
 9567 
 9568   ins_encode %{
 9569     __ cselw(as_Register($dst$$reg),
 9570              as_Register($src2$$reg),
 9571              as_Register($src1$$reg),
 9572              (Assembler::Condition)$cmp$$cmpcode);
 9573   %}
 9574 
 9575   ins_pipe(icond_reg_reg);
 9576 %}
 9577 
 9578 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9579   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9580 
 9581   ins_cost(INSN_COST * 2);
 9582   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
 9583 
 9584   ins_encode %{
 9585     __ cselw(as_Register($dst$$reg),
 9586              as_Register($src2$$reg),
 9587              as_Register($src1$$reg),
 9588              (Assembler::Condition)$cmp$$cmpcode);
 9589   %}
 9590 
 9591   ins_pipe(icond_reg_reg);
 9592 %}
 9593 
 9594 // special cases where one arg is zero
 9595 
 9596 // n.b. this is selected in preference to the rule above because it
 9597 // avoids loading constant 0 into a source register
 9598 
 9599 // TODO
 9600 // we ought only to be able to cull one of these variants as the ideal
 9601 // transforms ought always to order the zero consistently (to left/right?)
 9602 
 9603 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9604   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9605 
 9606   ins_cost(INSN_COST * 2);
 9607   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
 9608 
 9609   ins_encode %{
 9610     __ cselw(as_Register($dst$$reg),
 9611              as_Register($src$$reg),
 9612              zr,
 9613              (Assembler::Condition)$cmp$$cmpcode);
 9614   %}
 9615 
 9616   ins_pipe(icond_reg);
 9617 %}
 9618 
 9619 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9620   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9621 
 9622   ins_cost(INSN_COST * 2);
 9623   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
 9624 
 9625   ins_encode %{
 9626     __ cselw(as_Register($dst$$reg),
 9627              as_Register($src$$reg),
 9628              zr,
 9629              (Assembler::Condition)$cmp$$cmpcode);
 9630   %}
 9631 
 9632   ins_pipe(icond_reg);
 9633 %}
 9634 
 9635 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9636   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9637 
 9638   ins_cost(INSN_COST * 2);
 9639   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
 9640 
 9641   ins_encode %{
 9642     __ cselw(as_Register($dst$$reg),
 9643              zr,
 9644              as_Register($src$$reg),
 9645              (Assembler::Condition)$cmp$$cmpcode);
 9646   %}
 9647 
 9648   ins_pipe(icond_reg);
 9649 %}
 9650 
 9651 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9652   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9653 
 9654   ins_cost(INSN_COST * 2);
 9655   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
 9656 
 9657   ins_encode %{
 9658     __ cselw(as_Register($dst$$reg),
 9659              zr,
 9660              as_Register($src$$reg),
 9661              (Assembler::Condition)$cmp$$cmpcode);
 9662   %}
 9663 
 9664   ins_pipe(icond_reg);
 9665 %}
 9666 
 9667 // special case for creating a boolean 0 or 1
 9668 
 9669 // n.b. this is selected in preference to the rule above because it
 9670 // avoids loading constants 0 and 1 into a source register
 9671 
 9672 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9673   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9674 
 9675   ins_cost(INSN_COST * 2);
 9676   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
 9677 
 9678   ins_encode %{
 9679     // equivalently
 9680     // cset(as_Register($dst$$reg),
 9681     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9682     __ csincw(as_Register($dst$$reg),
 9683              zr,
 9684              zr,
 9685              (Assembler::Condition)$cmp$$cmpcode);
 9686   %}
 9687 
 9688   ins_pipe(icond_none);
 9689 %}
 9690 
 9691 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9692   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9693 
 9694   ins_cost(INSN_COST * 2);
 9695   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
 9696 
 9697   ins_encode %{
 9698     // equivalently
 9699     // cset(as_Register($dst$$reg),
 9700     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9701     __ csincw(as_Register($dst$$reg),
 9702              zr,
 9703              zr,
 9704              (Assembler::Condition)$cmp$$cmpcode);
 9705   %}
 9706 
 9707   ins_pipe(icond_none);
 9708 %}
 9709 
 9710 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9711   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9712 
 9713   ins_cost(INSN_COST * 2);
 9714   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
 9715 
 9716   ins_encode %{
 9717     __ csel(as_Register($dst$$reg),
 9718             as_Register($src2$$reg),
 9719             as_Register($src1$$reg),
 9720             (Assembler::Condition)$cmp$$cmpcode);
 9721   %}
 9722 
 9723   ins_pipe(icond_reg_reg);
 9724 %}
 9725 
 9726 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9727   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9728 
 9729   ins_cost(INSN_COST * 2);
 9730   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
 9731 
 9732   ins_encode %{
 9733     __ csel(as_Register($dst$$reg),
 9734             as_Register($src2$$reg),
 9735             as_Register($src1$$reg),
 9736             (Assembler::Condition)$cmp$$cmpcode);
 9737   %}
 9738 
 9739   ins_pipe(icond_reg_reg);
 9740 %}
 9741 
 9742 // special cases where one arg is zero
 9743 
 9744 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9745   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9746 
 9747   ins_cost(INSN_COST * 2);
 9748   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
 9749 
 9750   ins_encode %{
 9751     __ csel(as_Register($dst$$reg),
 9752             zr,
 9753             as_Register($src$$reg),
 9754             (Assembler::Condition)$cmp$$cmpcode);
 9755   %}
 9756 
 9757   ins_pipe(icond_reg);
 9758 %}
 9759 
 9760 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9761   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9762 
 9763   ins_cost(INSN_COST * 2);
 9764   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
 9765 
 9766   ins_encode %{
 9767     __ csel(as_Register($dst$$reg),
 9768             zr,
 9769             as_Register($src$$reg),
 9770             (Assembler::Condition)$cmp$$cmpcode);
 9771   %}
 9772 
 9773   ins_pipe(icond_reg);
 9774 %}
 9775 
 9776 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9777   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9778 
 9779   ins_cost(INSN_COST * 2);
 9780   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
 9781 
 9782   ins_encode %{
 9783     __ csel(as_Register($dst$$reg),
 9784             as_Register($src$$reg),
 9785             zr,
 9786             (Assembler::Condition)$cmp$$cmpcode);
 9787   %}
 9788 
 9789   ins_pipe(icond_reg);
 9790 %}
 9791 
 9792 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9793   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9794 
 9795   ins_cost(INSN_COST * 2);
 9796   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
 9797 
 9798   ins_encode %{
 9799     __ csel(as_Register($dst$$reg),
 9800             as_Register($src$$reg),
 9801             zr,
 9802             (Assembler::Condition)$cmp$$cmpcode);
 9803   %}
 9804 
 9805   ins_pipe(icond_reg);
 9806 %}
 9807 
 9808 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
 9809   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
 9810 
 9811   ins_cost(INSN_COST * 2);
 9812   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
 9813 
 9814   ins_encode %{
 9815     __ csel(as_Register($dst$$reg),
 9816             as_Register($src2$$reg),
 9817             as_Register($src1$$reg),
 9818             (Assembler::Condition)$cmp$$cmpcode);
 9819   %}
 9820 
 9821   ins_pipe(icond_reg_reg);
 9822 %}
 9823 
 9824 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
 9825   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
 9826 
 9827   ins_cost(INSN_COST * 2);
 9828   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
 9829 
 9830   ins_encode %{
 9831     __ csel(as_Register($dst$$reg),
 9832             as_Register($src2$$reg),
 9833             as_Register($src1$$reg),
 9834             (Assembler::Condition)$cmp$$cmpcode);
 9835   %}
 9836 
 9837   ins_pipe(icond_reg_reg);
 9838 %}
 9839 
 9840 // special cases where one arg is zero
 9841 
 9842 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
 9843   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
 9844 
 9845   ins_cost(INSN_COST * 2);
 9846   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
 9847 
 9848   ins_encode %{
 9849     __ csel(as_Register($dst$$reg),
 9850             zr,
 9851             as_Register($src$$reg),
 9852             (Assembler::Condition)$cmp$$cmpcode);
 9853   %}
 9854 
 9855   ins_pipe(icond_reg);
 9856 %}
 9857 
 9858 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
 9859   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
 9860 
 9861   ins_cost(INSN_COST * 2);
 9862   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
 9863 
 9864   ins_encode %{
 9865     __ csel(as_Register($dst$$reg),
 9866             zr,
 9867             as_Register($src$$reg),
 9868             (Assembler::Condition)$cmp$$cmpcode);
 9869   %}
 9870 
 9871   ins_pipe(icond_reg);
 9872 %}
 9873 
 9874 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
 9875   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
 9876 
 9877   ins_cost(INSN_COST * 2);
 9878   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
 9879 
 9880   ins_encode %{
 9881     __ csel(as_Register($dst$$reg),
 9882             as_Register($src$$reg),
 9883             zr,
 9884             (Assembler::Condition)$cmp$$cmpcode);
 9885   %}
 9886 
 9887   ins_pipe(icond_reg);
 9888 %}
 9889 
 9890 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
 9891   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
 9892 
 9893   ins_cost(INSN_COST * 2);
 9894   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
 9895 
 9896   ins_encode %{
 9897     __ csel(as_Register($dst$$reg),
 9898             as_Register($src$$reg),
 9899             zr,
 9900             (Assembler::Condition)$cmp$$cmpcode);
 9901   %}
 9902 
 9903   ins_pipe(icond_reg);
 9904 %}
 9905 
 9906 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
 9907   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
 9908 
 9909   ins_cost(INSN_COST * 2);
 9910   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
 9911 
 9912   ins_encode %{
 9913     __ cselw(as_Register($dst$$reg),
 9914              as_Register($src2$$reg),
 9915              as_Register($src1$$reg),
 9916              (Assembler::Condition)$cmp$$cmpcode);
 9917   %}
 9918 
 9919   ins_pipe(icond_reg_reg);
 9920 %}
 9921 
 9922 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
 9923   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
 9924 
 9925   ins_cost(INSN_COST * 2);
 9926   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
 9927 
 9928   ins_encode %{
 9929     __ cselw(as_Register($dst$$reg),
 9930              as_Register($src2$$reg),
 9931              as_Register($src1$$reg),
 9932              (Assembler::Condition)$cmp$$cmpcode);
 9933   %}
 9934 
 9935   ins_pipe(icond_reg_reg);
 9936 %}
 9937 
 9938 // special cases where one arg is zero
 9939 
 9940 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
 9941   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
 9942 
 9943   ins_cost(INSN_COST * 2);
 9944   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
 9945 
 9946   ins_encode %{
 9947     __ cselw(as_Register($dst$$reg),
 9948              zr,
 9949              as_Register($src$$reg),
 9950              (Assembler::Condition)$cmp$$cmpcode);
 9951   %}
 9952 
 9953   ins_pipe(icond_reg);
 9954 %}
 9955 
 9956 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
 9957   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
 9958 
 9959   ins_cost(INSN_COST * 2);
 9960   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
 9961 
 9962   ins_encode %{
 9963     __ cselw(as_Register($dst$$reg),
 9964              zr,
 9965              as_Register($src$$reg),
 9966              (Assembler::Condition)$cmp$$cmpcode);
 9967   %}
 9968 
 9969   ins_pipe(icond_reg);
 9970 %}
 9971 
 9972 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
 9973   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
 9974 
 9975   ins_cost(INSN_COST * 2);
 9976   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
 9977 
 9978   ins_encode %{
 9979     __ cselw(as_Register($dst$$reg),
 9980              as_Register($src$$reg),
 9981              zr,
 9982              (Assembler::Condition)$cmp$$cmpcode);
 9983   %}
 9984 
 9985   ins_pipe(icond_reg);
 9986 %}
 9987 
 9988 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
 9989   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
 9990 
 9991   ins_cost(INSN_COST * 2);
 9992   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
 9993 
 9994   ins_encode %{
 9995     __ cselw(as_Register($dst$$reg),
 9996              as_Register($src$$reg),
 9997              zr,
 9998              (Assembler::Condition)$cmp$$cmpcode);
 9999   %}
10000 
10001   ins_pipe(icond_reg);
10002 %}
10003 
10004 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10005 %{
10006   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10007 
10008   ins_cost(INSN_COST * 3);
10009 
10010   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10011   ins_encode %{
10012     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10013     __ fcsels(as_FloatRegister($dst$$reg),
10014               as_FloatRegister($src2$$reg),
10015               as_FloatRegister($src1$$reg),
10016               cond);
10017   %}
10018 
10019   ins_pipe(fp_cond_reg_reg_s);
10020 %}
10021 
10022 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10023 %{
10024   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10025 
10026   ins_cost(INSN_COST * 3);
10027 
10028   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10029   ins_encode %{
10030     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10031     __ fcsels(as_FloatRegister($dst$$reg),
10032               as_FloatRegister($src2$$reg),
10033               as_FloatRegister($src1$$reg),
10034               cond);
10035   %}
10036 
10037   ins_pipe(fp_cond_reg_reg_s);
10038 %}
10039 
10040 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10041 %{
10042   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10043 
10044   ins_cost(INSN_COST * 3);
10045 
10046   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10047   ins_encode %{
10048     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10049     __ fcseld(as_FloatRegister($dst$$reg),
10050               as_FloatRegister($src2$$reg),
10051               as_FloatRegister($src1$$reg),
10052               cond);
10053   %}
10054 
10055   ins_pipe(fp_cond_reg_reg_d);
10056 %}
10057 
10058 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10059 %{
10060   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10061 
10062   ins_cost(INSN_COST * 3);
10063 
10064   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10065   ins_encode %{
10066     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10067     __ fcseld(as_FloatRegister($dst$$reg),
10068               as_FloatRegister($src2$$reg),
10069               as_FloatRegister($src1$$reg),
10070               cond);
10071   %}
10072 
10073   ins_pipe(fp_cond_reg_reg_d);
10074 %}
10075 
10076 // ============================================================================
10077 // Arithmetic Instructions
10078 //
10079 
10080 // Integer Addition
10081 
10082 // TODO
10083 // these currently employ operations which do not set CR and hence are
10084 // not flagged as killing CR but we would like to isolate the cases
10085 // where we want to set flags from those where we don't. need to work
10086 // out how to do that.
10087 
10088 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10089   match(Set dst (AddI src1 src2));
10090 
10091   ins_cost(INSN_COST);
10092   format %{ "addw  $dst, $src1, $src2" %}
10093 
10094   ins_encode %{
10095     __ addw(as_Register($dst$$reg),
10096             as_Register($src1$$reg),
10097             as_Register($src2$$reg));
10098   %}
10099 
10100   ins_pipe(ialu_reg_reg);
10101 %}
10102 
10103 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10104   match(Set dst (AddI src1 src2));
10105 
10106   ins_cost(INSN_COST);
10107   format %{ "addw $dst, $src1, $src2" %}
10108 
10109   // use opcode to indicate that this is an add not a sub
10110   opcode(0x0);
10111 
10112   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10113 
10114   ins_pipe(ialu_reg_imm);
10115 %}
10116 
10117 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10118   match(Set dst (AddI (ConvL2I src1) src2));
10119 
10120   ins_cost(INSN_COST);
10121   format %{ "addw $dst, $src1, $src2" %}
10122 
10123   // use opcode to indicate that this is an add not a sub
10124   opcode(0x0);
10125 
10126   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10127 
10128   ins_pipe(ialu_reg_imm);
10129 %}
10130 
10131 // Pointer Addition
10132 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
10133   match(Set dst (AddP src1 src2));
10134 
10135   ins_cost(INSN_COST);
10136   format %{ "add $dst, $src1, $src2\t# ptr" %}
10137 
10138   ins_encode %{
10139     __ add(as_Register($dst$$reg),
10140            as_Register($src1$$reg),
10141            as_Register($src2$$reg));
10142   %}
10143 
10144   ins_pipe(ialu_reg_reg);
10145 %}
10146 
10147 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
10148   match(Set dst (AddP src1 (ConvI2L src2)));
10149 
10150   ins_cost(1.9 * INSN_COST);
10151   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10152 
10153   ins_encode %{
10154     __ add(as_Register($dst$$reg),
10155            as_Register($src1$$reg),
10156            as_Register($src2$$reg), ext::sxtw);
10157   %}
10158 
10159   ins_pipe(ialu_reg_reg);
10160 %}
10161 
10162 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
10163   match(Set dst (AddP src1 (LShiftL src2 scale)));
10164 
10165   ins_cost(1.9 * INSN_COST);
10166   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10167 
10168   ins_encode %{
10169     __ lea(as_Register($dst$$reg),
10170            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10171                    Address::lsl($scale$$constant)));
10172   %}
10173 
10174   ins_pipe(ialu_reg_reg_shift);
10175 %}
10176 
10177 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
10178   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10179 
10180   ins_cost(1.9 * INSN_COST);
10181   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10182 
10183   ins_encode %{
10184     __ lea(as_Register($dst$$reg),
10185            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10186                    Address::sxtw($scale$$constant)));
10187   %}
10188 
10189   ins_pipe(ialu_reg_reg_shift);
10190 %}
10191 
10192 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10193   match(Set dst (LShiftL (ConvI2L src) scale));
10194 
10195   ins_cost(INSN_COST);
10196   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10197 
10198   ins_encode %{
10199     __ sbfiz(as_Register($dst$$reg),
10200           as_Register($src$$reg),
10201           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10202   %}
10203 
10204   ins_pipe(ialu_reg_shift);
10205 %}
10206 
10207 // Pointer Immediate Addition
10208 // n.b. this needs to be more expensive than using an indirect memory
10209 // operand
10210 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10211   match(Set dst (AddP src1 src2));
10212 
10213   ins_cost(INSN_COST);
10214   format %{ "add $dst, $src1, $src2\t# ptr" %}
10215 
10216   // use opcode to indicate that this is an add not a sub
10217   opcode(0x0);
10218 
10219   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10220 
10221   ins_pipe(ialu_reg_imm);
10222 %}
10223 
10224 // Long Addition
10225 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10226 
10227   match(Set dst (AddL src1 src2));
10228 
10229   ins_cost(INSN_COST);
10230   format %{ "add  $dst, $src1, $src2" %}
10231 
10232   ins_encode %{
10233     __ add(as_Register($dst$$reg),
10234            as_Register($src1$$reg),
10235            as_Register($src2$$reg));
10236   %}
10237 
10238   ins_pipe(ialu_reg_reg);
10239 %}
10240 
10241 // No constant pool entries requiredLong Immediate Addition.
10242 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10243   match(Set dst (AddL src1 src2));
10244 
10245   ins_cost(INSN_COST);
10246   format %{ "add $dst, $src1, $src2" %}
10247 
10248   // use opcode to indicate that this is an add not a sub
10249   opcode(0x0);
10250 
10251   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10252 
10253   ins_pipe(ialu_reg_imm);
10254 %}
10255 
10256 // Integer Subtraction
10257 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10258   match(Set dst (SubI src1 src2));
10259 
10260   ins_cost(INSN_COST);
10261   format %{ "subw  $dst, $src1, $src2" %}
10262 
10263   ins_encode %{
10264     __ subw(as_Register($dst$$reg),
10265             as_Register($src1$$reg),
10266             as_Register($src2$$reg));
10267   %}
10268 
10269   ins_pipe(ialu_reg_reg);
10270 %}
10271 
10272 // Immediate Subtraction
10273 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10274   match(Set dst (SubI src1 src2));
10275 
10276   ins_cost(INSN_COST);
10277   format %{ "subw $dst, $src1, $src2" %}
10278 
10279   // use opcode to indicate that this is a sub not an add
10280   opcode(0x1);
10281 
10282   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10283 
10284   ins_pipe(ialu_reg_imm);
10285 %}
10286 
10287 // Long Subtraction
10288 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10289 
10290   match(Set dst (SubL src1 src2));
10291 
10292   ins_cost(INSN_COST);
10293   format %{ "sub  $dst, $src1, $src2" %}
10294 
10295   ins_encode %{
10296     __ sub(as_Register($dst$$reg),
10297            as_Register($src1$$reg),
10298            as_Register($src2$$reg));
10299   %}
10300 
10301   ins_pipe(ialu_reg_reg);
10302 %}
10303 
10304 // No constant pool entries requiredLong Immediate Subtraction.
10305 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10306   match(Set dst (SubL src1 src2));
10307 
10308   ins_cost(INSN_COST);
10309   format %{ "sub$dst, $src1, $src2" %}
10310 
10311   // use opcode to indicate that this is a sub not an add
10312   opcode(0x1);
10313 
10314   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10315 
10316   ins_pipe(ialu_reg_imm);
10317 %}
10318 
10319 // Integer Negation (special case for sub)
10320 
10321 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10322   match(Set dst (SubI zero src));
10323 
10324   ins_cost(INSN_COST);
10325   format %{ "negw $dst, $src\t# int" %}
10326 
10327   ins_encode %{
10328     __ negw(as_Register($dst$$reg),
10329             as_Register($src$$reg));
10330   %}
10331 
10332   ins_pipe(ialu_reg);
10333 %}
10334 
10335 // Long Negation
10336 
10337 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10338   match(Set dst (SubL zero src));
10339 
10340   ins_cost(INSN_COST);
10341   format %{ "neg $dst, $src\t# long" %}
10342 
10343   ins_encode %{
10344     __ neg(as_Register($dst$$reg),
10345            as_Register($src$$reg));
10346   %}
10347 
10348   ins_pipe(ialu_reg);
10349 %}
10350 
10351 // Integer Multiply
10352 
10353 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10354   match(Set dst (MulI src1 src2));
10355 
10356   ins_cost(INSN_COST * 3);
10357   format %{ "mulw  $dst, $src1, $src2" %}
10358 
10359   ins_encode %{
10360     __ mulw(as_Register($dst$$reg),
10361             as_Register($src1$$reg),
10362             as_Register($src2$$reg));
10363   %}
10364 
10365   ins_pipe(imul_reg_reg);
10366 %}
10367 
10368 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10369   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10370 
10371   ins_cost(INSN_COST * 3);
10372   format %{ "smull  $dst, $src1, $src2" %}
10373 
10374   ins_encode %{
10375     __ smull(as_Register($dst$$reg),
10376              as_Register($src1$$reg),
10377              as_Register($src2$$reg));
10378   %}
10379 
10380   ins_pipe(imul_reg_reg);
10381 %}
10382 
10383 // Long Multiply
10384 
10385 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10386   match(Set dst (MulL src1 src2));
10387 
10388   ins_cost(INSN_COST * 5);
10389   format %{ "mul  $dst, $src1, $src2" %}
10390 
10391   ins_encode %{
10392     __ mul(as_Register($dst$$reg),
10393            as_Register($src1$$reg),
10394            as_Register($src2$$reg));
10395   %}
10396 
10397   ins_pipe(lmul_reg_reg);
10398 %}
10399 
10400 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10401 %{
10402   match(Set dst (MulHiL src1 src2));
10403 
10404   ins_cost(INSN_COST * 7);
10405   format %{ "smulh   $dst, $src1, $src2\t# mulhi" %}
10406 
10407   ins_encode %{
10408     __ smulh(as_Register($dst$$reg),
10409              as_Register($src1$$reg),
10410              as_Register($src2$$reg));
10411   %}
10412 
10413   ins_pipe(lmul_reg_reg);
10414 %}
10415 
10416 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10417 %{
10418   match(Set dst (UMulHiL src1 src2));
10419 
10420   ins_cost(INSN_COST * 7);
10421   format %{ "umulh   $dst, $src1, $src2\t# umulhi" %}
10422 
10423   ins_encode %{
10424     __ umulh(as_Register($dst$$reg),
10425              as_Register($src1$$reg),
10426              as_Register($src2$$reg));
10427   %}
10428 
10429   ins_pipe(lmul_reg_reg);
10430 %}
10431 
10432 // Combined Integer Multiply & Add/Sub
10433 
10434 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10435   match(Set dst (AddI src3 (MulI src1 src2)));
10436 
10437   ins_cost(INSN_COST * 3);
10438   format %{ "madd  $dst, $src1, $src2, $src3" %}
10439 
10440   ins_encode %{
10441     __ maddw(as_Register($dst$$reg),
10442              as_Register($src1$$reg),
10443              as_Register($src2$$reg),
10444              as_Register($src3$$reg));
10445   %}
10446 
10447   ins_pipe(imac_reg_reg);
10448 %}
10449 
10450 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10451   match(Set dst (SubI src3 (MulI src1 src2)));
10452 
10453   ins_cost(INSN_COST * 3);
10454   format %{ "msub  $dst, $src1, $src2, $src3" %}
10455 
10456   ins_encode %{
10457     __ msubw(as_Register($dst$$reg),
10458              as_Register($src1$$reg),
10459              as_Register($src2$$reg),
10460              as_Register($src3$$reg));
10461   %}
10462 
10463   ins_pipe(imac_reg_reg);
10464 %}
10465 
10466 // Combined Integer Multiply & Neg
10467 
10468 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10469   match(Set dst (MulI (SubI zero src1) src2));
10470 
10471   ins_cost(INSN_COST * 3);
10472   format %{ "mneg  $dst, $src1, $src2" %}
10473 
10474   ins_encode %{
10475     __ mnegw(as_Register($dst$$reg),
10476              as_Register($src1$$reg),
10477              as_Register($src2$$reg));
10478   %}
10479 
10480   ins_pipe(imac_reg_reg);
10481 %}
10482 
10483 // Combined Long Multiply & Add/Sub
10484 
10485 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10486   match(Set dst (AddL src3 (MulL src1 src2)));
10487 
10488   ins_cost(INSN_COST * 5);
10489   format %{ "madd  $dst, $src1, $src2, $src3" %}
10490 
10491   ins_encode %{
10492     __ madd(as_Register($dst$$reg),
10493             as_Register($src1$$reg),
10494             as_Register($src2$$reg),
10495             as_Register($src3$$reg));
10496   %}
10497 
10498   ins_pipe(lmac_reg_reg);
10499 %}
10500 
10501 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10502   match(Set dst (SubL src3 (MulL src1 src2)));
10503 
10504   ins_cost(INSN_COST * 5);
10505   format %{ "msub  $dst, $src1, $src2, $src3" %}
10506 
10507   ins_encode %{
10508     __ msub(as_Register($dst$$reg),
10509             as_Register($src1$$reg),
10510             as_Register($src2$$reg),
10511             as_Register($src3$$reg));
10512   %}
10513 
10514   ins_pipe(lmac_reg_reg);
10515 %}
10516 
10517 // Combined Long Multiply & Neg
10518 
10519 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10520   match(Set dst (MulL (SubL zero src1) src2));
10521 
10522   ins_cost(INSN_COST * 5);
10523   format %{ "mneg  $dst, $src1, $src2" %}
10524 
10525   ins_encode %{
10526     __ mneg(as_Register($dst$$reg),
10527             as_Register($src1$$reg),
10528             as_Register($src2$$reg));
10529   %}
10530 
10531   ins_pipe(lmac_reg_reg);
10532 %}
10533 
10534 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10535 
10536 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10537   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10538 
10539   ins_cost(INSN_COST * 3);
10540   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10541 
10542   ins_encode %{
10543     __ smaddl(as_Register($dst$$reg),
10544               as_Register($src1$$reg),
10545               as_Register($src2$$reg),
10546               as_Register($src3$$reg));
10547   %}
10548 
10549   ins_pipe(imac_reg_reg);
10550 %}
10551 
10552 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10553   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10554 
10555   ins_cost(INSN_COST * 3);
10556   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10557 
10558   ins_encode %{
10559     __ smsubl(as_Register($dst$$reg),
10560               as_Register($src1$$reg),
10561               as_Register($src2$$reg),
10562               as_Register($src3$$reg));
10563   %}
10564 
10565   ins_pipe(imac_reg_reg);
10566 %}
10567 
10568 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10569   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10570 
10571   ins_cost(INSN_COST * 3);
10572   format %{ "smnegl  $dst, $src1, $src2" %}
10573 
10574   ins_encode %{
10575     __ smnegl(as_Register($dst$$reg),
10576               as_Register($src1$$reg),
10577               as_Register($src2$$reg));
10578   %}
10579 
10580   ins_pipe(imac_reg_reg);
10581 %}
10582 
10583 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10584 
10585 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10586   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10587 
10588   ins_cost(INSN_COST * 5);
10589   format %{ "mulw  rscratch1, $src1, $src2\n\t"
10590             "maddw $dst, $src3, $src4, rscratch1" %}
10591 
10592   ins_encode %{
10593     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10594     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10595 
10596   ins_pipe(imac_reg_reg);
10597 %}
10598 
10599 // Integer Divide
10600 
10601 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10602   match(Set dst (DivI src1 src2));
10603 
10604   ins_cost(INSN_COST * 19);
10605   format %{ "sdivw  $dst, $src1, $src2" %}
10606 
10607   ins_encode(aarch64_enc_divw(dst, src1, src2));
10608   ins_pipe(idiv_reg_reg);
10609 %}
10610 
10611 // Long Divide
10612 
10613 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10614   match(Set dst (DivL src1 src2));
10615 
10616   ins_cost(INSN_COST * 35);
10617   format %{ "sdiv   $dst, $src1, $src2" %}
10618 
10619   ins_encode(aarch64_enc_div(dst, src1, src2));
10620   ins_pipe(ldiv_reg_reg);
10621 %}
10622 
10623 // Integer Remainder
10624 
10625 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10626   match(Set dst (ModI src1 src2));
10627 
10628   ins_cost(INSN_COST * 22);
10629   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
10630             "msubw  $dst, rscratch1, $src2, $src1" %}
10631 
10632   ins_encode(aarch64_enc_modw(dst, src1, src2));
10633   ins_pipe(idiv_reg_reg);
10634 %}
10635 
10636 // Long Remainder
10637 
10638 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10639   match(Set dst (ModL src1 src2));
10640 
10641   ins_cost(INSN_COST * 38);
10642   format %{ "sdiv   rscratch1, $src1, $src2\n"
10643             "msub   $dst, rscratch1, $src2, $src1" %}
10644 
10645   ins_encode(aarch64_enc_mod(dst, src1, src2));
10646   ins_pipe(ldiv_reg_reg);
10647 %}
10648 
10649 // Unsigned Integer Divide
10650 
10651 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10652   match(Set dst (UDivI src1 src2));
10653 
10654   ins_cost(INSN_COST * 19);
10655   format %{ "udivw  $dst, $src1, $src2" %}
10656 
10657   ins_encode %{
10658     __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10659   %}
10660 
10661   ins_pipe(idiv_reg_reg);
10662 %}
10663 
10664 //  Unsigned Long Divide
10665 
10666 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10667   match(Set dst (UDivL src1 src2));
10668 
10669   ins_cost(INSN_COST * 35);
10670   format %{ "udiv   $dst, $src1, $src2" %}
10671 
10672   ins_encode %{
10673     __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10674   %}
10675 
10676   ins_pipe(ldiv_reg_reg);
10677 %}
10678 
10679 // Unsigned Integer Remainder
10680 
10681 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10682   match(Set dst (UModI src1 src2));
10683 
10684   ins_cost(INSN_COST * 22);
10685   format %{ "udivw  rscratch1, $src1, $src2\n\t"
10686             "msubw  $dst, rscratch1, $src2, $src1" %}
10687 
10688   ins_encode %{
10689     __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10690     __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10691   %}
10692 
10693   ins_pipe(idiv_reg_reg);
10694 %}
10695 
10696 // Unsigned Long Remainder
10697 
10698 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10699   match(Set dst (UModL src1 src2));
10700 
10701   ins_cost(INSN_COST * 38);
10702   format %{ "udiv   rscratch1, $src1, $src2\n"
10703             "msub   $dst, rscratch1, $src2, $src1" %}
10704 
10705   ins_encode %{
10706     __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10707     __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10708   %}
10709 
10710   ins_pipe(ldiv_reg_reg);
10711 %}
10712 
10713 // Integer Shifts
10714 
10715 // Shift Left Register
10716 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10717   match(Set dst (LShiftI src1 src2));
10718 
10719   ins_cost(INSN_COST * 2);
10720   format %{ "lslvw  $dst, $src1, $src2" %}
10721 
10722   ins_encode %{
10723     __ lslvw(as_Register($dst$$reg),
10724              as_Register($src1$$reg),
10725              as_Register($src2$$reg));
10726   %}
10727 
10728   ins_pipe(ialu_reg_reg_vshift);
10729 %}
10730 
10731 // Shift Left Immediate
10732 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10733   match(Set dst (LShiftI src1 src2));
10734 
10735   ins_cost(INSN_COST);
10736   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10737 
10738   ins_encode %{
10739     __ lslw(as_Register($dst$$reg),
10740             as_Register($src1$$reg),
10741             $src2$$constant & 0x1f);
10742   %}
10743 
10744   ins_pipe(ialu_reg_shift);
10745 %}
10746 
10747 // Shift Right Logical Register
10748 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10749   match(Set dst (URShiftI src1 src2));
10750 
10751   ins_cost(INSN_COST * 2);
10752   format %{ "lsrvw  $dst, $src1, $src2" %}
10753 
10754   ins_encode %{
10755     __ lsrvw(as_Register($dst$$reg),
10756              as_Register($src1$$reg),
10757              as_Register($src2$$reg));
10758   %}
10759 
10760   ins_pipe(ialu_reg_reg_vshift);
10761 %}
10762 
10763 // Shift Right Logical Immediate
10764 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10765   match(Set dst (URShiftI src1 src2));
10766 
10767   ins_cost(INSN_COST);
10768   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10769 
10770   ins_encode %{
10771     __ lsrw(as_Register($dst$$reg),
10772             as_Register($src1$$reg),
10773             $src2$$constant & 0x1f);
10774   %}
10775 
10776   ins_pipe(ialu_reg_shift);
10777 %}
10778 
10779 // Shift Right Arithmetic Register
10780 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10781   match(Set dst (RShiftI src1 src2));
10782 
10783   ins_cost(INSN_COST * 2);
10784   format %{ "asrvw  $dst, $src1, $src2" %}
10785 
10786   ins_encode %{
10787     __ asrvw(as_Register($dst$$reg),
10788              as_Register($src1$$reg),
10789              as_Register($src2$$reg));
10790   %}
10791 
10792   ins_pipe(ialu_reg_reg_vshift);
10793 %}
10794 
10795 // Shift Right Arithmetic Immediate
10796 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10797   match(Set dst (RShiftI src1 src2));
10798 
10799   ins_cost(INSN_COST);
10800   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10801 
10802   ins_encode %{
10803     __ asrw(as_Register($dst$$reg),
10804             as_Register($src1$$reg),
10805             $src2$$constant & 0x1f);
10806   %}
10807 
10808   ins_pipe(ialu_reg_shift);
10809 %}
10810 
10811 // Combined Int Mask and Right Shift (using UBFM)
10812 // TODO
10813 
10814 // Long Shifts
10815 
10816 // Shift Left Register
10817 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10818   match(Set dst (LShiftL src1 src2));
10819 
10820   ins_cost(INSN_COST * 2);
10821   format %{ "lslv  $dst, $src1, $src2" %}
10822 
10823   ins_encode %{
10824     __ lslv(as_Register($dst$$reg),
10825             as_Register($src1$$reg),
10826             as_Register($src2$$reg));
10827   %}
10828 
10829   ins_pipe(ialu_reg_reg_vshift);
10830 %}
10831 
10832 // Shift Left Immediate
10833 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10834   match(Set dst (LShiftL src1 src2));
10835 
10836   ins_cost(INSN_COST);
10837   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10838 
10839   ins_encode %{
10840     __ lsl(as_Register($dst$$reg),
10841             as_Register($src1$$reg),
10842             $src2$$constant & 0x3f);
10843   %}
10844 
10845   ins_pipe(ialu_reg_shift);
10846 %}
10847 
10848 // Shift Right Logical Register
10849 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10850   match(Set dst (URShiftL src1 src2));
10851 
10852   ins_cost(INSN_COST * 2);
10853   format %{ "lsrv  $dst, $src1, $src2" %}
10854 
10855   ins_encode %{
10856     __ lsrv(as_Register($dst$$reg),
10857             as_Register($src1$$reg),
10858             as_Register($src2$$reg));
10859   %}
10860 
10861   ins_pipe(ialu_reg_reg_vshift);
10862 %}
10863 
10864 // Shift Right Logical Immediate
10865 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10866   match(Set dst (URShiftL src1 src2));
10867 
10868   ins_cost(INSN_COST);
10869   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10870 
10871   ins_encode %{
10872     __ lsr(as_Register($dst$$reg),
10873            as_Register($src1$$reg),
10874            $src2$$constant & 0x3f);
10875   %}
10876 
10877   ins_pipe(ialu_reg_shift);
10878 %}
10879 
10880 // A special-case pattern for card table stores.
10881 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10882   match(Set dst (URShiftL (CastP2X src1) src2));
10883 
10884   ins_cost(INSN_COST);
10885   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10886 
10887   ins_encode %{
10888     __ lsr(as_Register($dst$$reg),
10889            as_Register($src1$$reg),
10890            $src2$$constant & 0x3f);
10891   %}
10892 
10893   ins_pipe(ialu_reg_shift);
10894 %}
10895 
10896 // Shift Right Arithmetic Register
10897 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10898   match(Set dst (RShiftL src1 src2));
10899 
10900   ins_cost(INSN_COST * 2);
10901   format %{ "asrv  $dst, $src1, $src2" %}
10902 
10903   ins_encode %{
10904     __ asrv(as_Register($dst$$reg),
10905             as_Register($src1$$reg),
10906             as_Register($src2$$reg));
10907   %}
10908 
10909   ins_pipe(ialu_reg_reg_vshift);
10910 %}
10911 
10912 // Shift Right Arithmetic Immediate
10913 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10914   match(Set dst (RShiftL src1 src2));
10915 
10916   ins_cost(INSN_COST);
10917   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10918 
10919   ins_encode %{
10920     __ asr(as_Register($dst$$reg),
10921            as_Register($src1$$reg),
10922            $src2$$constant & 0x3f);
10923   %}
10924 
10925   ins_pipe(ialu_reg_shift);
10926 %}
10927 
10928 // BEGIN This section of the file is automatically generated. Do not edit --------------
10929 // This section is generated from aarch64_ad.m4
10930 
10931 // This pattern is automatically generated from aarch64_ad.m4
10932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10933 instruct regL_not_reg(iRegLNoSp dst,
10934                          iRegL src1, immL_M1 m1,
10935                          rFlagsReg cr) %{
10936   match(Set dst (XorL src1 m1));
10937   ins_cost(INSN_COST);
10938   format %{ "eon  $dst, $src1, zr" %}
10939 
10940   ins_encode %{
10941     __ eon(as_Register($dst$$reg),
10942               as_Register($src1$$reg),
10943               zr,
10944               Assembler::LSL, 0);
10945   %}
10946 
10947   ins_pipe(ialu_reg);
10948 %}
10949 
10950 // This pattern is automatically generated from aarch64_ad.m4
10951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10952 instruct regI_not_reg(iRegINoSp dst,
10953                          iRegIorL2I src1, immI_M1 m1,
10954                          rFlagsReg cr) %{
10955   match(Set dst (XorI src1 m1));
10956   ins_cost(INSN_COST);
10957   format %{ "eonw  $dst, $src1, zr" %}
10958 
10959   ins_encode %{
10960     __ eonw(as_Register($dst$$reg),
10961               as_Register($src1$$reg),
10962               zr,
10963               Assembler::LSL, 0);
10964   %}
10965 
10966   ins_pipe(ialu_reg);
10967 %}
10968 
10969 // This pattern is automatically generated from aarch64_ad.m4
10970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10971 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10972                               immI0 zero, iRegIorL2I src1, immI src2) %{
10973   match(Set dst (SubI zero (URShiftI src1 src2)));
10974 
10975   ins_cost(1.9 * INSN_COST);
10976   format %{ "negw  $dst, $src1, LSR $src2" %}
10977 
10978   ins_encode %{
10979     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10980             Assembler::LSR, $src2$$constant & 0x1f);
10981   %}
10982 
10983   ins_pipe(ialu_reg_shift);
10984 %}
10985 
10986 // This pattern is automatically generated from aarch64_ad.m4
10987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10988 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10989                               immI0 zero, iRegIorL2I src1, immI src2) %{
10990   match(Set dst (SubI zero (RShiftI src1 src2)));
10991 
10992   ins_cost(1.9 * INSN_COST);
10993   format %{ "negw  $dst, $src1, ASR $src2" %}
10994 
10995   ins_encode %{
10996     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10997             Assembler::ASR, $src2$$constant & 0x1f);
10998   %}
10999 
11000   ins_pipe(ialu_reg_shift);
11001 %}
11002 
11003 // This pattern is automatically generated from aarch64_ad.m4
11004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11005 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11006                               immI0 zero, iRegIorL2I src1, immI src2) %{
11007   match(Set dst (SubI zero (LShiftI src1 src2)));
11008 
11009   ins_cost(1.9 * INSN_COST);
11010   format %{ "negw  $dst, $src1, LSL $src2" %}
11011 
11012   ins_encode %{
11013     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11014             Assembler::LSL, $src2$$constant & 0x1f);
11015   %}
11016 
11017   ins_pipe(ialu_reg_shift);
11018 %}
11019 
11020 // This pattern is automatically generated from aarch64_ad.m4
11021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11022 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11023                               immL0 zero, iRegL src1, immI src2) %{
11024   match(Set dst (SubL zero (URShiftL src1 src2)));
11025 
11026   ins_cost(1.9 * INSN_COST);
11027   format %{ "neg  $dst, $src1, LSR $src2" %}
11028 
11029   ins_encode %{
11030     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11031             Assembler::LSR, $src2$$constant & 0x3f);
11032   %}
11033 
11034   ins_pipe(ialu_reg_shift);
11035 %}
11036 
11037 // This pattern is automatically generated from aarch64_ad.m4
11038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11039 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11040                               immL0 zero, iRegL src1, immI src2) %{
11041   match(Set dst (SubL zero (RShiftL src1 src2)));
11042 
11043   ins_cost(1.9 * INSN_COST);
11044   format %{ "neg  $dst, $src1, ASR $src2" %}
11045 
11046   ins_encode %{
11047     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11048             Assembler::ASR, $src2$$constant & 0x3f);
11049   %}
11050 
11051   ins_pipe(ialu_reg_shift);
11052 %}
11053 
11054 // This pattern is automatically generated from aarch64_ad.m4
11055 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11056 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11057                               immL0 zero, iRegL src1, immI src2) %{
11058   match(Set dst (SubL zero (LShiftL src1 src2)));
11059 
11060   ins_cost(1.9 * INSN_COST);
11061   format %{ "neg  $dst, $src1, LSL $src2" %}
11062 
11063   ins_encode %{
11064     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11065             Assembler::LSL, $src2$$constant & 0x3f);
11066   %}
11067 
11068   ins_pipe(ialu_reg_shift);
11069 %}
11070 
11071 // This pattern is automatically generated from aarch64_ad.m4
11072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11073 instruct AndI_reg_not_reg(iRegINoSp dst,
11074                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11075   match(Set dst (AndI src1 (XorI src2 m1)));
11076   ins_cost(INSN_COST);
11077   format %{ "bicw  $dst, $src1, $src2" %}
11078 
11079   ins_encode %{
11080     __ bicw(as_Register($dst$$reg),
11081               as_Register($src1$$reg),
11082               as_Register($src2$$reg),
11083               Assembler::LSL, 0);
11084   %}
11085 
11086   ins_pipe(ialu_reg_reg);
11087 %}
11088 
11089 // This pattern is automatically generated from aarch64_ad.m4
11090 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11091 instruct AndL_reg_not_reg(iRegLNoSp dst,
11092                          iRegL src1, iRegL src2, immL_M1 m1) %{
11093   match(Set dst (AndL src1 (XorL src2 m1)));
11094   ins_cost(INSN_COST);
11095   format %{ "bic  $dst, $src1, $src2" %}
11096 
11097   ins_encode %{
11098     __ bic(as_Register($dst$$reg),
11099               as_Register($src1$$reg),
11100               as_Register($src2$$reg),
11101               Assembler::LSL, 0);
11102   %}
11103 
11104   ins_pipe(ialu_reg_reg);
11105 %}
11106 
11107 // This pattern is automatically generated from aarch64_ad.m4
11108 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11109 instruct OrI_reg_not_reg(iRegINoSp dst,
11110                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11111   match(Set dst (OrI src1 (XorI src2 m1)));
11112   ins_cost(INSN_COST);
11113   format %{ "ornw  $dst, $src1, $src2" %}
11114 
11115   ins_encode %{
11116     __ ornw(as_Register($dst$$reg),
11117               as_Register($src1$$reg),
11118               as_Register($src2$$reg),
11119               Assembler::LSL, 0);
11120   %}
11121 
11122   ins_pipe(ialu_reg_reg);
11123 %}
11124 
11125 // This pattern is automatically generated from aarch64_ad.m4
11126 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11127 instruct OrL_reg_not_reg(iRegLNoSp dst,
11128                          iRegL src1, iRegL src2, immL_M1 m1) %{
11129   match(Set dst (OrL src1 (XorL src2 m1)));
11130   ins_cost(INSN_COST);
11131   format %{ "orn  $dst, $src1, $src2" %}
11132 
11133   ins_encode %{
11134     __ orn(as_Register($dst$$reg),
11135               as_Register($src1$$reg),
11136               as_Register($src2$$reg),
11137               Assembler::LSL, 0);
11138   %}
11139 
11140   ins_pipe(ialu_reg_reg);
11141 %}
11142 
11143 // This pattern is automatically generated from aarch64_ad.m4
11144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11145 instruct XorI_reg_not_reg(iRegINoSp dst,
11146                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11147   match(Set dst (XorI m1 (XorI src2 src1)));
11148   ins_cost(INSN_COST);
11149   format %{ "eonw  $dst, $src1, $src2" %}
11150 
11151   ins_encode %{
11152     __ eonw(as_Register($dst$$reg),
11153               as_Register($src1$$reg),
11154               as_Register($src2$$reg),
11155               Assembler::LSL, 0);
11156   %}
11157 
11158   ins_pipe(ialu_reg_reg);
11159 %}
11160 
11161 // This pattern is automatically generated from aarch64_ad.m4
11162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11163 instruct XorL_reg_not_reg(iRegLNoSp dst,
11164                          iRegL src1, iRegL src2, immL_M1 m1) %{
11165   match(Set dst (XorL m1 (XorL src2 src1)));
11166   ins_cost(INSN_COST);
11167   format %{ "eon  $dst, $src1, $src2" %}
11168 
11169   ins_encode %{
11170     __ eon(as_Register($dst$$reg),
11171               as_Register($src1$$reg),
11172               as_Register($src2$$reg),
11173               Assembler::LSL, 0);
11174   %}
11175 
11176   ins_pipe(ialu_reg_reg);
11177 %}
11178 
11179 // This pattern is automatically generated from aarch64_ad.m4
11180 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11181 // val & (-1 ^ (val >>> shift)) ==> bicw
11182 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11183                          iRegIorL2I src1, iRegIorL2I src2,
11184                          immI src3, immI_M1 src4) %{
11185   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11186   ins_cost(1.9 * INSN_COST);
11187   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11188 
11189   ins_encode %{
11190     __ bicw(as_Register($dst$$reg),
11191               as_Register($src1$$reg),
11192               as_Register($src2$$reg),
11193               Assembler::LSR,
11194               $src3$$constant & 0x1f);
11195   %}
11196 
11197   ins_pipe(ialu_reg_reg_shift);
11198 %}
11199 
11200 // This pattern is automatically generated from aarch64_ad.m4
11201 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11202 // val & (-1 ^ (val >>> shift)) ==> bic
11203 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11204                          iRegL src1, iRegL src2,
11205                          immI src3, immL_M1 src4) %{
11206   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11207   ins_cost(1.9 * INSN_COST);
11208   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11209 
11210   ins_encode %{
11211     __ bic(as_Register($dst$$reg),
11212               as_Register($src1$$reg),
11213               as_Register($src2$$reg),
11214               Assembler::LSR,
11215               $src3$$constant & 0x3f);
11216   %}
11217 
11218   ins_pipe(ialu_reg_reg_shift);
11219 %}
11220 
11221 // This pattern is automatically generated from aarch64_ad.m4
11222 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11223 // val & (-1 ^ (val >> shift)) ==> bicw
11224 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11225                          iRegIorL2I src1, iRegIorL2I src2,
11226                          immI src3, immI_M1 src4) %{
11227   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11228   ins_cost(1.9 * INSN_COST);
11229   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11230 
11231   ins_encode %{
11232     __ bicw(as_Register($dst$$reg),
11233               as_Register($src1$$reg),
11234               as_Register($src2$$reg),
11235               Assembler::ASR,
11236               $src3$$constant & 0x1f);
11237   %}
11238 
11239   ins_pipe(ialu_reg_reg_shift);
11240 %}
11241 
11242 // This pattern is automatically generated from aarch64_ad.m4
11243 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11244 // val & (-1 ^ (val >> shift)) ==> bic
11245 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11246                          iRegL src1, iRegL src2,
11247                          immI src3, immL_M1 src4) %{
11248   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11249   ins_cost(1.9 * INSN_COST);
11250   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11251 
11252   ins_encode %{
11253     __ bic(as_Register($dst$$reg),
11254               as_Register($src1$$reg),
11255               as_Register($src2$$reg),
11256               Assembler::ASR,
11257               $src3$$constant & 0x3f);
11258   %}
11259 
11260   ins_pipe(ialu_reg_reg_shift);
11261 %}
11262 
11263 // This pattern is automatically generated from aarch64_ad.m4
11264 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11265 // val & (-1 ^ (val ror shift)) ==> bicw
11266 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11267                          iRegIorL2I src1, iRegIorL2I src2,
11268                          immI src3, immI_M1 src4) %{
11269   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11270   ins_cost(1.9 * INSN_COST);
11271   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11272 
11273   ins_encode %{
11274     __ bicw(as_Register($dst$$reg),
11275               as_Register($src1$$reg),
11276               as_Register($src2$$reg),
11277               Assembler::ROR,
11278               $src3$$constant & 0x1f);
11279   %}
11280 
11281   ins_pipe(ialu_reg_reg_shift);
11282 %}
11283 
11284 // This pattern is automatically generated from aarch64_ad.m4
11285 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11286 // val & (-1 ^ (val ror shift)) ==> bic
11287 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11288                          iRegL src1, iRegL src2,
11289                          immI src3, immL_M1 src4) %{
11290   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11291   ins_cost(1.9 * INSN_COST);
11292   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11293 
11294   ins_encode %{
11295     __ bic(as_Register($dst$$reg),
11296               as_Register($src1$$reg),
11297               as_Register($src2$$reg),
11298               Assembler::ROR,
11299               $src3$$constant & 0x3f);
11300   %}
11301 
11302   ins_pipe(ialu_reg_reg_shift);
11303 %}
11304 
11305 // This pattern is automatically generated from aarch64_ad.m4
11306 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11307 // val & (-1 ^ (val << shift)) ==> bicw
11308 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11309                          iRegIorL2I src1, iRegIorL2I src2,
11310                          immI src3, immI_M1 src4) %{
11311   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11312   ins_cost(1.9 * INSN_COST);
11313   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11314 
11315   ins_encode %{
11316     __ bicw(as_Register($dst$$reg),
11317               as_Register($src1$$reg),
11318               as_Register($src2$$reg),
11319               Assembler::LSL,
11320               $src3$$constant & 0x1f);
11321   %}
11322 
11323   ins_pipe(ialu_reg_reg_shift);
11324 %}
11325 
11326 // This pattern is automatically generated from aarch64_ad.m4
11327 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11328 // val & (-1 ^ (val << shift)) ==> bic
11329 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11330                          iRegL src1, iRegL src2,
11331                          immI src3, immL_M1 src4) %{
11332   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11333   ins_cost(1.9 * INSN_COST);
11334   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11335 
11336   ins_encode %{
11337     __ bic(as_Register($dst$$reg),
11338               as_Register($src1$$reg),
11339               as_Register($src2$$reg),
11340               Assembler::LSL,
11341               $src3$$constant & 0x3f);
11342   %}
11343 
11344   ins_pipe(ialu_reg_reg_shift);
11345 %}
11346 
11347 // This pattern is automatically generated from aarch64_ad.m4
11348 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11349 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11350 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11351                          iRegIorL2I src1, iRegIorL2I src2,
11352                          immI src3, immI_M1 src4) %{
11353   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11354   ins_cost(1.9 * INSN_COST);
11355   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11356 
11357   ins_encode %{
11358     __ eonw(as_Register($dst$$reg),
11359               as_Register($src1$$reg),
11360               as_Register($src2$$reg),
11361               Assembler::LSR,
11362               $src3$$constant & 0x1f);
11363   %}
11364 
11365   ins_pipe(ialu_reg_reg_shift);
11366 %}
11367 
11368 // This pattern is automatically generated from aarch64_ad.m4
11369 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11370 // val ^ (-1 ^ (val >>> shift)) ==> eon
11371 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11372                          iRegL src1, iRegL src2,
11373                          immI src3, immL_M1 src4) %{
11374   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11375   ins_cost(1.9 * INSN_COST);
11376   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11377 
11378   ins_encode %{
11379     __ eon(as_Register($dst$$reg),
11380               as_Register($src1$$reg),
11381               as_Register($src2$$reg),
11382               Assembler::LSR,
11383               $src3$$constant & 0x3f);
11384   %}
11385 
11386   ins_pipe(ialu_reg_reg_shift);
11387 %}
11388 
11389 // This pattern is automatically generated from aarch64_ad.m4
11390 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11391 // val ^ (-1 ^ (val >> shift)) ==> eonw
11392 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11393                          iRegIorL2I src1, iRegIorL2I src2,
11394                          immI src3, immI_M1 src4) %{
11395   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11396   ins_cost(1.9 * INSN_COST);
11397   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11398 
11399   ins_encode %{
11400     __ eonw(as_Register($dst$$reg),
11401               as_Register($src1$$reg),
11402               as_Register($src2$$reg),
11403               Assembler::ASR,
11404               $src3$$constant & 0x1f);
11405   %}
11406 
11407   ins_pipe(ialu_reg_reg_shift);
11408 %}
11409 
11410 // This pattern is automatically generated from aarch64_ad.m4
11411 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11412 // val ^ (-1 ^ (val >> shift)) ==> eon
11413 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11414                          iRegL src1, iRegL src2,
11415                          immI src3, immL_M1 src4) %{
11416   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11417   ins_cost(1.9 * INSN_COST);
11418   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11419 
11420   ins_encode %{
11421     __ eon(as_Register($dst$$reg),
11422               as_Register($src1$$reg),
11423               as_Register($src2$$reg),
11424               Assembler::ASR,
11425               $src3$$constant & 0x3f);
11426   %}
11427 
11428   ins_pipe(ialu_reg_reg_shift);
11429 %}
11430 
11431 // This pattern is automatically generated from aarch64_ad.m4
11432 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11433 // val ^ (-1 ^ (val ror shift)) ==> eonw
11434 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11435                          iRegIorL2I src1, iRegIorL2I src2,
11436                          immI src3, immI_M1 src4) %{
11437   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11438   ins_cost(1.9 * INSN_COST);
11439   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11440 
11441   ins_encode %{
11442     __ eonw(as_Register($dst$$reg),
11443               as_Register($src1$$reg),
11444               as_Register($src2$$reg),
11445               Assembler::ROR,
11446               $src3$$constant & 0x1f);
11447   %}
11448 
11449   ins_pipe(ialu_reg_reg_shift);
11450 %}
11451 
11452 // This pattern is automatically generated from aarch64_ad.m4
11453 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11454 // val ^ (-1 ^ (val ror shift)) ==> eon
11455 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11456                          iRegL src1, iRegL src2,
11457                          immI src3, immL_M1 src4) %{
11458   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11459   ins_cost(1.9 * INSN_COST);
11460   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11461 
11462   ins_encode %{
11463     __ eon(as_Register($dst$$reg),
11464               as_Register($src1$$reg),
11465               as_Register($src2$$reg),
11466               Assembler::ROR,
11467               $src3$$constant & 0x3f);
11468   %}
11469 
11470   ins_pipe(ialu_reg_reg_shift);
11471 %}
11472 
11473 // This pattern is automatically generated from aarch64_ad.m4
11474 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11475 // val ^ (-1 ^ (val << shift)) ==> eonw
11476 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11477                          iRegIorL2I src1, iRegIorL2I src2,
11478                          immI src3, immI_M1 src4) %{
11479   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11480   ins_cost(1.9 * INSN_COST);
11481   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11482 
11483   ins_encode %{
11484     __ eonw(as_Register($dst$$reg),
11485               as_Register($src1$$reg),
11486               as_Register($src2$$reg),
11487               Assembler::LSL,
11488               $src3$$constant & 0x1f);
11489   %}
11490 
11491   ins_pipe(ialu_reg_reg_shift);
11492 %}
11493 
11494 // This pattern is automatically generated from aarch64_ad.m4
11495 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11496 // val ^ (-1 ^ (val << shift)) ==> eon
11497 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11498                          iRegL src1, iRegL src2,
11499                          immI src3, immL_M1 src4) %{
11500   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11501   ins_cost(1.9 * INSN_COST);
11502   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11503 
11504   ins_encode %{
11505     __ eon(as_Register($dst$$reg),
11506               as_Register($src1$$reg),
11507               as_Register($src2$$reg),
11508               Assembler::LSL,
11509               $src3$$constant & 0x3f);
11510   %}
11511 
11512   ins_pipe(ialu_reg_reg_shift);
11513 %}
11514 
11515 // This pattern is automatically generated from aarch64_ad.m4
11516 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11517 // val | (-1 ^ (val >>> shift)) ==> ornw
11518 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11519                          iRegIorL2I src1, iRegIorL2I src2,
11520                          immI src3, immI_M1 src4) %{
11521   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11522   ins_cost(1.9 * INSN_COST);
11523   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11524 
11525   ins_encode %{
11526     __ ornw(as_Register($dst$$reg),
11527               as_Register($src1$$reg),
11528               as_Register($src2$$reg),
11529               Assembler::LSR,
11530               $src3$$constant & 0x1f);
11531   %}
11532 
11533   ins_pipe(ialu_reg_reg_shift);
11534 %}
11535 
11536 // This pattern is automatically generated from aarch64_ad.m4
11537 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11538 // val | (-1 ^ (val >>> shift)) ==> orn
11539 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11540                          iRegL src1, iRegL src2,
11541                          immI src3, immL_M1 src4) %{
11542   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11543   ins_cost(1.9 * INSN_COST);
11544   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11545 
11546   ins_encode %{
11547     __ orn(as_Register($dst$$reg),
11548               as_Register($src1$$reg),
11549               as_Register($src2$$reg),
11550               Assembler::LSR,
11551               $src3$$constant & 0x3f);
11552   %}
11553 
11554   ins_pipe(ialu_reg_reg_shift);
11555 %}
11556 
11557 // This pattern is automatically generated from aarch64_ad.m4
11558 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11559 // val | (-1 ^ (val >> shift)) ==> ornw
11560 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11561                          iRegIorL2I src1, iRegIorL2I src2,
11562                          immI src3, immI_M1 src4) %{
11563   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11564   ins_cost(1.9 * INSN_COST);
11565   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11566 
11567   ins_encode %{
11568     __ ornw(as_Register($dst$$reg),
11569               as_Register($src1$$reg),
11570               as_Register($src2$$reg),
11571               Assembler::ASR,
11572               $src3$$constant & 0x1f);
11573   %}
11574 
11575   ins_pipe(ialu_reg_reg_shift);
11576 %}
11577 
11578 // This pattern is automatically generated from aarch64_ad.m4
11579 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11580 // val | (-1 ^ (val >> shift)) ==> orn
11581 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11582                          iRegL src1, iRegL src2,
11583                          immI src3, immL_M1 src4) %{
11584   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11585   ins_cost(1.9 * INSN_COST);
11586   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11587 
11588   ins_encode %{
11589     __ orn(as_Register($dst$$reg),
11590               as_Register($src1$$reg),
11591               as_Register($src2$$reg),
11592               Assembler::ASR,
11593               $src3$$constant & 0x3f);
11594   %}
11595 
11596   ins_pipe(ialu_reg_reg_shift);
11597 %}
11598 
11599 // This pattern is automatically generated from aarch64_ad.m4
11600 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11601 // val | (-1 ^ (val ror shift)) ==> ornw
11602 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11603                          iRegIorL2I src1, iRegIorL2I src2,
11604                          immI src3, immI_M1 src4) %{
11605   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11606   ins_cost(1.9 * INSN_COST);
11607   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11608 
11609   ins_encode %{
11610     __ ornw(as_Register($dst$$reg),
11611               as_Register($src1$$reg),
11612               as_Register($src2$$reg),
11613               Assembler::ROR,
11614               $src3$$constant & 0x1f);
11615   %}
11616 
11617   ins_pipe(ialu_reg_reg_shift);
11618 %}
11619 
11620 // This pattern is automatically generated from aarch64_ad.m4
11621 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11622 // val | (-1 ^ (val ror shift)) ==> orn
11623 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11624                          iRegL src1, iRegL src2,
11625                          immI src3, immL_M1 src4) %{
11626   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11627   ins_cost(1.9 * INSN_COST);
11628   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11629 
11630   ins_encode %{
11631     __ orn(as_Register($dst$$reg),
11632               as_Register($src1$$reg),
11633               as_Register($src2$$reg),
11634               Assembler::ROR,
11635               $src3$$constant & 0x3f);
11636   %}
11637 
11638   ins_pipe(ialu_reg_reg_shift);
11639 %}
11640 
11641 // This pattern is automatically generated from aarch64_ad.m4
11642 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11643 // val | (-1 ^ (val << shift)) ==> ornw
11644 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11645                          iRegIorL2I src1, iRegIorL2I src2,
11646                          immI src3, immI_M1 src4) %{
11647   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11648   ins_cost(1.9 * INSN_COST);
11649   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11650 
11651   ins_encode %{
11652     __ ornw(as_Register($dst$$reg),
11653               as_Register($src1$$reg),
11654               as_Register($src2$$reg),
11655               Assembler::LSL,
11656               $src3$$constant & 0x1f);
11657   %}
11658 
11659   ins_pipe(ialu_reg_reg_shift);
11660 %}
11661 
11662 // This pattern is automatically generated from aarch64_ad.m4
11663 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11664 // val | (-1 ^ (val << shift)) ==> orn
11665 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11666                          iRegL src1, iRegL src2,
11667                          immI src3, immL_M1 src4) %{
11668   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11669   ins_cost(1.9 * INSN_COST);
11670   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11671 
11672   ins_encode %{
11673     __ orn(as_Register($dst$$reg),
11674               as_Register($src1$$reg),
11675               as_Register($src2$$reg),
11676               Assembler::LSL,
11677               $src3$$constant & 0x3f);
11678   %}
11679 
11680   ins_pipe(ialu_reg_reg_shift);
11681 %}
11682 
11683 // This pattern is automatically generated from aarch64_ad.m4
11684 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11685 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11686                          iRegIorL2I src1, iRegIorL2I src2,
11687                          immI src3) %{
11688   match(Set dst (AndI src1 (URShiftI src2 src3)));
11689 
11690   ins_cost(1.9 * INSN_COST);
11691   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
11692 
11693   ins_encode %{
11694     __ andw(as_Register($dst$$reg),
11695               as_Register($src1$$reg),
11696               as_Register($src2$$reg),
11697               Assembler::LSR,
11698               $src3$$constant & 0x1f);
11699   %}
11700 
11701   ins_pipe(ialu_reg_reg_shift);
11702 %}
11703 
11704 // This pattern is automatically generated from aarch64_ad.m4
11705 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11706 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11707                          iRegL src1, iRegL src2,
11708                          immI src3) %{
11709   match(Set dst (AndL src1 (URShiftL src2 src3)));
11710 
11711   ins_cost(1.9 * INSN_COST);
11712   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
11713 
11714   ins_encode %{
11715     __ andr(as_Register($dst$$reg),
11716               as_Register($src1$$reg),
11717               as_Register($src2$$reg),
11718               Assembler::LSR,
11719               $src3$$constant & 0x3f);
11720   %}
11721 
11722   ins_pipe(ialu_reg_reg_shift);
11723 %}
11724 
11725 // This pattern is automatically generated from aarch64_ad.m4
11726 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11727 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11728                          iRegIorL2I src1, iRegIorL2I src2,
11729                          immI src3) %{
11730   match(Set dst (AndI src1 (RShiftI src2 src3)));
11731 
11732   ins_cost(1.9 * INSN_COST);
11733   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
11734 
11735   ins_encode %{
11736     __ andw(as_Register($dst$$reg),
11737               as_Register($src1$$reg),
11738               as_Register($src2$$reg),
11739               Assembler::ASR,
11740               $src3$$constant & 0x1f);
11741   %}
11742 
11743   ins_pipe(ialu_reg_reg_shift);
11744 %}
11745 
11746 // This pattern is automatically generated from aarch64_ad.m4
11747 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11748 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11749                          iRegL src1, iRegL src2,
11750                          immI src3) %{
11751   match(Set dst (AndL src1 (RShiftL src2 src3)));
11752 
11753   ins_cost(1.9 * INSN_COST);
11754   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
11755 
11756   ins_encode %{
11757     __ andr(as_Register($dst$$reg),
11758               as_Register($src1$$reg),
11759               as_Register($src2$$reg),
11760               Assembler::ASR,
11761               $src3$$constant & 0x3f);
11762   %}
11763 
11764   ins_pipe(ialu_reg_reg_shift);
11765 %}
11766 
11767 // This pattern is automatically generated from aarch64_ad.m4
11768 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11769 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11770                          iRegIorL2I src1, iRegIorL2I src2,
11771                          immI src3) %{
11772   match(Set dst (AndI src1 (LShiftI src2 src3)));
11773 
11774   ins_cost(1.9 * INSN_COST);
11775   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
11776 
11777   ins_encode %{
11778     __ andw(as_Register($dst$$reg),
11779               as_Register($src1$$reg),
11780               as_Register($src2$$reg),
11781               Assembler::LSL,
11782               $src3$$constant & 0x1f);
11783   %}
11784 
11785   ins_pipe(ialu_reg_reg_shift);
11786 %}
11787 
11788 // This pattern is automatically generated from aarch64_ad.m4
11789 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11790 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11791                          iRegL src1, iRegL src2,
11792                          immI src3) %{
11793   match(Set dst (AndL src1 (LShiftL src2 src3)));
11794 
11795   ins_cost(1.9 * INSN_COST);
11796   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
11797 
11798   ins_encode %{
11799     __ andr(as_Register($dst$$reg),
11800               as_Register($src1$$reg),
11801               as_Register($src2$$reg),
11802               Assembler::LSL,
11803               $src3$$constant & 0x3f);
11804   %}
11805 
11806   ins_pipe(ialu_reg_reg_shift);
11807 %}
11808 
11809 // This pattern is automatically generated from aarch64_ad.m4
11810 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11811 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11812                          iRegIorL2I src1, iRegIorL2I src2,
11813                          immI src3) %{
11814   match(Set dst (AndI src1 (RotateRight src2 src3)));
11815 
11816   ins_cost(1.9 * INSN_COST);
11817   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
11818 
11819   ins_encode %{
11820     __ andw(as_Register($dst$$reg),
11821               as_Register($src1$$reg),
11822               as_Register($src2$$reg),
11823               Assembler::ROR,
11824               $src3$$constant & 0x1f);
11825   %}
11826 
11827   ins_pipe(ialu_reg_reg_shift);
11828 %}
11829 
11830 // This pattern is automatically generated from aarch64_ad.m4
11831 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11832 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11833                          iRegL src1, iRegL src2,
11834                          immI src3) %{
11835   match(Set dst (AndL src1 (RotateRight src2 src3)));
11836 
11837   ins_cost(1.9 * INSN_COST);
11838   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
11839 
11840   ins_encode %{
11841     __ andr(as_Register($dst$$reg),
11842               as_Register($src1$$reg),
11843               as_Register($src2$$reg),
11844               Assembler::ROR,
11845               $src3$$constant & 0x3f);
11846   %}
11847 
11848   ins_pipe(ialu_reg_reg_shift);
11849 %}
11850 
11851 // This pattern is automatically generated from aarch64_ad.m4
11852 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11853 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11854                          iRegIorL2I src1, iRegIorL2I src2,
11855                          immI src3) %{
11856   match(Set dst (XorI src1 (URShiftI src2 src3)));
11857 
11858   ins_cost(1.9 * INSN_COST);
11859   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
11860 
11861   ins_encode %{
11862     __ eorw(as_Register($dst$$reg),
11863               as_Register($src1$$reg),
11864               as_Register($src2$$reg),
11865               Assembler::LSR,
11866               $src3$$constant & 0x1f);
11867   %}
11868 
11869   ins_pipe(ialu_reg_reg_shift);
11870 %}
11871 
11872 // This pattern is automatically generated from aarch64_ad.m4
11873 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11874 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11875                          iRegL src1, iRegL src2,
11876                          immI src3) %{
11877   match(Set dst (XorL src1 (URShiftL src2 src3)));
11878 
11879   ins_cost(1.9 * INSN_COST);
11880   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
11881 
11882   ins_encode %{
11883     __ eor(as_Register($dst$$reg),
11884               as_Register($src1$$reg),
11885               as_Register($src2$$reg),
11886               Assembler::LSR,
11887               $src3$$constant & 0x3f);
11888   %}
11889 
11890   ins_pipe(ialu_reg_reg_shift);
11891 %}
11892 
11893 // This pattern is automatically generated from aarch64_ad.m4
11894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11895 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11896                          iRegIorL2I src1, iRegIorL2I src2,
11897                          immI src3) %{
11898   match(Set dst (XorI src1 (RShiftI src2 src3)));
11899 
11900   ins_cost(1.9 * INSN_COST);
11901   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
11902 
11903   ins_encode %{
11904     __ eorw(as_Register($dst$$reg),
11905               as_Register($src1$$reg),
11906               as_Register($src2$$reg),
11907               Assembler::ASR,
11908               $src3$$constant & 0x1f);
11909   %}
11910 
11911   ins_pipe(ialu_reg_reg_shift);
11912 %}
11913 
11914 // This pattern is automatically generated from aarch64_ad.m4
11915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11916 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11917                          iRegL src1, iRegL src2,
11918                          immI src3) %{
11919   match(Set dst (XorL src1 (RShiftL src2 src3)));
11920 
11921   ins_cost(1.9 * INSN_COST);
11922   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
11923 
11924   ins_encode %{
11925     __ eor(as_Register($dst$$reg),
11926               as_Register($src1$$reg),
11927               as_Register($src2$$reg),
11928               Assembler::ASR,
11929               $src3$$constant & 0x3f);
11930   %}
11931 
11932   ins_pipe(ialu_reg_reg_shift);
11933 %}
11934 
11935 // This pattern is automatically generated from aarch64_ad.m4
11936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11937 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11938                          iRegIorL2I src1, iRegIorL2I src2,
11939                          immI src3) %{
11940   match(Set dst (XorI src1 (LShiftI src2 src3)));
11941 
11942   ins_cost(1.9 * INSN_COST);
11943   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
11944 
11945   ins_encode %{
11946     __ eorw(as_Register($dst$$reg),
11947               as_Register($src1$$reg),
11948               as_Register($src2$$reg),
11949               Assembler::LSL,
11950               $src3$$constant & 0x1f);
11951   %}
11952 
11953   ins_pipe(ialu_reg_reg_shift);
11954 %}
11955 
11956 // This pattern is automatically generated from aarch64_ad.m4
11957 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11958 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11959                          iRegL src1, iRegL src2,
11960                          immI src3) %{
11961   match(Set dst (XorL src1 (LShiftL src2 src3)));
11962 
11963   ins_cost(1.9 * INSN_COST);
11964   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
11965 
11966   ins_encode %{
11967     __ eor(as_Register($dst$$reg),
11968               as_Register($src1$$reg),
11969               as_Register($src2$$reg),
11970               Assembler::LSL,
11971               $src3$$constant & 0x3f);
11972   %}
11973 
11974   ins_pipe(ialu_reg_reg_shift);
11975 %}
11976 
11977 // This pattern is automatically generated from aarch64_ad.m4
11978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11979 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11980                          iRegIorL2I src1, iRegIorL2I src2,
11981                          immI src3) %{
11982   match(Set dst (XorI src1 (RotateRight src2 src3)));
11983 
11984   ins_cost(1.9 * INSN_COST);
11985   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
11986 
11987   ins_encode %{
11988     __ eorw(as_Register($dst$$reg),
11989               as_Register($src1$$reg),
11990               as_Register($src2$$reg),
11991               Assembler::ROR,
11992               $src3$$constant & 0x1f);
11993   %}
11994 
11995   ins_pipe(ialu_reg_reg_shift);
11996 %}
11997 
11998 // This pattern is automatically generated from aarch64_ad.m4
11999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12000 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12001                          iRegL src1, iRegL src2,
12002                          immI src3) %{
12003   match(Set dst (XorL src1 (RotateRight src2 src3)));
12004 
12005   ins_cost(1.9 * INSN_COST);
12006   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12007 
12008   ins_encode %{
12009     __ eor(as_Register($dst$$reg),
12010               as_Register($src1$$reg),
12011               as_Register($src2$$reg),
12012               Assembler::ROR,
12013               $src3$$constant & 0x3f);
12014   %}
12015 
12016   ins_pipe(ialu_reg_reg_shift);
12017 %}
12018 
12019 // This pattern is automatically generated from aarch64_ad.m4
12020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12021 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12022                          iRegIorL2I src1, iRegIorL2I src2,
12023                          immI src3) %{
12024   match(Set dst (OrI src1 (URShiftI src2 src3)));
12025 
12026   ins_cost(1.9 * INSN_COST);
12027   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12028 
12029   ins_encode %{
12030     __ orrw(as_Register($dst$$reg),
12031               as_Register($src1$$reg),
12032               as_Register($src2$$reg),
12033               Assembler::LSR,
12034               $src3$$constant & 0x1f);
12035   %}
12036 
12037   ins_pipe(ialu_reg_reg_shift);
12038 %}
12039 
12040 // This pattern is automatically generated from aarch64_ad.m4
12041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12042 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12043                          iRegL src1, iRegL src2,
12044                          immI src3) %{
12045   match(Set dst (OrL src1 (URShiftL src2 src3)));
12046 
12047   ins_cost(1.9 * INSN_COST);
12048   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12049 
12050   ins_encode %{
12051     __ orr(as_Register($dst$$reg),
12052               as_Register($src1$$reg),
12053               as_Register($src2$$reg),
12054               Assembler::LSR,
12055               $src3$$constant & 0x3f);
12056   %}
12057 
12058   ins_pipe(ialu_reg_reg_shift);
12059 %}
12060 
12061 // This pattern is automatically generated from aarch64_ad.m4
12062 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12063 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12064                          iRegIorL2I src1, iRegIorL2I src2,
12065                          immI src3) %{
12066   match(Set dst (OrI src1 (RShiftI src2 src3)));
12067 
12068   ins_cost(1.9 * INSN_COST);
12069   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12070 
12071   ins_encode %{
12072     __ orrw(as_Register($dst$$reg),
12073               as_Register($src1$$reg),
12074               as_Register($src2$$reg),
12075               Assembler::ASR,
12076               $src3$$constant & 0x1f);
12077   %}
12078 
12079   ins_pipe(ialu_reg_reg_shift);
12080 %}
12081 
12082 // This pattern is automatically generated from aarch64_ad.m4
12083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12084 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12085                          iRegL src1, iRegL src2,
12086                          immI src3) %{
12087   match(Set dst (OrL src1 (RShiftL src2 src3)));
12088 
12089   ins_cost(1.9 * INSN_COST);
12090   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12091 
12092   ins_encode %{
12093     __ orr(as_Register($dst$$reg),
12094               as_Register($src1$$reg),
12095               as_Register($src2$$reg),
12096               Assembler::ASR,
12097               $src3$$constant & 0x3f);
12098   %}
12099 
12100   ins_pipe(ialu_reg_reg_shift);
12101 %}
12102 
12103 // This pattern is automatically generated from aarch64_ad.m4
12104 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12105 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12106                          iRegIorL2I src1, iRegIorL2I src2,
12107                          immI src3) %{
12108   match(Set dst (OrI src1 (LShiftI src2 src3)));
12109 
12110   ins_cost(1.9 * INSN_COST);
12111   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12112 
12113   ins_encode %{
12114     __ orrw(as_Register($dst$$reg),
12115               as_Register($src1$$reg),
12116               as_Register($src2$$reg),
12117               Assembler::LSL,
12118               $src3$$constant & 0x1f);
12119   %}
12120 
12121   ins_pipe(ialu_reg_reg_shift);
12122 %}
12123 
12124 // This pattern is automatically generated from aarch64_ad.m4
12125 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12126 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12127                          iRegL src1, iRegL src2,
12128                          immI src3) %{
12129   match(Set dst (OrL src1 (LShiftL src2 src3)));
12130 
12131   ins_cost(1.9 * INSN_COST);
12132   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12133 
12134   ins_encode %{
12135     __ orr(as_Register($dst$$reg),
12136               as_Register($src1$$reg),
12137               as_Register($src2$$reg),
12138               Assembler::LSL,
12139               $src3$$constant & 0x3f);
12140   %}
12141 
12142   ins_pipe(ialu_reg_reg_shift);
12143 %}
12144 
12145 // This pattern is automatically generated from aarch64_ad.m4
12146 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12147 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12148                          iRegIorL2I src1, iRegIorL2I src2,
12149                          immI src3) %{
12150   match(Set dst (OrI src1 (RotateRight src2 src3)));
12151 
12152   ins_cost(1.9 * INSN_COST);
12153   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12154 
12155   ins_encode %{
12156     __ orrw(as_Register($dst$$reg),
12157               as_Register($src1$$reg),
12158               as_Register($src2$$reg),
12159               Assembler::ROR,
12160               $src3$$constant & 0x1f);
12161   %}
12162 
12163   ins_pipe(ialu_reg_reg_shift);
12164 %}
12165 
12166 // This pattern is automatically generated from aarch64_ad.m4
12167 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12168 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12169                          iRegL src1, iRegL src2,
12170                          immI src3) %{
12171   match(Set dst (OrL src1 (RotateRight src2 src3)));
12172 
12173   ins_cost(1.9 * INSN_COST);
12174   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12175 
12176   ins_encode %{
12177     __ orr(as_Register($dst$$reg),
12178               as_Register($src1$$reg),
12179               as_Register($src2$$reg),
12180               Assembler::ROR,
12181               $src3$$constant & 0x3f);
12182   %}
12183 
12184   ins_pipe(ialu_reg_reg_shift);
12185 %}
12186 
12187 // This pattern is automatically generated from aarch64_ad.m4
12188 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12189 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12190                          iRegIorL2I src1, iRegIorL2I src2,
12191                          immI src3) %{
12192   match(Set dst (AddI src1 (URShiftI src2 src3)));
12193 
12194   ins_cost(1.9 * INSN_COST);
12195   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12196 
12197   ins_encode %{
12198     __ addw(as_Register($dst$$reg),
12199               as_Register($src1$$reg),
12200               as_Register($src2$$reg),
12201               Assembler::LSR,
12202               $src3$$constant & 0x1f);
12203   %}
12204 
12205   ins_pipe(ialu_reg_reg_shift);
12206 %}
12207 
12208 // This pattern is automatically generated from aarch64_ad.m4
12209 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12210 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12211                          iRegL src1, iRegL src2,
12212                          immI src3) %{
12213   match(Set dst (AddL src1 (URShiftL src2 src3)));
12214 
12215   ins_cost(1.9 * INSN_COST);
12216   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12217 
12218   ins_encode %{
12219     __ add(as_Register($dst$$reg),
12220               as_Register($src1$$reg),
12221               as_Register($src2$$reg),
12222               Assembler::LSR,
12223               $src3$$constant & 0x3f);
12224   %}
12225 
12226   ins_pipe(ialu_reg_reg_shift);
12227 %}
12228 
12229 // This pattern is automatically generated from aarch64_ad.m4
12230 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12231 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12232                          iRegIorL2I src1, iRegIorL2I src2,
12233                          immI src3) %{
12234   match(Set dst (AddI src1 (RShiftI src2 src3)));
12235 
12236   ins_cost(1.9 * INSN_COST);
12237   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12238 
12239   ins_encode %{
12240     __ addw(as_Register($dst$$reg),
12241               as_Register($src1$$reg),
12242               as_Register($src2$$reg),
12243               Assembler::ASR,
12244               $src3$$constant & 0x1f);
12245   %}
12246 
12247   ins_pipe(ialu_reg_reg_shift);
12248 %}
12249 
12250 // This pattern is automatically generated from aarch64_ad.m4
12251 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12252 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12253                          iRegL src1, iRegL src2,
12254                          immI src3) %{
12255   match(Set dst (AddL src1 (RShiftL src2 src3)));
12256 
12257   ins_cost(1.9 * INSN_COST);
12258   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12259 
12260   ins_encode %{
12261     __ add(as_Register($dst$$reg),
12262               as_Register($src1$$reg),
12263               as_Register($src2$$reg),
12264               Assembler::ASR,
12265               $src3$$constant & 0x3f);
12266   %}
12267 
12268   ins_pipe(ialu_reg_reg_shift);
12269 %}
12270 
12271 // This pattern is automatically generated from aarch64_ad.m4
12272 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12273 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12274                          iRegIorL2I src1, iRegIorL2I src2,
12275                          immI src3) %{
12276   match(Set dst (AddI src1 (LShiftI src2 src3)));
12277 
12278   ins_cost(1.9 * INSN_COST);
12279   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12280 
12281   ins_encode %{
12282     __ addw(as_Register($dst$$reg),
12283               as_Register($src1$$reg),
12284               as_Register($src2$$reg),
12285               Assembler::LSL,
12286               $src3$$constant & 0x1f);
12287   %}
12288 
12289   ins_pipe(ialu_reg_reg_shift);
12290 %}
12291 
12292 // This pattern is automatically generated from aarch64_ad.m4
12293 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12294 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12295                          iRegL src1, iRegL src2,
12296                          immI src3) %{
12297   match(Set dst (AddL src1 (LShiftL src2 src3)));
12298 
12299   ins_cost(1.9 * INSN_COST);
12300   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12301 
12302   ins_encode %{
12303     __ add(as_Register($dst$$reg),
12304               as_Register($src1$$reg),
12305               as_Register($src2$$reg),
12306               Assembler::LSL,
12307               $src3$$constant & 0x3f);
12308   %}
12309 
12310   ins_pipe(ialu_reg_reg_shift);
12311 %}
12312 
12313 // This pattern is automatically generated from aarch64_ad.m4
12314 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12315 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12316                          iRegIorL2I src1, iRegIorL2I src2,
12317                          immI src3) %{
12318   match(Set dst (SubI src1 (URShiftI src2 src3)));
12319 
12320   ins_cost(1.9 * INSN_COST);
12321   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12322 
12323   ins_encode %{
12324     __ subw(as_Register($dst$$reg),
12325               as_Register($src1$$reg),
12326               as_Register($src2$$reg),
12327               Assembler::LSR,
12328               $src3$$constant & 0x1f);
12329   %}
12330 
12331   ins_pipe(ialu_reg_reg_shift);
12332 %}
12333 
12334 // This pattern is automatically generated from aarch64_ad.m4
12335 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12336 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12337                          iRegL src1, iRegL src2,
12338                          immI src3) %{
12339   match(Set dst (SubL src1 (URShiftL src2 src3)));
12340 
12341   ins_cost(1.9 * INSN_COST);
12342   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12343 
12344   ins_encode %{
12345     __ sub(as_Register($dst$$reg),
12346               as_Register($src1$$reg),
12347               as_Register($src2$$reg),
12348               Assembler::LSR,
12349               $src3$$constant & 0x3f);
12350   %}
12351 
12352   ins_pipe(ialu_reg_reg_shift);
12353 %}
12354 
12355 // This pattern is automatically generated from aarch64_ad.m4
12356 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12357 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12358                          iRegIorL2I src1, iRegIorL2I src2,
12359                          immI src3) %{
12360   match(Set dst (SubI src1 (RShiftI src2 src3)));
12361 
12362   ins_cost(1.9 * INSN_COST);
12363   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12364 
12365   ins_encode %{
12366     __ subw(as_Register($dst$$reg),
12367               as_Register($src1$$reg),
12368               as_Register($src2$$reg),
12369               Assembler::ASR,
12370               $src3$$constant & 0x1f);
12371   %}
12372 
12373   ins_pipe(ialu_reg_reg_shift);
12374 %}
12375 
12376 // This pattern is automatically generated from aarch64_ad.m4
12377 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12378 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12379                          iRegL src1, iRegL src2,
12380                          immI src3) %{
12381   match(Set dst (SubL src1 (RShiftL src2 src3)));
12382 
12383   ins_cost(1.9 * INSN_COST);
12384   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12385 
12386   ins_encode %{
12387     __ sub(as_Register($dst$$reg),
12388               as_Register($src1$$reg),
12389               as_Register($src2$$reg),
12390               Assembler::ASR,
12391               $src3$$constant & 0x3f);
12392   %}
12393 
12394   ins_pipe(ialu_reg_reg_shift);
12395 %}
12396 
12397 // This pattern is automatically generated from aarch64_ad.m4
12398 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12399 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12400                          iRegIorL2I src1, iRegIorL2I src2,
12401                          immI src3) %{
12402   match(Set dst (SubI src1 (LShiftI src2 src3)));
12403 
12404   ins_cost(1.9 * INSN_COST);
12405   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12406 
12407   ins_encode %{
12408     __ subw(as_Register($dst$$reg),
12409               as_Register($src1$$reg),
12410               as_Register($src2$$reg),
12411               Assembler::LSL,
12412               $src3$$constant & 0x1f);
12413   %}
12414 
12415   ins_pipe(ialu_reg_reg_shift);
12416 %}
12417 
12418 // This pattern is automatically generated from aarch64_ad.m4
12419 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12420 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12421                          iRegL src1, iRegL src2,
12422                          immI src3) %{
12423   match(Set dst (SubL src1 (LShiftL src2 src3)));
12424 
12425   ins_cost(1.9 * INSN_COST);
12426   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12427 
12428   ins_encode %{
12429     __ sub(as_Register($dst$$reg),
12430               as_Register($src1$$reg),
12431               as_Register($src2$$reg),
12432               Assembler::LSL,
12433               $src3$$constant & 0x3f);
12434   %}
12435 
12436   ins_pipe(ialu_reg_reg_shift);
12437 %}
12438 
12439 // This pattern is automatically generated from aarch64_ad.m4
12440 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12441 
12442 // Shift Left followed by Shift Right.
12443 // This idiom is used by the compiler for the i2b bytecode etc.
12444 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12445 %{
12446   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12447   ins_cost(INSN_COST * 2);
12448   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12449   ins_encode %{
12450     int lshift = $lshift_count$$constant & 63;
12451     int rshift = $rshift_count$$constant & 63;
12452     int s = 63 - lshift;
12453     int r = (rshift - lshift) & 63;
12454     __ sbfm(as_Register($dst$$reg),
12455             as_Register($src$$reg),
12456             r, s);
12457   %}
12458 
12459   ins_pipe(ialu_reg_shift);
12460 %}
12461 
12462 // This pattern is automatically generated from aarch64_ad.m4
12463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12464 
12465 // Shift Left followed by Shift Right.
12466 // This idiom is used by the compiler for the i2b bytecode etc.
12467 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12468 %{
12469   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12470   ins_cost(INSN_COST * 2);
12471   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12472   ins_encode %{
12473     int lshift = $lshift_count$$constant & 31;
12474     int rshift = $rshift_count$$constant & 31;
12475     int s = 31 - lshift;
12476     int r = (rshift - lshift) & 31;
12477     __ sbfmw(as_Register($dst$$reg),
12478             as_Register($src$$reg),
12479             r, s);
12480   %}
12481 
12482   ins_pipe(ialu_reg_shift);
12483 %}
12484 
12485 // This pattern is automatically generated from aarch64_ad.m4
12486 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12487 
12488 // Shift Left followed by Shift Right.
12489 // This idiom is used by the compiler for the i2b bytecode etc.
12490 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12491 %{
12492   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12493   ins_cost(INSN_COST * 2);
12494   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12495   ins_encode %{
12496     int lshift = $lshift_count$$constant & 63;
12497     int rshift = $rshift_count$$constant & 63;
12498     int s = 63 - lshift;
12499     int r = (rshift - lshift) & 63;
12500     __ ubfm(as_Register($dst$$reg),
12501             as_Register($src$$reg),
12502             r, s);
12503   %}
12504 
12505   ins_pipe(ialu_reg_shift);
12506 %}
12507 
12508 // This pattern is automatically generated from aarch64_ad.m4
12509 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12510 
12511 // Shift Left followed by Shift Right.
12512 // This idiom is used by the compiler for the i2b bytecode etc.
12513 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12514 %{
12515   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12516   ins_cost(INSN_COST * 2);
12517   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12518   ins_encode %{
12519     int lshift = $lshift_count$$constant & 31;
12520     int rshift = $rshift_count$$constant & 31;
12521     int s = 31 - lshift;
12522     int r = (rshift - lshift) & 31;
12523     __ ubfmw(as_Register($dst$$reg),
12524             as_Register($src$$reg),
12525             r, s);
12526   %}
12527 
12528   ins_pipe(ialu_reg_shift);
12529 %}
12530 
12531 // Bitfield extract with shift & mask
12532 
12533 // This pattern is automatically generated from aarch64_ad.m4
12534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12535 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12536 %{
12537   match(Set dst (AndI (URShiftI src rshift) mask));
12538   // Make sure we are not going to exceed what ubfxw can do.
12539   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12540 
12541   ins_cost(INSN_COST);
12542   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12543   ins_encode %{
12544     int rshift = $rshift$$constant & 31;
12545     intptr_t mask = $mask$$constant;
12546     int width = exact_log2(mask+1);
12547     __ ubfxw(as_Register($dst$$reg),
12548             as_Register($src$$reg), rshift, width);
12549   %}
12550   ins_pipe(ialu_reg_shift);
12551 %}
12552 
12553 // This pattern is automatically generated from aarch64_ad.m4
12554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12555 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12556 %{
12557   match(Set dst (AndL (URShiftL src rshift) mask));
12558   // Make sure we are not going to exceed what ubfx can do.
12559   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12560 
12561   ins_cost(INSN_COST);
12562   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12563   ins_encode %{
12564     int rshift = $rshift$$constant & 63;
12565     intptr_t mask = $mask$$constant;
12566     int width = exact_log2_long(mask+1);
12567     __ ubfx(as_Register($dst$$reg),
12568             as_Register($src$$reg), rshift, width);
12569   %}
12570   ins_pipe(ialu_reg_shift);
12571 %}
12572 
12573 
12574 // This pattern is automatically generated from aarch64_ad.m4
12575 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12576 
12577 // We can use ubfx when extending an And with a mask when we know mask
12578 // is positive.  We know that because immI_bitmask guarantees it.
12579 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12580 %{
12581   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12582   // Make sure we are not going to exceed what ubfxw can do.
12583   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12584 
12585   ins_cost(INSN_COST * 2);
12586   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12587   ins_encode %{
12588     int rshift = $rshift$$constant & 31;
12589     intptr_t mask = $mask$$constant;
12590     int width = exact_log2(mask+1);
12591     __ ubfx(as_Register($dst$$reg),
12592             as_Register($src$$reg), rshift, width);
12593   %}
12594   ins_pipe(ialu_reg_shift);
12595 %}
12596 
12597 
12598 // This pattern is automatically generated from aarch64_ad.m4
12599 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12600 
12601 // We can use ubfiz when masking by a positive number and then left shifting the result.
12602 // We know that the mask is positive because immI_bitmask guarantees it.
12603 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12604 %{
12605   match(Set dst (LShiftI (AndI src mask) lshift));
12606   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12607 
12608   ins_cost(INSN_COST);
12609   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12610   ins_encode %{
12611     int lshift = $lshift$$constant & 31;
12612     intptr_t mask = $mask$$constant;
12613     int width = exact_log2(mask+1);
12614     __ ubfizw(as_Register($dst$$reg),
12615           as_Register($src$$reg), lshift, width);
12616   %}
12617   ins_pipe(ialu_reg_shift);
12618 %}
12619 
12620 // This pattern is automatically generated from aarch64_ad.m4
12621 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12622 
12623 // We can use ubfiz when masking by a positive number and then left shifting the result.
12624 // We know that the mask is positive because immL_bitmask guarantees it.
12625 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12626 %{
12627   match(Set dst (LShiftL (AndL src mask) lshift));
12628   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12629 
12630   ins_cost(INSN_COST);
12631   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12632   ins_encode %{
12633     int lshift = $lshift$$constant & 63;
12634     intptr_t mask = $mask$$constant;
12635     int width = exact_log2_long(mask+1);
12636     __ ubfiz(as_Register($dst$$reg),
12637           as_Register($src$$reg), lshift, width);
12638   %}
12639   ins_pipe(ialu_reg_shift);
12640 %}
12641 
12642 // This pattern is automatically generated from aarch64_ad.m4
12643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12644 
12645 // We can use ubfiz when masking by a positive number and then left shifting the result.
12646 // We know that the mask is positive because immI_bitmask guarantees it.
12647 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12648 %{
12649   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12650   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12651 
12652   ins_cost(INSN_COST);
12653   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12654   ins_encode %{
12655     int lshift = $lshift$$constant & 31;
12656     intptr_t mask = $mask$$constant;
12657     int width = exact_log2(mask+1);
12658     __ ubfizw(as_Register($dst$$reg),
12659           as_Register($src$$reg), lshift, width);
12660   %}
12661   ins_pipe(ialu_reg_shift);
12662 %}
12663 
12664 // This pattern is automatically generated from aarch64_ad.m4
12665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12666 
12667 // We can use ubfiz when masking by a positive number and then left shifting the result.
12668 // We know that the mask is positive because immL_bitmask guarantees it.
12669 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12670 %{
12671   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12672   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12673 
12674   ins_cost(INSN_COST);
12675   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12676   ins_encode %{
12677     int lshift = $lshift$$constant & 63;
12678     intptr_t mask = $mask$$constant;
12679     int width = exact_log2_long(mask+1);
12680     __ ubfiz(as_Register($dst$$reg),
12681           as_Register($src$$reg), lshift, width);
12682   %}
12683   ins_pipe(ialu_reg_shift);
12684 %}
12685 
12686 
12687 // This pattern is automatically generated from aarch64_ad.m4
12688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12689 
12690 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12691 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12692 %{
12693   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12694   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12695 
12696   ins_cost(INSN_COST);
12697   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12698   ins_encode %{
12699     int lshift = $lshift$$constant & 63;
12700     intptr_t mask = $mask$$constant;
12701     int width = exact_log2(mask+1);
12702     __ ubfiz(as_Register($dst$$reg),
12703              as_Register($src$$reg), lshift, width);
12704   %}
12705   ins_pipe(ialu_reg_shift);
12706 %}
12707 
12708 // This pattern is automatically generated from aarch64_ad.m4
12709 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12710 
12711 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12712 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12713 %{
12714   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12715   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12716 
12717   ins_cost(INSN_COST);
12718   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12719   ins_encode %{
12720     int lshift = $lshift$$constant & 31;
12721     intptr_t mask = $mask$$constant;
12722     int width = exact_log2(mask+1);
12723     __ ubfiz(as_Register($dst$$reg),
12724              as_Register($src$$reg), lshift, width);
12725   %}
12726   ins_pipe(ialu_reg_shift);
12727 %}
12728 
12729 // This pattern is automatically generated from aarch64_ad.m4
12730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12731 
12732 // Can skip int2long conversions after AND with small bitmask
12733 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12734 %{
12735   match(Set dst (ConvI2L (AndI src msk)));
12736   ins_cost(INSN_COST);
12737   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12738   ins_encode %{
12739     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12740   %}
12741   ins_pipe(ialu_reg_shift);
12742 %}
12743 
12744 
12745 // Rotations
12746 
12747 // This pattern is automatically generated from aarch64_ad.m4
12748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12749 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12750 %{
12751   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12752   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12753 
12754   ins_cost(INSN_COST);
12755   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12756 
12757   ins_encode %{
12758     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12759             $rshift$$constant & 63);
12760   %}
12761   ins_pipe(ialu_reg_reg_extr);
12762 %}
12763 
12764 
12765 // This pattern is automatically generated from aarch64_ad.m4
12766 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12767 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12768 %{
12769   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12770   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12771 
12772   ins_cost(INSN_COST);
12773   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12774 
12775   ins_encode %{
12776     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12777             $rshift$$constant & 31);
12778   %}
12779   ins_pipe(ialu_reg_reg_extr);
12780 %}
12781 
12782 
12783 // This pattern is automatically generated from aarch64_ad.m4
12784 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12785 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12786 %{
12787   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12788   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12789 
12790   ins_cost(INSN_COST);
12791   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12792 
12793   ins_encode %{
12794     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12795             $rshift$$constant & 63);
12796   %}
12797   ins_pipe(ialu_reg_reg_extr);
12798 %}
12799 
12800 
12801 // This pattern is automatically generated from aarch64_ad.m4
12802 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12803 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12804 %{
12805   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12806   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12807 
12808   ins_cost(INSN_COST);
12809   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12810 
12811   ins_encode %{
12812     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12813             $rshift$$constant & 31);
12814   %}
12815   ins_pipe(ialu_reg_reg_extr);
12816 %}
12817 
12818 // This pattern is automatically generated from aarch64_ad.m4
12819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12820 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12821 %{
12822   match(Set dst (RotateRight src shift));
12823 
12824   ins_cost(INSN_COST);
12825   format %{ "ror    $dst, $src, $shift" %}
12826 
12827   ins_encode %{
12828      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12829                $shift$$constant & 0x1f);
12830   %}
12831   ins_pipe(ialu_reg_reg_vshift);
12832 %}
12833 
12834 // This pattern is automatically generated from aarch64_ad.m4
12835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12836 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12837 %{
12838   match(Set dst (RotateRight src shift));
12839 
12840   ins_cost(INSN_COST);
12841   format %{ "ror    $dst, $src, $shift" %}
12842 
12843   ins_encode %{
12844      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12845                $shift$$constant & 0x3f);
12846   %}
12847   ins_pipe(ialu_reg_reg_vshift);
12848 %}
12849 
12850 // This pattern is automatically generated from aarch64_ad.m4
12851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12852 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12853 %{
12854   match(Set dst (RotateRight src shift));
12855 
12856   ins_cost(INSN_COST);
12857   format %{ "ror    $dst, $src, $shift" %}
12858 
12859   ins_encode %{
12860      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12861   %}
12862   ins_pipe(ialu_reg_reg_vshift);
12863 %}
12864 
12865 // This pattern is automatically generated from aarch64_ad.m4
12866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12867 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12868 %{
12869   match(Set dst (RotateRight src shift));
12870 
12871   ins_cost(INSN_COST);
12872   format %{ "ror    $dst, $src, $shift" %}
12873 
12874   ins_encode %{
12875      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12876   %}
12877   ins_pipe(ialu_reg_reg_vshift);
12878 %}
12879 
12880 // This pattern is automatically generated from aarch64_ad.m4
12881 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12882 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12883 %{
12884   match(Set dst (RotateLeft src shift));
12885 
12886   ins_cost(INSN_COST);
12887   format %{ "rol    $dst, $src, $shift" %}
12888 
12889   ins_encode %{
12890      __ subw(rscratch1, zr, as_Register($shift$$reg));
12891      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12892   %}
12893   ins_pipe(ialu_reg_reg_vshift);
12894 %}
12895 
12896 // This pattern is automatically generated from aarch64_ad.m4
12897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12898 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12899 %{
12900   match(Set dst (RotateLeft src shift));
12901 
12902   ins_cost(INSN_COST);
12903   format %{ "rol    $dst, $src, $shift" %}
12904 
12905   ins_encode %{
12906      __ subw(rscratch1, zr, as_Register($shift$$reg));
12907      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12908   %}
12909   ins_pipe(ialu_reg_reg_vshift);
12910 %}
12911 
12912 
12913 // Add/subtract (extended)
12914 
12915 // This pattern is automatically generated from aarch64_ad.m4
12916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12917 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12918 %{
12919   match(Set dst (AddL src1 (ConvI2L src2)));
12920   ins_cost(INSN_COST);
12921   format %{ "add  $dst, $src1, $src2, sxtw" %}
12922 
12923    ins_encode %{
12924      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12925             as_Register($src2$$reg), ext::sxtw);
12926    %}
12927   ins_pipe(ialu_reg_reg);
12928 %}
12929 
12930 // This pattern is automatically generated from aarch64_ad.m4
12931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12932 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12933 %{
12934   match(Set dst (SubL src1 (ConvI2L src2)));
12935   ins_cost(INSN_COST);
12936   format %{ "sub  $dst, $src1, $src2, sxtw" %}
12937 
12938    ins_encode %{
12939      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12940             as_Register($src2$$reg), ext::sxtw);
12941    %}
12942   ins_pipe(ialu_reg_reg);
12943 %}
12944 
12945 // This pattern is automatically generated from aarch64_ad.m4
12946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12947 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12948 %{
12949   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12950   ins_cost(INSN_COST);
12951   format %{ "add  $dst, $src1, $src2, sxth" %}
12952 
12953    ins_encode %{
12954      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12955             as_Register($src2$$reg), ext::sxth);
12956    %}
12957   ins_pipe(ialu_reg_reg);
12958 %}
12959 
12960 // This pattern is automatically generated from aarch64_ad.m4
12961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12962 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12963 %{
12964   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12965   ins_cost(INSN_COST);
12966   format %{ "add  $dst, $src1, $src2, sxtb" %}
12967 
12968    ins_encode %{
12969      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12970             as_Register($src2$$reg), ext::sxtb);
12971    %}
12972   ins_pipe(ialu_reg_reg);
12973 %}
12974 
12975 // This pattern is automatically generated from aarch64_ad.m4
12976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12977 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12978 %{
12979   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12980   ins_cost(INSN_COST);
12981   format %{ "add  $dst, $src1, $src2, uxtb" %}
12982 
12983    ins_encode %{
12984      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12985             as_Register($src2$$reg), ext::uxtb);
12986    %}
12987   ins_pipe(ialu_reg_reg);
12988 %}
12989 
12990 // This pattern is automatically generated from aarch64_ad.m4
12991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12992 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12993 %{
12994   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12995   ins_cost(INSN_COST);
12996   format %{ "add  $dst, $src1, $src2, sxth" %}
12997 
12998    ins_encode %{
12999      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13000             as_Register($src2$$reg), ext::sxth);
13001    %}
13002   ins_pipe(ialu_reg_reg);
13003 %}
13004 
13005 // This pattern is automatically generated from aarch64_ad.m4
13006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13007 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13008 %{
13009   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13010   ins_cost(INSN_COST);
13011   format %{ "add  $dst, $src1, $src2, sxtw" %}
13012 
13013    ins_encode %{
13014      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13015             as_Register($src2$$reg), ext::sxtw);
13016    %}
13017   ins_pipe(ialu_reg_reg);
13018 %}
13019 
13020 // This pattern is automatically generated from aarch64_ad.m4
13021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13022 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13023 %{
13024   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13025   ins_cost(INSN_COST);
13026   format %{ "add  $dst, $src1, $src2, sxtb" %}
13027 
13028    ins_encode %{
13029      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13030             as_Register($src2$$reg), ext::sxtb);
13031    %}
13032   ins_pipe(ialu_reg_reg);
13033 %}
13034 
13035 // This pattern is automatically generated from aarch64_ad.m4
13036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13037 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13038 %{
13039   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13040   ins_cost(INSN_COST);
13041   format %{ "add  $dst, $src1, $src2, uxtb" %}
13042 
13043    ins_encode %{
13044      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13045             as_Register($src2$$reg), ext::uxtb);
13046    %}
13047   ins_pipe(ialu_reg_reg);
13048 %}
13049 
13050 // This pattern is automatically generated from aarch64_ad.m4
13051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13052 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13053 %{
13054   match(Set dst (AddI src1 (AndI src2 mask)));
13055   ins_cost(INSN_COST);
13056   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13057 
13058    ins_encode %{
13059      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13060             as_Register($src2$$reg), ext::uxtb);
13061    %}
13062   ins_pipe(ialu_reg_reg);
13063 %}
13064 
13065 // This pattern is automatically generated from aarch64_ad.m4
13066 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13067 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13068 %{
13069   match(Set dst (AddI src1 (AndI src2 mask)));
13070   ins_cost(INSN_COST);
13071   format %{ "addw  $dst, $src1, $src2, uxth" %}
13072 
13073    ins_encode %{
13074      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13075             as_Register($src2$$reg), ext::uxth);
13076    %}
13077   ins_pipe(ialu_reg_reg);
13078 %}
13079 
13080 // This pattern is automatically generated from aarch64_ad.m4
13081 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13082 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13083 %{
13084   match(Set dst (AddL src1 (AndL src2 mask)));
13085   ins_cost(INSN_COST);
13086   format %{ "add  $dst, $src1, $src2, uxtb" %}
13087 
13088    ins_encode %{
13089      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13090             as_Register($src2$$reg), ext::uxtb);
13091    %}
13092   ins_pipe(ialu_reg_reg);
13093 %}
13094 
13095 // This pattern is automatically generated from aarch64_ad.m4
13096 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13097 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13098 %{
13099   match(Set dst (AddL src1 (AndL src2 mask)));
13100   ins_cost(INSN_COST);
13101   format %{ "add  $dst, $src1, $src2, uxth" %}
13102 
13103    ins_encode %{
13104      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13105             as_Register($src2$$reg), ext::uxth);
13106    %}
13107   ins_pipe(ialu_reg_reg);
13108 %}
13109 
13110 // This pattern is automatically generated from aarch64_ad.m4
13111 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13112 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13113 %{
13114   match(Set dst (AddL src1 (AndL src2 mask)));
13115   ins_cost(INSN_COST);
13116   format %{ "add  $dst, $src1, $src2, uxtw" %}
13117 
13118    ins_encode %{
13119      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13120             as_Register($src2$$reg), ext::uxtw);
13121    %}
13122   ins_pipe(ialu_reg_reg);
13123 %}
13124 
13125 // This pattern is automatically generated from aarch64_ad.m4
13126 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13127 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13128 %{
13129   match(Set dst (SubI src1 (AndI src2 mask)));
13130   ins_cost(INSN_COST);
13131   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13132 
13133    ins_encode %{
13134      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13135             as_Register($src2$$reg), ext::uxtb);
13136    %}
13137   ins_pipe(ialu_reg_reg);
13138 %}
13139 
13140 // This pattern is automatically generated from aarch64_ad.m4
13141 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13142 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13143 %{
13144   match(Set dst (SubI src1 (AndI src2 mask)));
13145   ins_cost(INSN_COST);
13146   format %{ "subw  $dst, $src1, $src2, uxth" %}
13147 
13148    ins_encode %{
13149      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13150             as_Register($src2$$reg), ext::uxth);
13151    %}
13152   ins_pipe(ialu_reg_reg);
13153 %}
13154 
13155 // This pattern is automatically generated from aarch64_ad.m4
13156 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13157 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13158 %{
13159   match(Set dst (SubL src1 (AndL src2 mask)));
13160   ins_cost(INSN_COST);
13161   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13162 
13163    ins_encode %{
13164      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13165             as_Register($src2$$reg), ext::uxtb);
13166    %}
13167   ins_pipe(ialu_reg_reg);
13168 %}
13169 
13170 // This pattern is automatically generated from aarch64_ad.m4
13171 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13172 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13173 %{
13174   match(Set dst (SubL src1 (AndL src2 mask)));
13175   ins_cost(INSN_COST);
13176   format %{ "sub  $dst, $src1, $src2, uxth" %}
13177 
13178    ins_encode %{
13179      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13180             as_Register($src2$$reg), ext::uxth);
13181    %}
13182   ins_pipe(ialu_reg_reg);
13183 %}
13184 
13185 // This pattern is automatically generated from aarch64_ad.m4
13186 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13187 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13188 %{
13189   match(Set dst (SubL src1 (AndL src2 mask)));
13190   ins_cost(INSN_COST);
13191   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13192 
13193    ins_encode %{
13194      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13195             as_Register($src2$$reg), ext::uxtw);
13196    %}
13197   ins_pipe(ialu_reg_reg);
13198 %}
13199 
13200 
13201 // This pattern is automatically generated from aarch64_ad.m4
13202 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13203 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13204 %{
13205   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13206   ins_cost(1.9 * INSN_COST);
13207   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13208 
13209    ins_encode %{
13210      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13211             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13212    %}
13213   ins_pipe(ialu_reg_reg_shift);
13214 %}
13215 
13216 // This pattern is automatically generated from aarch64_ad.m4
13217 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13218 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13219 %{
13220   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13221   ins_cost(1.9 * INSN_COST);
13222   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13223 
13224    ins_encode %{
13225      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13226             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13227    %}
13228   ins_pipe(ialu_reg_reg_shift);
13229 %}
13230 
13231 // This pattern is automatically generated from aarch64_ad.m4
13232 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13233 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13234 %{
13235   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13236   ins_cost(1.9 * INSN_COST);
13237   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13238 
13239    ins_encode %{
13240      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13241             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13242    %}
13243   ins_pipe(ialu_reg_reg_shift);
13244 %}
13245 
13246 // This pattern is automatically generated from aarch64_ad.m4
13247 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13248 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13249 %{
13250   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13251   ins_cost(1.9 * INSN_COST);
13252   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13253 
13254    ins_encode %{
13255      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13256             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13257    %}
13258   ins_pipe(ialu_reg_reg_shift);
13259 %}
13260 
13261 // This pattern is automatically generated from aarch64_ad.m4
13262 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13263 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13264 %{
13265   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13266   ins_cost(1.9 * INSN_COST);
13267   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13268 
13269    ins_encode %{
13270      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13271             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13272    %}
13273   ins_pipe(ialu_reg_reg_shift);
13274 %}
13275 
13276 // This pattern is automatically generated from aarch64_ad.m4
13277 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13278 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13279 %{
13280   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13281   ins_cost(1.9 * INSN_COST);
13282   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13283 
13284    ins_encode %{
13285      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13286             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13287    %}
13288   ins_pipe(ialu_reg_reg_shift);
13289 %}
13290 
13291 // This pattern is automatically generated from aarch64_ad.m4
13292 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13293 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13294 %{
13295   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13296   ins_cost(1.9 * INSN_COST);
13297   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13298 
13299    ins_encode %{
13300      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13301             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13302    %}
13303   ins_pipe(ialu_reg_reg_shift);
13304 %}
13305 
13306 // This pattern is automatically generated from aarch64_ad.m4
13307 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13308 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13309 %{
13310   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13311   ins_cost(1.9 * INSN_COST);
13312   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13313 
13314    ins_encode %{
13315      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13316             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13317    %}
13318   ins_pipe(ialu_reg_reg_shift);
13319 %}
13320 
13321 // This pattern is automatically generated from aarch64_ad.m4
13322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13323 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13324 %{
13325   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13326   ins_cost(1.9 * INSN_COST);
13327   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13328 
13329    ins_encode %{
13330      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13331             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13332    %}
13333   ins_pipe(ialu_reg_reg_shift);
13334 %}
13335 
13336 // This pattern is automatically generated from aarch64_ad.m4
13337 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13338 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13339 %{
13340   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13341   ins_cost(1.9 * INSN_COST);
13342   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13343 
13344    ins_encode %{
13345      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13346             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13347    %}
13348   ins_pipe(ialu_reg_reg_shift);
13349 %}
13350 
13351 // This pattern is automatically generated from aarch64_ad.m4
13352 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13353 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13354 %{
13355   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13356   ins_cost(1.9 * INSN_COST);
13357   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13358 
13359    ins_encode %{
13360      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13361             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13362    %}
13363   ins_pipe(ialu_reg_reg_shift);
13364 %}
13365 
13366 // This pattern is automatically generated from aarch64_ad.m4
13367 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13368 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13369 %{
13370   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13371   ins_cost(1.9 * INSN_COST);
13372   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13373 
13374    ins_encode %{
13375      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13376             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13377    %}
13378   ins_pipe(ialu_reg_reg_shift);
13379 %}
13380 
13381 // This pattern is automatically generated from aarch64_ad.m4
13382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13383 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13384 %{
13385   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13386   ins_cost(1.9 * INSN_COST);
13387   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13388 
13389    ins_encode %{
13390      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13391             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13392    %}
13393   ins_pipe(ialu_reg_reg_shift);
13394 %}
13395 
13396 // This pattern is automatically generated from aarch64_ad.m4
13397 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13398 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13399 %{
13400   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13401   ins_cost(1.9 * INSN_COST);
13402   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13403 
13404    ins_encode %{
13405      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13406             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13407    %}
13408   ins_pipe(ialu_reg_reg_shift);
13409 %}
13410 
13411 // This pattern is automatically generated from aarch64_ad.m4
13412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13413 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13414 %{
13415   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13416   ins_cost(1.9 * INSN_COST);
13417   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13418 
13419    ins_encode %{
13420      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13421             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13422    %}
13423   ins_pipe(ialu_reg_reg_shift);
13424 %}
13425 
13426 // This pattern is automatically generated from aarch64_ad.m4
13427 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13428 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13429 %{
13430   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13431   ins_cost(1.9 * INSN_COST);
13432   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13433 
13434    ins_encode %{
13435      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13436             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13437    %}
13438   ins_pipe(ialu_reg_reg_shift);
13439 %}
13440 
13441 // This pattern is automatically generated from aarch64_ad.m4
13442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13443 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13444 %{
13445   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13446   ins_cost(1.9 * INSN_COST);
13447   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13448 
13449    ins_encode %{
13450      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13451             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13452    %}
13453   ins_pipe(ialu_reg_reg_shift);
13454 %}
13455 
13456 // This pattern is automatically generated from aarch64_ad.m4
13457 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13458 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13459 %{
13460   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13461   ins_cost(1.9 * INSN_COST);
13462   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13463 
13464    ins_encode %{
13465      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13466             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13467    %}
13468   ins_pipe(ialu_reg_reg_shift);
13469 %}
13470 
13471 // This pattern is automatically generated from aarch64_ad.m4
13472 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13473 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13474 %{
13475   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13476   ins_cost(1.9 * INSN_COST);
13477   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13478 
13479    ins_encode %{
13480      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13481             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13482    %}
13483   ins_pipe(ialu_reg_reg_shift);
13484 %}
13485 
13486 // This pattern is automatically generated from aarch64_ad.m4
13487 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13488 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13489 %{
13490   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13491   ins_cost(1.9 * INSN_COST);
13492   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13493 
13494    ins_encode %{
13495      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13496             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13497    %}
13498   ins_pipe(ialu_reg_reg_shift);
13499 %}
13500 
13501 // This pattern is automatically generated from aarch64_ad.m4
13502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13503 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13504 %{
13505   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13506   ins_cost(1.9 * INSN_COST);
13507   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13508 
13509    ins_encode %{
13510      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13511             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13512    %}
13513   ins_pipe(ialu_reg_reg_shift);
13514 %}
13515 
13516 // This pattern is automatically generated from aarch64_ad.m4
13517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13518 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13519 %{
13520   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13521   ins_cost(1.9 * INSN_COST);
13522   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13523 
13524    ins_encode %{
13525      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13526             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13527    %}
13528   ins_pipe(ialu_reg_reg_shift);
13529 %}
13530 
13531 // This pattern is automatically generated from aarch64_ad.m4
13532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13533 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13534 %{
13535   effect(DEF dst, USE src1, USE src2, USE cr);
13536   ins_cost(INSN_COST * 2);
13537   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
13538 
13539   ins_encode %{
13540     __ cselw($dst$$Register,
13541              $src1$$Register,
13542              $src2$$Register,
13543              Assembler::LT);
13544   %}
13545   ins_pipe(icond_reg_reg);
13546 %}
13547 
13548 // This pattern is automatically generated from aarch64_ad.m4
13549 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13550 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13551 %{
13552   effect(DEF dst, USE src1, USE src2, USE cr);
13553   ins_cost(INSN_COST * 2);
13554   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
13555 
13556   ins_encode %{
13557     __ cselw($dst$$Register,
13558              $src1$$Register,
13559              $src2$$Register,
13560              Assembler::GT);
13561   %}
13562   ins_pipe(icond_reg_reg);
13563 %}
13564 
13565 // This pattern is automatically generated from aarch64_ad.m4
13566 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13567 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13568 %{
13569   effect(DEF dst, USE src1, USE cr);
13570   ins_cost(INSN_COST * 2);
13571   format %{ "cselw $dst, $src1, zr lt\t"  %}
13572 
13573   ins_encode %{
13574     __ cselw($dst$$Register,
13575              $src1$$Register,
13576              zr,
13577              Assembler::LT);
13578   %}
13579   ins_pipe(icond_reg);
13580 %}
13581 
13582 // This pattern is automatically generated from aarch64_ad.m4
13583 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13584 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13585 %{
13586   effect(DEF dst, USE src1, USE cr);
13587   ins_cost(INSN_COST * 2);
13588   format %{ "cselw $dst, $src1, zr gt\t"  %}
13589 
13590   ins_encode %{
13591     __ cselw($dst$$Register,
13592              $src1$$Register,
13593              zr,
13594              Assembler::GT);
13595   %}
13596   ins_pipe(icond_reg);
13597 %}
13598 
13599 // This pattern is automatically generated from aarch64_ad.m4
13600 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13601 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13602 %{
13603   effect(DEF dst, USE src1, USE cr);
13604   ins_cost(INSN_COST * 2);
13605   format %{ "csincw $dst, $src1, zr le\t"  %}
13606 
13607   ins_encode %{
13608     __ csincw($dst$$Register,
13609              $src1$$Register,
13610              zr,
13611              Assembler::LE);
13612   %}
13613   ins_pipe(icond_reg);
13614 %}
13615 
13616 // This pattern is automatically generated from aarch64_ad.m4
13617 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13618 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13619 %{
13620   effect(DEF dst, USE src1, USE cr);
13621   ins_cost(INSN_COST * 2);
13622   format %{ "csincw $dst, $src1, zr gt\t"  %}
13623 
13624   ins_encode %{
13625     __ csincw($dst$$Register,
13626              $src1$$Register,
13627              zr,
13628              Assembler::GT);
13629   %}
13630   ins_pipe(icond_reg);
13631 %}
13632 
13633 // This pattern is automatically generated from aarch64_ad.m4
13634 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13635 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13636 %{
13637   effect(DEF dst, USE src1, USE cr);
13638   ins_cost(INSN_COST * 2);
13639   format %{ "csinvw $dst, $src1, zr lt\t"  %}
13640 
13641   ins_encode %{
13642     __ csinvw($dst$$Register,
13643              $src1$$Register,
13644              zr,
13645              Assembler::LT);
13646   %}
13647   ins_pipe(icond_reg);
13648 %}
13649 
13650 // This pattern is automatically generated from aarch64_ad.m4
13651 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13652 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13653 %{
13654   effect(DEF dst, USE src1, USE cr);
13655   ins_cost(INSN_COST * 2);
13656   format %{ "csinvw $dst, $src1, zr ge\t"  %}
13657 
13658   ins_encode %{
13659     __ csinvw($dst$$Register,
13660              $src1$$Register,
13661              zr,
13662              Assembler::GE);
13663   %}
13664   ins_pipe(icond_reg);
13665 %}
13666 
13667 // This pattern is automatically generated from aarch64_ad.m4
13668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13669 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13670 %{
13671   match(Set dst (MinI src imm));
13672   ins_cost(INSN_COST * 3);
13673   expand %{
13674     rFlagsReg cr;
13675     compI_reg_imm0(cr, src);
13676     cmovI_reg_imm0_lt(dst, src, cr);
13677   %}
13678 %}
13679 
13680 // This pattern is automatically generated from aarch64_ad.m4
13681 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13682 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13683 %{
13684   match(Set dst (MinI imm src));
13685   ins_cost(INSN_COST * 3);
13686   expand %{
13687     rFlagsReg cr;
13688     compI_reg_imm0(cr, src);
13689     cmovI_reg_imm0_lt(dst, src, cr);
13690   %}
13691 %}
13692 
13693 // This pattern is automatically generated from aarch64_ad.m4
13694 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13695 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13696 %{
13697   match(Set dst (MinI src imm));
13698   ins_cost(INSN_COST * 3);
13699   expand %{
13700     rFlagsReg cr;
13701     compI_reg_imm0(cr, src);
13702     cmovI_reg_imm1_le(dst, src, cr);
13703   %}
13704 %}
13705 
13706 // This pattern is automatically generated from aarch64_ad.m4
13707 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13708 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13709 %{
13710   match(Set dst (MinI imm src));
13711   ins_cost(INSN_COST * 3);
13712   expand %{
13713     rFlagsReg cr;
13714     compI_reg_imm0(cr, src);
13715     cmovI_reg_imm1_le(dst, src, cr);
13716   %}
13717 %}
13718 
13719 // This pattern is automatically generated from aarch64_ad.m4
13720 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13721 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13722 %{
13723   match(Set dst (MinI src imm));
13724   ins_cost(INSN_COST * 3);
13725   expand %{
13726     rFlagsReg cr;
13727     compI_reg_imm0(cr, src);
13728     cmovI_reg_immM1_lt(dst, src, cr);
13729   %}
13730 %}
13731 
13732 // This pattern is automatically generated from aarch64_ad.m4
13733 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13734 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13735 %{
13736   match(Set dst (MinI imm src));
13737   ins_cost(INSN_COST * 3);
13738   expand %{
13739     rFlagsReg cr;
13740     compI_reg_imm0(cr, src);
13741     cmovI_reg_immM1_lt(dst, src, cr);
13742   %}
13743 %}
13744 
13745 // This pattern is automatically generated from aarch64_ad.m4
13746 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13747 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13748 %{
13749   match(Set dst (MaxI src imm));
13750   ins_cost(INSN_COST * 3);
13751   expand %{
13752     rFlagsReg cr;
13753     compI_reg_imm0(cr, src);
13754     cmovI_reg_imm0_gt(dst, src, cr);
13755   %}
13756 %}
13757 
13758 // This pattern is automatically generated from aarch64_ad.m4
13759 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13760 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13761 %{
13762   match(Set dst (MaxI imm src));
13763   ins_cost(INSN_COST * 3);
13764   expand %{
13765     rFlagsReg cr;
13766     compI_reg_imm0(cr, src);
13767     cmovI_reg_imm0_gt(dst, src, cr);
13768   %}
13769 %}
13770 
13771 // This pattern is automatically generated from aarch64_ad.m4
13772 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13773 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13774 %{
13775   match(Set dst (MaxI src imm));
13776   ins_cost(INSN_COST * 3);
13777   expand %{
13778     rFlagsReg cr;
13779     compI_reg_imm0(cr, src);
13780     cmovI_reg_imm1_gt(dst, src, cr);
13781   %}
13782 %}
13783 
13784 // This pattern is automatically generated from aarch64_ad.m4
13785 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13786 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13787 %{
13788   match(Set dst (MaxI imm src));
13789   ins_cost(INSN_COST * 3);
13790   expand %{
13791     rFlagsReg cr;
13792     compI_reg_imm0(cr, src);
13793     cmovI_reg_imm1_gt(dst, src, cr);
13794   %}
13795 %}
13796 
13797 // This pattern is automatically generated from aarch64_ad.m4
13798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13799 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13800 %{
13801   match(Set dst (MaxI src imm));
13802   ins_cost(INSN_COST * 3);
13803   expand %{
13804     rFlagsReg cr;
13805     compI_reg_imm0(cr, src);
13806     cmovI_reg_immM1_ge(dst, src, cr);
13807   %}
13808 %}
13809 
13810 // This pattern is automatically generated from aarch64_ad.m4
13811 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13812 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13813 %{
13814   match(Set dst (MaxI imm src));
13815   ins_cost(INSN_COST * 3);
13816   expand %{
13817     rFlagsReg cr;
13818     compI_reg_imm0(cr, src);
13819     cmovI_reg_immM1_ge(dst, src, cr);
13820   %}
13821 %}
13822 
13823 // This pattern is automatically generated from aarch64_ad.m4
13824 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13825 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13826 %{
13827   match(Set dst (ReverseI src));
13828   ins_cost(INSN_COST);
13829   format %{ "rbitw  $dst, $src" %}
13830   ins_encode %{
13831     __ rbitw($dst$$Register, $src$$Register);
13832   %}
13833   ins_pipe(ialu_reg);
13834 %}
13835 
13836 // This pattern is automatically generated from aarch64_ad.m4
13837 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13838 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13839 %{
13840   match(Set dst (ReverseL src));
13841   ins_cost(INSN_COST);
13842   format %{ "rbit  $dst, $src" %}
13843   ins_encode %{
13844     __ rbit($dst$$Register, $src$$Register);
13845   %}
13846   ins_pipe(ialu_reg);
13847 %}
13848 
13849 
13850 // END This section of the file is automatically generated. Do not edit --------------
13851 
13852 
13853 // ============================================================================
13854 // Floating Point Arithmetic Instructions
13855 
13856 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13857   match(Set dst (AddHF src1 src2));
13858   format %{ "faddh $dst, $src1, $src2" %}
13859   ins_encode %{
13860     __ faddh($dst$$FloatRegister,
13861              $src1$$FloatRegister,
13862              $src2$$FloatRegister);
13863   %}
13864   ins_pipe(fp_dop_reg_reg_s);
13865 %}
13866 
13867 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13868   match(Set dst (AddF src1 src2));
13869 
13870   ins_cost(INSN_COST * 5);
13871   format %{ "fadds   $dst, $src1, $src2" %}
13872 
13873   ins_encode %{
13874     __ fadds(as_FloatRegister($dst$$reg),
13875              as_FloatRegister($src1$$reg),
13876              as_FloatRegister($src2$$reg));
13877   %}
13878 
13879   ins_pipe(fp_dop_reg_reg_s);
13880 %}
13881 
13882 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13883   match(Set dst (AddD src1 src2));
13884 
13885   ins_cost(INSN_COST * 5);
13886   format %{ "faddd   $dst, $src1, $src2" %}
13887 
13888   ins_encode %{
13889     __ faddd(as_FloatRegister($dst$$reg),
13890              as_FloatRegister($src1$$reg),
13891              as_FloatRegister($src2$$reg));
13892   %}
13893 
13894   ins_pipe(fp_dop_reg_reg_d);
13895 %}
13896 
13897 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13898   match(Set dst (SubHF src1 src2));
13899   format %{ "fsubh $dst, $src1, $src2" %}
13900   ins_encode %{
13901     __ fsubh($dst$$FloatRegister,
13902              $src1$$FloatRegister,
13903              $src2$$FloatRegister);
13904   %}
13905   ins_pipe(fp_dop_reg_reg_s);
13906 %}
13907 
13908 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13909   match(Set dst (SubF src1 src2));
13910 
13911   ins_cost(INSN_COST * 5);
13912   format %{ "fsubs   $dst, $src1, $src2" %}
13913 
13914   ins_encode %{
13915     __ fsubs(as_FloatRegister($dst$$reg),
13916              as_FloatRegister($src1$$reg),
13917              as_FloatRegister($src2$$reg));
13918   %}
13919 
13920   ins_pipe(fp_dop_reg_reg_s);
13921 %}
13922 
13923 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13924   match(Set dst (SubD src1 src2));
13925 
13926   ins_cost(INSN_COST * 5);
13927   format %{ "fsubd   $dst, $src1, $src2" %}
13928 
13929   ins_encode %{
13930     __ fsubd(as_FloatRegister($dst$$reg),
13931              as_FloatRegister($src1$$reg),
13932              as_FloatRegister($src2$$reg));
13933   %}
13934 
13935   ins_pipe(fp_dop_reg_reg_d);
13936 %}
13937 
13938 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13939   match(Set dst (MulHF src1 src2));
13940   format %{ "fmulh $dst, $src1, $src2" %}
13941   ins_encode %{
13942     __ fmulh($dst$$FloatRegister,
13943              $src1$$FloatRegister,
13944              $src2$$FloatRegister);
13945   %}
13946   ins_pipe(fp_dop_reg_reg_s);
13947 %}
13948 
13949 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13950   match(Set dst (MulF src1 src2));
13951 
13952   ins_cost(INSN_COST * 6);
13953   format %{ "fmuls   $dst, $src1, $src2" %}
13954 
13955   ins_encode %{
13956     __ fmuls(as_FloatRegister($dst$$reg),
13957              as_FloatRegister($src1$$reg),
13958              as_FloatRegister($src2$$reg));
13959   %}
13960 
13961   ins_pipe(fp_dop_reg_reg_s);
13962 %}
13963 
13964 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13965   match(Set dst (MulD src1 src2));
13966 
13967   ins_cost(INSN_COST * 6);
13968   format %{ "fmuld   $dst, $src1, $src2" %}
13969 
13970   ins_encode %{
13971     __ fmuld(as_FloatRegister($dst$$reg),
13972              as_FloatRegister($src1$$reg),
13973              as_FloatRegister($src2$$reg));
13974   %}
13975 
13976   ins_pipe(fp_dop_reg_reg_d);
13977 %}
13978 
13979 // src1 * src2 + src3 (half-precision float)
13980 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13981   match(Set dst (FmaHF src3 (Binary src1 src2)));
13982   format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13983   ins_encode %{
13984     assert(UseFMA, "Needs FMA instructions support.");
13985     __ fmaddh($dst$$FloatRegister,
13986               $src1$$FloatRegister,
13987               $src2$$FloatRegister,
13988               $src3$$FloatRegister);
13989   %}
13990   ins_pipe(pipe_class_default);
13991 %}
13992 
13993 // src1 * src2 + src3
13994 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13995   match(Set dst (FmaF src3 (Binary src1 src2)));
13996 
13997   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
13998 
13999   ins_encode %{
14000     assert(UseFMA, "Needs FMA instructions support.");
14001     __ fmadds(as_FloatRegister($dst$$reg),
14002              as_FloatRegister($src1$$reg),
14003              as_FloatRegister($src2$$reg),
14004              as_FloatRegister($src3$$reg));
14005   %}
14006 
14007   ins_pipe(pipe_class_default);
14008 %}
14009 
14010 // src1 * src2 + src3
14011 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14012   match(Set dst (FmaD src3 (Binary src1 src2)));
14013 
14014   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
14015 
14016   ins_encode %{
14017     assert(UseFMA, "Needs FMA instructions support.");
14018     __ fmaddd(as_FloatRegister($dst$$reg),
14019              as_FloatRegister($src1$$reg),
14020              as_FloatRegister($src2$$reg),
14021              as_FloatRegister($src3$$reg));
14022   %}
14023 
14024   ins_pipe(pipe_class_default);
14025 %}
14026 
14027 // src1 * (-src2) + src3
14028 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
14029 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14030   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14031 
14032   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
14033 
14034   ins_encode %{
14035     assert(UseFMA, "Needs FMA instructions support.");
14036     __ fmsubs(as_FloatRegister($dst$$reg),
14037               as_FloatRegister($src1$$reg),
14038               as_FloatRegister($src2$$reg),
14039               as_FloatRegister($src3$$reg));
14040   %}
14041 
14042   ins_pipe(pipe_class_default);
14043 %}
14044 
14045 // src1 * (-src2) + src3
14046 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
14047 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14048   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14049 
14050   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
14051 
14052   ins_encode %{
14053     assert(UseFMA, "Needs FMA instructions support.");
14054     __ fmsubd(as_FloatRegister($dst$$reg),
14055               as_FloatRegister($src1$$reg),
14056               as_FloatRegister($src2$$reg),
14057               as_FloatRegister($src3$$reg));
14058   %}
14059 
14060   ins_pipe(pipe_class_default);
14061 %}
14062 
14063 // src1 * (-src2) - src3
14064 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14065 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14066   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14067 
14068   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14069 
14070   ins_encode %{
14071     assert(UseFMA, "Needs FMA instructions support.");
14072     __ fnmadds(as_FloatRegister($dst$$reg),
14073                as_FloatRegister($src1$$reg),
14074                as_FloatRegister($src2$$reg),
14075                as_FloatRegister($src3$$reg));
14076   %}
14077 
14078   ins_pipe(pipe_class_default);
14079 %}
14080 
14081 // src1 * (-src2) - src3
14082 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14083 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14084   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14085 
14086   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14087 
14088   ins_encode %{
14089     assert(UseFMA, "Needs FMA instructions support.");
14090     __ fnmaddd(as_FloatRegister($dst$$reg),
14091                as_FloatRegister($src1$$reg),
14092                as_FloatRegister($src2$$reg),
14093                as_FloatRegister($src3$$reg));
14094   %}
14095 
14096   ins_pipe(pipe_class_default);
14097 %}
14098 
14099 // src1 * src2 - src3
14100 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14101   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14102 
14103   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14104 
14105   ins_encode %{
14106     assert(UseFMA, "Needs FMA instructions support.");
14107     __ fnmsubs(as_FloatRegister($dst$$reg),
14108                as_FloatRegister($src1$$reg),
14109                as_FloatRegister($src2$$reg),
14110                as_FloatRegister($src3$$reg));
14111   %}
14112 
14113   ins_pipe(pipe_class_default);
14114 %}
14115 
14116 // src1 * src2 - src3
14117 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14118   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14119 
14120   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14121 
14122   ins_encode %{
14123     assert(UseFMA, "Needs FMA instructions support.");
14124     // n.b. insn name should be fnmsubd
14125     __ fnmsub(as_FloatRegister($dst$$reg),
14126               as_FloatRegister($src1$$reg),
14127               as_FloatRegister($src2$$reg),
14128               as_FloatRegister($src3$$reg));
14129   %}
14130 
14131   ins_pipe(pipe_class_default);
14132 %}
14133 
14134 // Math.max(HH)H (half-precision float)
14135 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14136   match(Set dst (MaxHF src1 src2));
14137   format %{ "fmaxh $dst, $src1, $src2" %}
14138   ins_encode %{
14139     __ fmaxh($dst$$FloatRegister,
14140              $src1$$FloatRegister,
14141              $src2$$FloatRegister);
14142   %}
14143   ins_pipe(fp_dop_reg_reg_s);
14144 %}
14145 
14146 // Math.min(HH)H (half-precision float)
14147 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14148   match(Set dst (MinHF src1 src2));
14149   format %{ "fminh $dst, $src1, $src2" %}
14150   ins_encode %{
14151     __ fminh($dst$$FloatRegister,
14152              $src1$$FloatRegister,
14153              $src2$$FloatRegister);
14154   %}
14155   ins_pipe(fp_dop_reg_reg_s);
14156 %}
14157 
14158 // Math.max(FF)F
14159 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14160   match(Set dst (MaxF src1 src2));
14161 
14162   format %{ "fmaxs   $dst, $src1, $src2" %}
14163   ins_encode %{
14164     __ fmaxs(as_FloatRegister($dst$$reg),
14165              as_FloatRegister($src1$$reg),
14166              as_FloatRegister($src2$$reg));
14167   %}
14168 
14169   ins_pipe(fp_dop_reg_reg_s);
14170 %}
14171 
14172 // Math.min(FF)F
14173 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14174   match(Set dst (MinF src1 src2));
14175 
14176   format %{ "fmins   $dst, $src1, $src2" %}
14177   ins_encode %{
14178     __ fmins(as_FloatRegister($dst$$reg),
14179              as_FloatRegister($src1$$reg),
14180              as_FloatRegister($src2$$reg));
14181   %}
14182 
14183   ins_pipe(fp_dop_reg_reg_s);
14184 %}
14185 
14186 // Math.max(DD)D
14187 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14188   match(Set dst (MaxD src1 src2));
14189 
14190   format %{ "fmaxd   $dst, $src1, $src2" %}
14191   ins_encode %{
14192     __ fmaxd(as_FloatRegister($dst$$reg),
14193              as_FloatRegister($src1$$reg),
14194              as_FloatRegister($src2$$reg));
14195   %}
14196 
14197   ins_pipe(fp_dop_reg_reg_d);
14198 %}
14199 
14200 // Math.min(DD)D
14201 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14202   match(Set dst (MinD src1 src2));
14203 
14204   format %{ "fmind   $dst, $src1, $src2" %}
14205   ins_encode %{
14206     __ fmind(as_FloatRegister($dst$$reg),
14207              as_FloatRegister($src1$$reg),
14208              as_FloatRegister($src2$$reg));
14209   %}
14210 
14211   ins_pipe(fp_dop_reg_reg_d);
14212 %}
14213 
14214 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14215   match(Set dst (DivHF src1  src2));
14216   format %{ "fdivh $dst, $src1, $src2" %}
14217   ins_encode %{
14218     __ fdivh($dst$$FloatRegister,
14219              $src1$$FloatRegister,
14220              $src2$$FloatRegister);
14221   %}
14222   ins_pipe(fp_div_s);
14223 %}
14224 
14225 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14226   match(Set dst (DivF src1  src2));
14227 
14228   ins_cost(INSN_COST * 18);
14229   format %{ "fdivs   $dst, $src1, $src2" %}
14230 
14231   ins_encode %{
14232     __ fdivs(as_FloatRegister($dst$$reg),
14233              as_FloatRegister($src1$$reg),
14234              as_FloatRegister($src2$$reg));
14235   %}
14236 
14237   ins_pipe(fp_div_s);
14238 %}
14239 
14240 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14241   match(Set dst (DivD src1  src2));
14242 
14243   ins_cost(INSN_COST * 32);
14244   format %{ "fdivd   $dst, $src1, $src2" %}
14245 
14246   ins_encode %{
14247     __ fdivd(as_FloatRegister($dst$$reg),
14248              as_FloatRegister($src1$$reg),
14249              as_FloatRegister($src2$$reg));
14250   %}
14251 
14252   ins_pipe(fp_div_d);
14253 %}
14254 
14255 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14256   match(Set dst (NegF src));
14257 
14258   ins_cost(INSN_COST * 3);
14259   format %{ "fneg   $dst, $src" %}
14260 
14261   ins_encode %{
14262     __ fnegs(as_FloatRegister($dst$$reg),
14263              as_FloatRegister($src$$reg));
14264   %}
14265 
14266   ins_pipe(fp_uop_s);
14267 %}
14268 
14269 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14270   match(Set dst (NegD src));
14271 
14272   ins_cost(INSN_COST * 3);
14273   format %{ "fnegd   $dst, $src" %}
14274 
14275   ins_encode %{
14276     __ fnegd(as_FloatRegister($dst$$reg),
14277              as_FloatRegister($src$$reg));
14278   %}
14279 
14280   ins_pipe(fp_uop_d);
14281 %}
14282 
14283 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14284 %{
14285   match(Set dst (AbsI src));
14286 
14287   effect(KILL cr);
14288   ins_cost(INSN_COST * 2);
14289   format %{ "cmpw  $src, zr\n\t"
14290             "cnegw $dst, $src, Assembler::LT\t# int abs"
14291   %}
14292 
14293   ins_encode %{
14294     __ cmpw(as_Register($src$$reg), zr);
14295     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14296   %}
14297   ins_pipe(pipe_class_default);
14298 %}
14299 
14300 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14301 %{
14302   match(Set dst (AbsL src));
14303 
14304   effect(KILL cr);
14305   ins_cost(INSN_COST * 2);
14306   format %{ "cmp  $src, zr\n\t"
14307             "cneg $dst, $src, Assembler::LT\t# long abs"
14308   %}
14309 
14310   ins_encode %{
14311     __ cmp(as_Register($src$$reg), zr);
14312     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14313   %}
14314   ins_pipe(pipe_class_default);
14315 %}
14316 
14317 instruct absF_reg(vRegF dst, vRegF src) %{
14318   match(Set dst (AbsF src));
14319 
14320   ins_cost(INSN_COST * 3);
14321   format %{ "fabss   $dst, $src" %}
14322   ins_encode %{
14323     __ fabss(as_FloatRegister($dst$$reg),
14324              as_FloatRegister($src$$reg));
14325   %}
14326 
14327   ins_pipe(fp_uop_s);
14328 %}
14329 
14330 instruct absD_reg(vRegD dst, vRegD src) %{
14331   match(Set dst (AbsD src));
14332 
14333   ins_cost(INSN_COST * 3);
14334   format %{ "fabsd   $dst, $src" %}
14335   ins_encode %{
14336     __ fabsd(as_FloatRegister($dst$$reg),
14337              as_FloatRegister($src$$reg));
14338   %}
14339 
14340   ins_pipe(fp_uop_d);
14341 %}
14342 
14343 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14344   match(Set dst (AbsF (SubF src1 src2)));
14345 
14346   ins_cost(INSN_COST * 3);
14347   format %{ "fabds   $dst, $src1, $src2" %}
14348   ins_encode %{
14349     __ fabds(as_FloatRegister($dst$$reg),
14350              as_FloatRegister($src1$$reg),
14351              as_FloatRegister($src2$$reg));
14352   %}
14353 
14354   ins_pipe(fp_uop_s);
14355 %}
14356 
14357 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14358   match(Set dst (AbsD (SubD src1 src2)));
14359 
14360   ins_cost(INSN_COST * 3);
14361   format %{ "fabdd   $dst, $src1, $src2" %}
14362   ins_encode %{
14363     __ fabdd(as_FloatRegister($dst$$reg),
14364              as_FloatRegister($src1$$reg),
14365              as_FloatRegister($src2$$reg));
14366   %}
14367 
14368   ins_pipe(fp_uop_d);
14369 %}
14370 
14371 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14372   match(Set dst (SqrtD src));
14373 
14374   ins_cost(INSN_COST * 50);
14375   format %{ "fsqrtd  $dst, $src" %}
14376   ins_encode %{
14377     __ fsqrtd(as_FloatRegister($dst$$reg),
14378              as_FloatRegister($src$$reg));
14379   %}
14380 
14381   ins_pipe(fp_div_s);
14382 %}
14383 
14384 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14385   match(Set dst (SqrtF src));
14386 
14387   ins_cost(INSN_COST * 50);
14388   format %{ "fsqrts  $dst, $src" %}
14389   ins_encode %{
14390     __ fsqrts(as_FloatRegister($dst$$reg),
14391              as_FloatRegister($src$$reg));
14392   %}
14393 
14394   ins_pipe(fp_div_d);
14395 %}
14396 
14397 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
14398   match(Set dst (SqrtHF src));
14399   format %{ "fsqrth $dst, $src" %}
14400   ins_encode %{
14401     __ fsqrth($dst$$FloatRegister,
14402               $src$$FloatRegister);
14403   %}
14404   ins_pipe(fp_div_s);
14405 %}
14406 
14407 // Math.rint, floor, ceil
14408 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14409   match(Set dst (RoundDoubleMode src rmode));
14410   format %{ "frint  $dst, $src, $rmode" %}
14411   ins_encode %{
14412     switch ($rmode$$constant) {
14413       case RoundDoubleModeNode::rmode_rint:
14414         __ frintnd(as_FloatRegister($dst$$reg),
14415                    as_FloatRegister($src$$reg));
14416         break;
14417       case RoundDoubleModeNode::rmode_floor:
14418         __ frintmd(as_FloatRegister($dst$$reg),
14419                    as_FloatRegister($src$$reg));
14420         break;
14421       case RoundDoubleModeNode::rmode_ceil:
14422         __ frintpd(as_FloatRegister($dst$$reg),
14423                    as_FloatRegister($src$$reg));
14424         break;
14425     }
14426   %}
14427   ins_pipe(fp_uop_d);
14428 %}
14429 
14430 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14431   match(Set dst (CopySignD src1 (Binary src2 zero)));
14432   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14433   format %{ "CopySignD  $dst $src1 $src2" %}
14434   ins_encode %{
14435     FloatRegister dst = as_FloatRegister($dst$$reg),
14436                   src1 = as_FloatRegister($src1$$reg),
14437                   src2 = as_FloatRegister($src2$$reg),
14438                   zero = as_FloatRegister($zero$$reg);
14439     __ fnegd(dst, zero);
14440     __ bsl(dst, __ T8B, src2, src1);
14441   %}
14442   ins_pipe(fp_uop_d);
14443 %}
14444 
14445 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14446   match(Set dst (CopySignF src1 src2));
14447   effect(TEMP_DEF dst, USE src1, USE src2);
14448   format %{ "CopySignF  $dst $src1 $src2" %}
14449   ins_encode %{
14450     FloatRegister dst = as_FloatRegister($dst$$reg),
14451                   src1 = as_FloatRegister($src1$$reg),
14452                   src2 = as_FloatRegister($src2$$reg);
14453     __ movi(dst, __ T2S, 0x80, 24);
14454     __ bsl(dst, __ T8B, src2, src1);
14455   %}
14456   ins_pipe(fp_uop_d);
14457 %}
14458 
14459 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14460   match(Set dst (SignumD src (Binary zero one)));
14461   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14462   format %{ "signumD  $dst, $src" %}
14463   ins_encode %{
14464     FloatRegister src = as_FloatRegister($src$$reg),
14465                   dst = as_FloatRegister($dst$$reg),
14466                   zero = as_FloatRegister($zero$$reg),
14467                   one = as_FloatRegister($one$$reg);
14468     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14469     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14470     // Bit selection instruction gets bit from "one" for each enabled bit in
14471     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14472     // NaN the whole "src" will be copied because "dst" is zero. For all other
14473     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14474     // from "src", and all other bits are copied from 1.0.
14475     __ bsl(dst, __ T8B, one, src);
14476   %}
14477   ins_pipe(fp_uop_d);
14478 %}
14479 
14480 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14481   match(Set dst (SignumF src (Binary zero one)));
14482   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14483   format %{ "signumF  $dst, $src" %}
14484   ins_encode %{
14485     FloatRegister src = as_FloatRegister($src$$reg),
14486                   dst = as_FloatRegister($dst$$reg),
14487                   zero = as_FloatRegister($zero$$reg),
14488                   one = as_FloatRegister($one$$reg);
14489     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14490     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14491     // Bit selection instruction gets bit from "one" for each enabled bit in
14492     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14493     // NaN the whole "src" will be copied because "dst" is zero. For all other
14494     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14495     // from "src", and all other bits are copied from 1.0.
14496     __ bsl(dst, __ T8B, one, src);
14497   %}
14498   ins_pipe(fp_uop_d);
14499 %}
14500 
14501 instruct onspinwait() %{
14502   match(OnSpinWait);
14503   ins_cost(INSN_COST);
14504 
14505   format %{ "onspinwait" %}
14506 
14507   ins_encode %{
14508     __ spin_wait();
14509   %}
14510   ins_pipe(pipe_class_empty);
14511 %}
14512 
14513 // ============================================================================
14514 // Logical Instructions
14515 
14516 // Integer Logical Instructions
14517 
14518 // And Instructions
14519 
14520 
14521 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14522   match(Set dst (AndI src1 src2));
14523 
14524   format %{ "andw  $dst, $src1, $src2\t# int" %}
14525 
14526   ins_cost(INSN_COST);
14527   ins_encode %{
14528     __ andw(as_Register($dst$$reg),
14529             as_Register($src1$$reg),
14530             as_Register($src2$$reg));
14531   %}
14532 
14533   ins_pipe(ialu_reg_reg);
14534 %}
14535 
14536 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14537   match(Set dst (AndI src1 src2));
14538 
14539   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14540 
14541   ins_cost(INSN_COST);
14542   ins_encode %{
14543     __ andw(as_Register($dst$$reg),
14544             as_Register($src1$$reg),
14545             (uint64_t)($src2$$constant));
14546   %}
14547 
14548   ins_pipe(ialu_reg_imm);
14549 %}
14550 
14551 // Or Instructions
14552 
14553 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14554   match(Set dst (OrI src1 src2));
14555 
14556   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14557 
14558   ins_cost(INSN_COST);
14559   ins_encode %{
14560     __ orrw(as_Register($dst$$reg),
14561             as_Register($src1$$reg),
14562             as_Register($src2$$reg));
14563   %}
14564 
14565   ins_pipe(ialu_reg_reg);
14566 %}
14567 
14568 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14569   match(Set dst (OrI src1 src2));
14570 
14571   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14572 
14573   ins_cost(INSN_COST);
14574   ins_encode %{
14575     __ orrw(as_Register($dst$$reg),
14576             as_Register($src1$$reg),
14577             (uint64_t)($src2$$constant));
14578   %}
14579 
14580   ins_pipe(ialu_reg_imm);
14581 %}
14582 
14583 // Xor Instructions
14584 
14585 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14586   match(Set dst (XorI src1 src2));
14587 
14588   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14589 
14590   ins_cost(INSN_COST);
14591   ins_encode %{
14592     __ eorw(as_Register($dst$$reg),
14593             as_Register($src1$$reg),
14594             as_Register($src2$$reg));
14595   %}
14596 
14597   ins_pipe(ialu_reg_reg);
14598 %}
14599 
14600 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14601   match(Set dst (XorI src1 src2));
14602 
14603   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14604 
14605   ins_cost(INSN_COST);
14606   ins_encode %{
14607     __ eorw(as_Register($dst$$reg),
14608             as_Register($src1$$reg),
14609             (uint64_t)($src2$$constant));
14610   %}
14611 
14612   ins_pipe(ialu_reg_imm);
14613 %}
14614 
14615 // Long Logical Instructions
14616 // TODO
14617 
14618 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14619   match(Set dst (AndL src1 src2));
14620 
14621   format %{ "and  $dst, $src1, $src2\t# int" %}
14622 
14623   ins_cost(INSN_COST);
14624   ins_encode %{
14625     __ andr(as_Register($dst$$reg),
14626             as_Register($src1$$reg),
14627             as_Register($src2$$reg));
14628   %}
14629 
14630   ins_pipe(ialu_reg_reg);
14631 %}
14632 
14633 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14634   match(Set dst (AndL src1 src2));
14635 
14636   format %{ "and  $dst, $src1, $src2\t# int" %}
14637 
14638   ins_cost(INSN_COST);
14639   ins_encode %{
14640     __ andr(as_Register($dst$$reg),
14641             as_Register($src1$$reg),
14642             (uint64_t)($src2$$constant));
14643   %}
14644 
14645   ins_pipe(ialu_reg_imm);
14646 %}
14647 
14648 // Or Instructions
14649 
14650 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14651   match(Set dst (OrL src1 src2));
14652 
14653   format %{ "orr  $dst, $src1, $src2\t# int" %}
14654 
14655   ins_cost(INSN_COST);
14656   ins_encode %{
14657     __ orr(as_Register($dst$$reg),
14658            as_Register($src1$$reg),
14659            as_Register($src2$$reg));
14660   %}
14661 
14662   ins_pipe(ialu_reg_reg);
14663 %}
14664 
14665 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14666   match(Set dst (OrL src1 src2));
14667 
14668   format %{ "orr  $dst, $src1, $src2\t# int" %}
14669 
14670   ins_cost(INSN_COST);
14671   ins_encode %{
14672     __ orr(as_Register($dst$$reg),
14673            as_Register($src1$$reg),
14674            (uint64_t)($src2$$constant));
14675   %}
14676 
14677   ins_pipe(ialu_reg_imm);
14678 %}
14679 
14680 // Xor Instructions
14681 
14682 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14683   match(Set dst (XorL src1 src2));
14684 
14685   format %{ "eor  $dst, $src1, $src2\t# int" %}
14686 
14687   ins_cost(INSN_COST);
14688   ins_encode %{
14689     __ eor(as_Register($dst$$reg),
14690            as_Register($src1$$reg),
14691            as_Register($src2$$reg));
14692   %}
14693 
14694   ins_pipe(ialu_reg_reg);
14695 %}
14696 
14697 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14698   match(Set dst (XorL src1 src2));
14699 
14700   ins_cost(INSN_COST);
14701   format %{ "eor  $dst, $src1, $src2\t# int" %}
14702 
14703   ins_encode %{
14704     __ eor(as_Register($dst$$reg),
14705            as_Register($src1$$reg),
14706            (uint64_t)($src2$$constant));
14707   %}
14708 
14709   ins_pipe(ialu_reg_imm);
14710 %}
14711 
14712 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14713 %{
14714   match(Set dst (ConvI2L src));
14715 
14716   ins_cost(INSN_COST);
14717   format %{ "sxtw  $dst, $src\t# i2l" %}
14718   ins_encode %{
14719     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14720   %}
14721   ins_pipe(ialu_reg_shift);
14722 %}
14723 
14724 // this pattern occurs in bigmath arithmetic
14725 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14726 %{
14727   match(Set dst (AndL (ConvI2L src) mask));
14728 
14729   ins_cost(INSN_COST);
14730   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14731   ins_encode %{
14732     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14733   %}
14734 
14735   ins_pipe(ialu_reg_shift);
14736 %}
14737 
14738 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14739   match(Set dst (ConvL2I src));
14740 
14741   ins_cost(INSN_COST);
14742   format %{ "movw  $dst, $src \t// l2i" %}
14743 
14744   ins_encode %{
14745     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14746   %}
14747 
14748   ins_pipe(ialu_reg);
14749 %}
14750 
14751 instruct convD2F_reg(vRegF dst, vRegD src) %{
14752   match(Set dst (ConvD2F src));
14753 
14754   ins_cost(INSN_COST * 5);
14755   format %{ "fcvtd  $dst, $src \t// d2f" %}
14756 
14757   ins_encode %{
14758     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14759   %}
14760 
14761   ins_pipe(fp_d2f);
14762 %}
14763 
14764 instruct convF2D_reg(vRegD dst, vRegF src) %{
14765   match(Set dst (ConvF2D src));
14766 
14767   ins_cost(INSN_COST * 5);
14768   format %{ "fcvts  $dst, $src \t// f2d" %}
14769 
14770   ins_encode %{
14771     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14772   %}
14773 
14774   ins_pipe(fp_f2d);
14775 %}
14776 
14777 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14778   match(Set dst (ConvF2I src));
14779 
14780   ins_cost(INSN_COST * 5);
14781   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14782 
14783   ins_encode %{
14784     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14785   %}
14786 
14787   ins_pipe(fp_f2i);
14788 %}
14789 
14790 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14791   match(Set dst (ConvF2L src));
14792 
14793   ins_cost(INSN_COST * 5);
14794   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14795 
14796   ins_encode %{
14797     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14798   %}
14799 
14800   ins_pipe(fp_f2l);
14801 %}
14802 
14803 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14804   match(Set dst (ConvF2HF src));
14805   format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14806             "smov $dst, $tmp\t# move result from $tmp to $dst"
14807   %}
14808   effect(TEMP tmp);
14809   ins_encode %{
14810       __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14811   %}
14812   ins_pipe(pipe_slow);
14813 %}
14814 
14815 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14816   match(Set dst (ConvHF2F src));
14817   format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14818             "fcvt $dst, $tmp\t# convert half to single precision"
14819   %}
14820   effect(TEMP tmp);
14821   ins_encode %{
14822       __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14823   %}
14824   ins_pipe(pipe_slow);
14825 %}
14826 
14827 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14828   match(Set dst (ConvI2F src));
14829 
14830   ins_cost(INSN_COST * 5);
14831   format %{ "scvtfws  $dst, $src \t// i2f" %}
14832 
14833   ins_encode %{
14834     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14835   %}
14836 
14837   ins_pipe(fp_i2f);
14838 %}
14839 
14840 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14841   match(Set dst (ConvL2F src));
14842 
14843   ins_cost(INSN_COST * 5);
14844   format %{ "scvtfs  $dst, $src \t// l2f" %}
14845 
14846   ins_encode %{
14847     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14848   %}
14849 
14850   ins_pipe(fp_l2f);
14851 %}
14852 
14853 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14854   match(Set dst (ConvD2I src));
14855 
14856   ins_cost(INSN_COST * 5);
14857   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14858 
14859   ins_encode %{
14860     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14861   %}
14862 
14863   ins_pipe(fp_d2i);
14864 %}
14865 
14866 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14867   match(Set dst (ConvD2L src));
14868 
14869   ins_cost(INSN_COST * 5);
14870   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14871 
14872   ins_encode %{
14873     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14874   %}
14875 
14876   ins_pipe(fp_d2l);
14877 %}
14878 
14879 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14880   match(Set dst (ConvI2D src));
14881 
14882   ins_cost(INSN_COST * 5);
14883   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14884 
14885   ins_encode %{
14886     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14887   %}
14888 
14889   ins_pipe(fp_i2d);
14890 %}
14891 
14892 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14893   match(Set dst (ConvL2D src));
14894 
14895   ins_cost(INSN_COST * 5);
14896   format %{ "scvtfd  $dst, $src \t// l2d" %}
14897 
14898   ins_encode %{
14899     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14900   %}
14901 
14902   ins_pipe(fp_l2d);
14903 %}
14904 
14905 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14906 %{
14907   match(Set dst (RoundD src));
14908   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14909   format %{ "java_round_double $dst,$src"%}
14910   ins_encode %{
14911     __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14912                          as_FloatRegister($ftmp$$reg));
14913   %}
14914   ins_pipe(pipe_slow);
14915 %}
14916 
14917 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14918 %{
14919   match(Set dst (RoundF src));
14920   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14921   format %{ "java_round_float $dst,$src"%}
14922   ins_encode %{
14923     __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14924                         as_FloatRegister($ftmp$$reg));
14925   %}
14926   ins_pipe(pipe_slow);
14927 %}
14928 
14929 // stack <-> reg and reg <-> reg shuffles with no conversion
14930 
14931 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14932 
14933   match(Set dst (MoveF2I src));
14934 
14935   effect(DEF dst, USE src);
14936 
14937   ins_cost(4 * INSN_COST);
14938 
14939   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14940 
14941   ins_encode %{
14942     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14943   %}
14944 
14945   ins_pipe(iload_reg_reg);
14946 
14947 %}
14948 
14949 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14950 
14951   match(Set dst (MoveI2F src));
14952 
14953   effect(DEF dst, USE src);
14954 
14955   ins_cost(4 * INSN_COST);
14956 
14957   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14958 
14959   ins_encode %{
14960     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14961   %}
14962 
14963   ins_pipe(pipe_class_memory);
14964 
14965 %}
14966 
14967 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14968 
14969   match(Set dst (MoveD2L src));
14970 
14971   effect(DEF dst, USE src);
14972 
14973   ins_cost(4 * INSN_COST);
14974 
14975   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14976 
14977   ins_encode %{
14978     __ ldr($dst$$Register, Address(sp, $src$$disp));
14979   %}
14980 
14981   ins_pipe(iload_reg_reg);
14982 
14983 %}
14984 
14985 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14986 
14987   match(Set dst (MoveL2D src));
14988 
14989   effect(DEF dst, USE src);
14990 
14991   ins_cost(4 * INSN_COST);
14992 
14993   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14994 
14995   ins_encode %{
14996     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14997   %}
14998 
14999   ins_pipe(pipe_class_memory);
15000 
15001 %}
15002 
15003 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15004 
15005   match(Set dst (MoveF2I src));
15006 
15007   effect(DEF dst, USE src);
15008 
15009   ins_cost(INSN_COST);
15010 
15011   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15012 
15013   ins_encode %{
15014     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15015   %}
15016 
15017   ins_pipe(pipe_class_memory);
15018 
15019 %}
15020 
15021 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15022 
15023   match(Set dst (MoveI2F src));
15024 
15025   effect(DEF dst, USE src);
15026 
15027   ins_cost(INSN_COST);
15028 
15029   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15030 
15031   ins_encode %{
15032     __ strw($src$$Register, Address(sp, $dst$$disp));
15033   %}
15034 
15035   ins_pipe(istore_reg_reg);
15036 
15037 %}
15038 
15039 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15040 
15041   match(Set dst (MoveD2L src));
15042 
15043   effect(DEF dst, USE src);
15044 
15045   ins_cost(INSN_COST);
15046 
15047   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15048 
15049   ins_encode %{
15050     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15051   %}
15052 
15053   ins_pipe(pipe_class_memory);
15054 
15055 %}
15056 
15057 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15058 
15059   match(Set dst (MoveL2D src));
15060 
15061   effect(DEF dst, USE src);
15062 
15063   ins_cost(INSN_COST);
15064 
15065   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15066 
15067   ins_encode %{
15068     __ str($src$$Register, Address(sp, $dst$$disp));
15069   %}
15070 
15071   ins_pipe(istore_reg_reg);
15072 
15073 %}
15074 
15075 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15076 
15077   match(Set dst (MoveF2I src));
15078 
15079   effect(DEF dst, USE src);
15080 
15081   ins_cost(INSN_COST);
15082 
15083   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15084 
15085   ins_encode %{
15086     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15087   %}
15088 
15089   ins_pipe(fp_f2i);
15090 
15091 %}
15092 
15093 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15094 
15095   match(Set dst (MoveI2F src));
15096 
15097   effect(DEF dst, USE src);
15098 
15099   ins_cost(INSN_COST);
15100 
15101   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15102 
15103   ins_encode %{
15104     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15105   %}
15106 
15107   ins_pipe(fp_i2f);
15108 
15109 %}
15110 
15111 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15112 
15113   match(Set dst (MoveD2L src));
15114 
15115   effect(DEF dst, USE src);
15116 
15117   ins_cost(INSN_COST);
15118 
15119   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15120 
15121   ins_encode %{
15122     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15123   %}
15124 
15125   ins_pipe(fp_d2l);
15126 
15127 %}
15128 
15129 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15130 
15131   match(Set dst (MoveL2D src));
15132 
15133   effect(DEF dst, USE src);
15134 
15135   ins_cost(INSN_COST);
15136 
15137   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15138 
15139   ins_encode %{
15140     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15141   %}
15142 
15143   ins_pipe(fp_l2d);
15144 
15145 %}
15146 
15147 // ============================================================================
15148 // clearing of an array
15149 
15150 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15151 %{
15152   match(Set dummy (ClearArray cnt base));
15153   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15154 
15155   ins_cost(4 * INSN_COST);
15156   format %{ "ClearArray $cnt, $base" %}
15157 
15158   ins_encode %{
15159     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15160     if (tpc == nullptr) {
15161       ciEnv::current()->record_failure("CodeCache is full");
15162       return;
15163     }
15164   %}
15165 
15166   ins_pipe(pipe_class_memory);
15167 %}
15168 
15169 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15170 %{
15171   predicate((uint64_t)n->in(2)->get_long()
15172             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15173   match(Set dummy (ClearArray cnt base));
15174   effect(TEMP temp, USE_KILL base, KILL cr);
15175 
15176   ins_cost(4 * INSN_COST);
15177   format %{ "ClearArray $cnt, $base" %}
15178 
15179   ins_encode %{
15180     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15181     if (tpc == nullptr) {
15182       ciEnv::current()->record_failure("CodeCache is full");
15183       return;
15184     }
15185   %}
15186 
15187   ins_pipe(pipe_class_memory);
15188 %}
15189 
15190 // ============================================================================
15191 // Overflow Math Instructions
15192 
15193 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15194 %{
15195   match(Set cr (OverflowAddI op1 op2));
15196 
15197   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15198   ins_cost(INSN_COST);
15199   ins_encode %{
15200     __ cmnw($op1$$Register, $op2$$Register);
15201   %}
15202 
15203   ins_pipe(icmp_reg_reg);
15204 %}
15205 
15206 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15207 %{
15208   match(Set cr (OverflowAddI op1 op2));
15209 
15210   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15211   ins_cost(INSN_COST);
15212   ins_encode %{
15213     __ cmnw($op1$$Register, $op2$$constant);
15214   %}
15215 
15216   ins_pipe(icmp_reg_imm);
15217 %}
15218 
15219 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15220 %{
15221   match(Set cr (OverflowAddL op1 op2));
15222 
15223   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15224   ins_cost(INSN_COST);
15225   ins_encode %{
15226     __ cmn($op1$$Register, $op2$$Register);
15227   %}
15228 
15229   ins_pipe(icmp_reg_reg);
15230 %}
15231 
15232 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15233 %{
15234   match(Set cr (OverflowAddL op1 op2));
15235 
15236   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
15237   ins_cost(INSN_COST);
15238   ins_encode %{
15239     __ adds(zr, $op1$$Register, $op2$$constant);
15240   %}
15241 
15242   ins_pipe(icmp_reg_imm);
15243 %}
15244 
15245 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15246 %{
15247   match(Set cr (OverflowSubI op1 op2));
15248 
15249   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15250   ins_cost(INSN_COST);
15251   ins_encode %{
15252     __ cmpw($op1$$Register, $op2$$Register);
15253   %}
15254 
15255   ins_pipe(icmp_reg_reg);
15256 %}
15257 
15258 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15259 %{
15260   match(Set cr (OverflowSubI op1 op2));
15261 
15262   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15263   ins_cost(INSN_COST);
15264   ins_encode %{
15265     __ cmpw($op1$$Register, $op2$$constant);
15266   %}
15267 
15268   ins_pipe(icmp_reg_imm);
15269 %}
15270 
15271 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15272 %{
15273   match(Set cr (OverflowSubL op1 op2));
15274 
15275   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15276   ins_cost(INSN_COST);
15277   ins_encode %{
15278     __ cmp($op1$$Register, $op2$$Register);
15279   %}
15280 
15281   ins_pipe(icmp_reg_reg);
15282 %}
15283 
15284 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15285 %{
15286   match(Set cr (OverflowSubL op1 op2));
15287 
15288   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15289   ins_cost(INSN_COST);
15290   ins_encode %{
15291     __ subs(zr, $op1$$Register, $op2$$constant);
15292   %}
15293 
15294   ins_pipe(icmp_reg_imm);
15295 %}
15296 
15297 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15298 %{
15299   match(Set cr (OverflowSubI zero op1));
15300 
15301   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15302   ins_cost(INSN_COST);
15303   ins_encode %{
15304     __ cmpw(zr, $op1$$Register);
15305   %}
15306 
15307   ins_pipe(icmp_reg_imm);
15308 %}
15309 
15310 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15311 %{
15312   match(Set cr (OverflowSubL zero op1));
15313 
15314   format %{ "cmp   zr, $op1\t# overflow check long" %}
15315   ins_cost(INSN_COST);
15316   ins_encode %{
15317     __ cmp(zr, $op1$$Register);
15318   %}
15319 
15320   ins_pipe(icmp_reg_imm);
15321 %}
15322 
15323 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15324 %{
15325   match(Set cr (OverflowMulI op1 op2));
15326 
15327   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15328             "cmp   rscratch1, rscratch1, sxtw\n\t"
15329             "movw  rscratch1, #0x80000000\n\t"
15330             "cselw rscratch1, rscratch1, zr, NE\n\t"
15331             "cmpw  rscratch1, #1" %}
15332   ins_cost(5 * INSN_COST);
15333   ins_encode %{
15334     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15335     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15336     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15337     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15338     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15339   %}
15340 
15341   ins_pipe(pipe_slow);
15342 %}
15343 
15344 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15345 %{
15346   match(If cmp (OverflowMulI op1 op2));
15347   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15348             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15349   effect(USE labl, KILL cr);
15350 
15351   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15352             "cmp   rscratch1, rscratch1, sxtw\n\t"
15353             "b$cmp   $labl" %}
15354   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15355   ins_encode %{
15356     Label* L = $labl$$label;
15357     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15358     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15359     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15360     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15361   %}
15362 
15363   ins_pipe(pipe_serial);
15364 %}
15365 
15366 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15367 %{
15368   match(Set cr (OverflowMulL op1 op2));
15369 
15370   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15371             "smulh rscratch2, $op1, $op2\n\t"
15372             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15373             "movw  rscratch1, #0x80000000\n\t"
15374             "cselw rscratch1, rscratch1, zr, NE\n\t"
15375             "cmpw  rscratch1, #1" %}
15376   ins_cost(6 * INSN_COST);
15377   ins_encode %{
15378     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15379     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15380     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15381     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15382     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15383     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15384   %}
15385 
15386   ins_pipe(pipe_slow);
15387 %}
15388 
15389 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15390 %{
15391   match(If cmp (OverflowMulL op1 op2));
15392   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15393             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15394   effect(USE labl, KILL cr);
15395 
15396   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15397             "smulh rscratch2, $op1, $op2\n\t"
15398             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15399             "b$cmp $labl" %}
15400   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15401   ins_encode %{
15402     Label* L = $labl$$label;
15403     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15404     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15405     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15406     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15407     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15408   %}
15409 
15410   ins_pipe(pipe_serial);
15411 %}
15412 
15413 // ============================================================================
15414 // Compare Instructions
15415 
15416 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15417 %{
15418   match(Set cr (CmpI op1 op2));
15419 
15420   effect(DEF cr, USE op1, USE op2);
15421 
15422   ins_cost(INSN_COST);
15423   format %{ "cmpw  $op1, $op2" %}
15424 
15425   ins_encode(aarch64_enc_cmpw(op1, op2));
15426 
15427   ins_pipe(icmp_reg_reg);
15428 %}
15429 
15430 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15431 %{
15432   match(Set cr (CmpI op1 zero));
15433 
15434   effect(DEF cr, USE op1);
15435 
15436   ins_cost(INSN_COST);
15437   format %{ "cmpw $op1, 0" %}
15438 
15439   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15440 
15441   ins_pipe(icmp_reg_imm);
15442 %}
15443 
15444 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15445 %{
15446   match(Set cr (CmpI op1 op2));
15447 
15448   effect(DEF cr, USE op1);
15449 
15450   ins_cost(INSN_COST);
15451   format %{ "cmpw  $op1, $op2" %}
15452 
15453   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15454 
15455   ins_pipe(icmp_reg_imm);
15456 %}
15457 
15458 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15459 %{
15460   match(Set cr (CmpI op1 op2));
15461 
15462   effect(DEF cr, USE op1);
15463 
15464   ins_cost(INSN_COST * 2);
15465   format %{ "cmpw  $op1, $op2" %}
15466 
15467   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15468 
15469   ins_pipe(icmp_reg_imm);
15470 %}
15471 
15472 // Unsigned compare Instructions; really, same as signed compare
15473 // except it should only be used to feed an If or a CMovI which takes a
15474 // cmpOpU.
15475 
15476 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15477 %{
15478   match(Set cr (CmpU op1 op2));
15479 
15480   effect(DEF cr, USE op1, USE op2);
15481 
15482   ins_cost(INSN_COST);
15483   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15484 
15485   ins_encode(aarch64_enc_cmpw(op1, op2));
15486 
15487   ins_pipe(icmp_reg_reg);
15488 %}
15489 
15490 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15491 %{
15492   match(Set cr (CmpU op1 zero));
15493 
15494   effect(DEF cr, USE op1);
15495 
15496   ins_cost(INSN_COST);
15497   format %{ "cmpw $op1, #0\t# unsigned" %}
15498 
15499   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15500 
15501   ins_pipe(icmp_reg_imm);
15502 %}
15503 
15504 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15505 %{
15506   match(Set cr (CmpU op1 op2));
15507 
15508   effect(DEF cr, USE op1);
15509 
15510   ins_cost(INSN_COST);
15511   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15512 
15513   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15514 
15515   ins_pipe(icmp_reg_imm);
15516 %}
15517 
15518 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15519 %{
15520   match(Set cr (CmpU op1 op2));
15521 
15522   effect(DEF cr, USE op1);
15523 
15524   ins_cost(INSN_COST * 2);
15525   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15526 
15527   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15528 
15529   ins_pipe(icmp_reg_imm);
15530 %}
15531 
15532 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15533 %{
15534   match(Set cr (CmpL op1 op2));
15535 
15536   effect(DEF cr, USE op1, USE op2);
15537 
15538   ins_cost(INSN_COST);
15539   format %{ "cmp  $op1, $op2" %}
15540 
15541   ins_encode(aarch64_enc_cmp(op1, op2));
15542 
15543   ins_pipe(icmp_reg_reg);
15544 %}
15545 
15546 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15547 %{
15548   match(Set cr (CmpL op1 zero));
15549 
15550   effect(DEF cr, USE op1);
15551 
15552   ins_cost(INSN_COST);
15553   format %{ "tst  $op1" %}
15554 
15555   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15556 
15557   ins_pipe(icmp_reg_imm);
15558 %}
15559 
15560 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15561 %{
15562   match(Set cr (CmpL op1 op2));
15563 
15564   effect(DEF cr, USE op1);
15565 
15566   ins_cost(INSN_COST);
15567   format %{ "cmp  $op1, $op2" %}
15568 
15569   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15570 
15571   ins_pipe(icmp_reg_imm);
15572 %}
15573 
15574 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15575 %{
15576   match(Set cr (CmpL op1 op2));
15577 
15578   effect(DEF cr, USE op1);
15579 
15580   ins_cost(INSN_COST * 2);
15581   format %{ "cmp  $op1, $op2" %}
15582 
15583   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15584 
15585   ins_pipe(icmp_reg_imm);
15586 %}
15587 
15588 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15589 %{
15590   match(Set cr (CmpUL op1 op2));
15591 
15592   effect(DEF cr, USE op1, USE op2);
15593 
15594   ins_cost(INSN_COST);
15595   format %{ "cmp  $op1, $op2" %}
15596 
15597   ins_encode(aarch64_enc_cmp(op1, op2));
15598 
15599   ins_pipe(icmp_reg_reg);
15600 %}
15601 
15602 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15603 %{
15604   match(Set cr (CmpUL op1 zero));
15605 
15606   effect(DEF cr, USE op1);
15607 
15608   ins_cost(INSN_COST);
15609   format %{ "tst  $op1" %}
15610 
15611   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15612 
15613   ins_pipe(icmp_reg_imm);
15614 %}
15615 
15616 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15617 %{
15618   match(Set cr (CmpUL op1 op2));
15619 
15620   effect(DEF cr, USE op1);
15621 
15622   ins_cost(INSN_COST);
15623   format %{ "cmp  $op1, $op2" %}
15624 
15625   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15626 
15627   ins_pipe(icmp_reg_imm);
15628 %}
15629 
15630 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15631 %{
15632   match(Set cr (CmpUL op1 op2));
15633 
15634   effect(DEF cr, USE op1);
15635 
15636   ins_cost(INSN_COST * 2);
15637   format %{ "cmp  $op1, $op2" %}
15638 
15639   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15640 
15641   ins_pipe(icmp_reg_imm);
15642 %}
15643 
15644 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15645 %{
15646   match(Set cr (CmpP op1 op2));
15647 
15648   effect(DEF cr, USE op1, USE op2);
15649 
15650   ins_cost(INSN_COST);
15651   format %{ "cmp  $op1, $op2\t // ptr" %}
15652 
15653   ins_encode(aarch64_enc_cmpp(op1, op2));
15654 
15655   ins_pipe(icmp_reg_reg);
15656 %}
15657 
15658 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15659 %{
15660   match(Set cr (CmpN op1 op2));
15661 
15662   effect(DEF cr, USE op1, USE op2);
15663 
15664   ins_cost(INSN_COST);
15665   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15666 
15667   ins_encode(aarch64_enc_cmpn(op1, op2));
15668 
15669   ins_pipe(icmp_reg_reg);
15670 %}
15671 
15672 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15673 %{
15674   match(Set cr (CmpP op1 zero));
15675 
15676   effect(DEF cr, USE op1, USE zero);
15677 
15678   ins_cost(INSN_COST);
15679   format %{ "cmp  $op1, 0\t // ptr" %}
15680 
15681   ins_encode(aarch64_enc_testp(op1));
15682 
15683   ins_pipe(icmp_reg_imm);
15684 %}
15685 
15686 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15687 %{
15688   match(Set cr (CmpN op1 zero));
15689 
15690   effect(DEF cr, USE op1, USE zero);
15691 
15692   ins_cost(INSN_COST);
15693   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15694 
15695   ins_encode(aarch64_enc_testn(op1));
15696 
15697   ins_pipe(icmp_reg_imm);
15698 %}
15699 
15700 // FP comparisons
15701 //
15702 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15703 // using normal cmpOp. See declaration of rFlagsReg for details.
15704 
15705 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15706 %{
15707   match(Set cr (CmpF src1 src2));
15708 
15709   ins_cost(3 * INSN_COST);
15710   format %{ "fcmps $src1, $src2" %}
15711 
15712   ins_encode %{
15713     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15714   %}
15715 
15716   ins_pipe(pipe_class_compare);
15717 %}
15718 
15719 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15720 %{
15721   match(Set cr (CmpF src1 src2));
15722 
15723   ins_cost(3 * INSN_COST);
15724   format %{ "fcmps $src1, 0.0" %}
15725 
15726   ins_encode %{
15727     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15728   %}
15729 
15730   ins_pipe(pipe_class_compare);
15731 %}
15732 // FROM HERE
15733 
15734 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15735 %{
15736   match(Set cr (CmpD src1 src2));
15737 
15738   ins_cost(3 * INSN_COST);
15739   format %{ "fcmpd $src1, $src2" %}
15740 
15741   ins_encode %{
15742     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15743   %}
15744 
15745   ins_pipe(pipe_class_compare);
15746 %}
15747 
15748 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15749 %{
15750   match(Set cr (CmpD src1 src2));
15751 
15752   ins_cost(3 * INSN_COST);
15753   format %{ "fcmpd $src1, 0.0" %}
15754 
15755   ins_encode %{
15756     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15757   %}
15758 
15759   ins_pipe(pipe_class_compare);
15760 %}
15761 
15762 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15763 %{
15764   match(Set dst (CmpF3 src1 src2));
15765   effect(KILL cr);
15766 
15767   ins_cost(5 * INSN_COST);
15768   format %{ "fcmps $src1, $src2\n\t"
15769             "csinvw($dst, zr, zr, eq\n\t"
15770             "csnegw($dst, $dst, $dst, lt)"
15771   %}
15772 
15773   ins_encode %{
15774     Label done;
15775     FloatRegister s1 = as_FloatRegister($src1$$reg);
15776     FloatRegister s2 = as_FloatRegister($src2$$reg);
15777     Register d = as_Register($dst$$reg);
15778     __ fcmps(s1, s2);
15779     // installs 0 if EQ else -1
15780     __ csinvw(d, zr, zr, Assembler::EQ);
15781     // keeps -1 if less or unordered else installs 1
15782     __ csnegw(d, d, d, Assembler::LT);
15783     __ bind(done);
15784   %}
15785 
15786   ins_pipe(pipe_class_default);
15787 
15788 %}
15789 
15790 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15791 %{
15792   match(Set dst (CmpD3 src1 src2));
15793   effect(KILL cr);
15794 
15795   ins_cost(5 * INSN_COST);
15796   format %{ "fcmpd $src1, $src2\n\t"
15797             "csinvw($dst, zr, zr, eq\n\t"
15798             "csnegw($dst, $dst, $dst, lt)"
15799   %}
15800 
15801   ins_encode %{
15802     Label done;
15803     FloatRegister s1 = as_FloatRegister($src1$$reg);
15804     FloatRegister s2 = as_FloatRegister($src2$$reg);
15805     Register d = as_Register($dst$$reg);
15806     __ fcmpd(s1, s2);
15807     // installs 0 if EQ else -1
15808     __ csinvw(d, zr, zr, Assembler::EQ);
15809     // keeps -1 if less or unordered else installs 1
15810     __ csnegw(d, d, d, Assembler::LT);
15811     __ bind(done);
15812   %}
15813   ins_pipe(pipe_class_default);
15814 
15815 %}
15816 
15817 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15818 %{
15819   match(Set dst (CmpF3 src1 zero));
15820   effect(KILL cr);
15821 
15822   ins_cost(5 * INSN_COST);
15823   format %{ "fcmps $src1, 0.0\n\t"
15824             "csinvw($dst, zr, zr, eq\n\t"
15825             "csnegw($dst, $dst, $dst, lt)"
15826   %}
15827 
15828   ins_encode %{
15829     Label done;
15830     FloatRegister s1 = as_FloatRegister($src1$$reg);
15831     Register d = as_Register($dst$$reg);
15832     __ fcmps(s1, 0.0);
15833     // installs 0 if EQ else -1
15834     __ csinvw(d, zr, zr, Assembler::EQ);
15835     // keeps -1 if less or unordered else installs 1
15836     __ csnegw(d, d, d, Assembler::LT);
15837     __ bind(done);
15838   %}
15839 
15840   ins_pipe(pipe_class_default);
15841 
15842 %}
15843 
15844 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15845 %{
15846   match(Set dst (CmpD3 src1 zero));
15847   effect(KILL cr);
15848 
15849   ins_cost(5 * INSN_COST);
15850   format %{ "fcmpd $src1, 0.0\n\t"
15851             "csinvw($dst, zr, zr, eq\n\t"
15852             "csnegw($dst, $dst, $dst, lt)"
15853   %}
15854 
15855   ins_encode %{
15856     Label done;
15857     FloatRegister s1 = as_FloatRegister($src1$$reg);
15858     Register d = as_Register($dst$$reg);
15859     __ fcmpd(s1, 0.0);
15860     // installs 0 if EQ else -1
15861     __ csinvw(d, zr, zr, Assembler::EQ);
15862     // keeps -1 if less or unordered else installs 1
15863     __ csnegw(d, d, d, Assembler::LT);
15864     __ bind(done);
15865   %}
15866   ins_pipe(pipe_class_default);
15867 
15868 %}
15869 
15870 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15871 %{
15872   match(Set dst (CmpLTMask p q));
15873   effect(KILL cr);
15874 
15875   ins_cost(3 * INSN_COST);
15876 
15877   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15878             "csetw $dst, lt\n\t"
15879             "subw $dst, zr, $dst"
15880   %}
15881 
15882   ins_encode %{
15883     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15884     __ csetw(as_Register($dst$$reg), Assembler::LT);
15885     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15886   %}
15887 
15888   ins_pipe(ialu_reg_reg);
15889 %}
15890 
15891 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15892 %{
15893   match(Set dst (CmpLTMask src zero));
15894   effect(KILL cr);
15895 
15896   ins_cost(INSN_COST);
15897 
15898   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15899 
15900   ins_encode %{
15901     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15902   %}
15903 
15904   ins_pipe(ialu_reg_shift);
15905 %}
15906 
15907 // ============================================================================
15908 // Max and Min
15909 
15910 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15911 
15912 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15913 %{
15914   effect(DEF cr, USE src);
15915   ins_cost(INSN_COST);
15916   format %{ "cmpw $src, 0" %}
15917 
15918   ins_encode %{
15919     __ cmpw($src$$Register, 0);
15920   %}
15921   ins_pipe(icmp_reg_imm);
15922 %}
15923 
15924 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15925 %{
15926   match(Set dst (MinI src1 src2));
15927   ins_cost(INSN_COST * 3);
15928 
15929   expand %{
15930     rFlagsReg cr;
15931     compI_reg_reg(cr, src1, src2);
15932     cmovI_reg_reg_lt(dst, src1, src2, cr);
15933   %}
15934 %}
15935 
15936 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15937 %{
15938   match(Set dst (MaxI src1 src2));
15939   ins_cost(INSN_COST * 3);
15940 
15941   expand %{
15942     rFlagsReg cr;
15943     compI_reg_reg(cr, src1, src2);
15944     cmovI_reg_reg_gt(dst, src1, src2, cr);
15945   %}
15946 %}
15947 
15948 
15949 // ============================================================================
15950 // Branch Instructions
15951 
15952 // Direct Branch.
15953 instruct branch(label lbl)
15954 %{
15955   match(Goto);
15956 
15957   effect(USE lbl);
15958 
15959   ins_cost(BRANCH_COST);
15960   format %{ "b  $lbl" %}
15961 
15962   ins_encode(aarch64_enc_b(lbl));
15963 
15964   ins_pipe(pipe_branch);
15965 %}
15966 
15967 // Conditional Near Branch
15968 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15969 %{
15970   // Same match rule as `branchConFar'.
15971   match(If cmp cr);
15972 
15973   effect(USE lbl);
15974 
15975   ins_cost(BRANCH_COST);
15976   // If set to 1 this indicates that the current instruction is a
15977   // short variant of a long branch. This avoids using this
15978   // instruction in first-pass matching. It will then only be used in
15979   // the `Shorten_branches' pass.
15980   // ins_short_branch(1);
15981   format %{ "b$cmp  $lbl" %}
15982 
15983   ins_encode(aarch64_enc_br_con(cmp, lbl));
15984 
15985   ins_pipe(pipe_branch_cond);
15986 %}
15987 
15988 // Conditional Near Branch Unsigned
15989 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15990 %{
15991   // Same match rule as `branchConFar'.
15992   match(If cmp cr);
15993 
15994   effect(USE lbl);
15995 
15996   ins_cost(BRANCH_COST);
15997   // If set to 1 this indicates that the current instruction is a
15998   // short variant of a long branch. This avoids using this
15999   // instruction in first-pass matching. It will then only be used in
16000   // the `Shorten_branches' pass.
16001   // ins_short_branch(1);
16002   format %{ "b$cmp  $lbl\t# unsigned" %}
16003 
16004   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16005 
16006   ins_pipe(pipe_branch_cond);
16007 %}
16008 
16009 // Make use of CBZ and CBNZ.  These instructions, as well as being
16010 // shorter than (cmp; branch), have the additional benefit of not
16011 // killing the flags.
16012 
16013 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16014   match(If cmp (CmpI op1 op2));
16015   effect(USE labl);
16016 
16017   ins_cost(BRANCH_COST);
16018   format %{ "cbw$cmp   $op1, $labl" %}
16019   ins_encode %{
16020     Label* L = $labl$$label;
16021     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16022     if (cond == Assembler::EQ)
16023       __ cbzw($op1$$Register, *L);
16024     else
16025       __ cbnzw($op1$$Register, *L);
16026   %}
16027   ins_pipe(pipe_cmp_branch);
16028 %}
16029 
16030 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16031   match(If cmp (CmpL op1 op2));
16032   effect(USE labl);
16033 
16034   ins_cost(BRANCH_COST);
16035   format %{ "cb$cmp   $op1, $labl" %}
16036   ins_encode %{
16037     Label* L = $labl$$label;
16038     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16039     if (cond == Assembler::EQ)
16040       __ cbz($op1$$Register, *L);
16041     else
16042       __ cbnz($op1$$Register, *L);
16043   %}
16044   ins_pipe(pipe_cmp_branch);
16045 %}
16046 
16047 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16048   match(If cmp (CmpP op1 op2));
16049   effect(USE labl);
16050 
16051   ins_cost(BRANCH_COST);
16052   format %{ "cb$cmp   $op1, $labl" %}
16053   ins_encode %{
16054     Label* L = $labl$$label;
16055     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16056     if (cond == Assembler::EQ)
16057       __ cbz($op1$$Register, *L);
16058     else
16059       __ cbnz($op1$$Register, *L);
16060   %}
16061   ins_pipe(pipe_cmp_branch);
16062 %}
16063 
16064 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16065   match(If cmp (CmpN op1 op2));
16066   effect(USE labl);
16067 
16068   ins_cost(BRANCH_COST);
16069   format %{ "cbw$cmp   $op1, $labl" %}
16070   ins_encode %{
16071     Label* L = $labl$$label;
16072     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16073     if (cond == Assembler::EQ)
16074       __ cbzw($op1$$Register, *L);
16075     else
16076       __ cbnzw($op1$$Register, *L);
16077   %}
16078   ins_pipe(pipe_cmp_branch);
16079 %}
16080 
16081 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16082   match(If cmp (CmpP (DecodeN oop) zero));
16083   effect(USE labl);
16084 
16085   ins_cost(BRANCH_COST);
16086   format %{ "cb$cmp   $oop, $labl" %}
16087   ins_encode %{
16088     Label* L = $labl$$label;
16089     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16090     if (cond == Assembler::EQ)
16091       __ cbzw($oop$$Register, *L);
16092     else
16093       __ cbnzw($oop$$Register, *L);
16094   %}
16095   ins_pipe(pipe_cmp_branch);
16096 %}
16097 
16098 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16099   match(If cmp (CmpU op1 op2));
16100   effect(USE labl);
16101 
16102   ins_cost(BRANCH_COST);
16103   format %{ "cbw$cmp   $op1, $labl" %}
16104   ins_encode %{
16105     Label* L = $labl$$label;
16106     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16107     if (cond == Assembler::EQ || cond == Assembler::LS) {
16108       __ cbzw($op1$$Register, *L);
16109     } else {
16110       assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16111       __ cbnzw($op1$$Register, *L);
16112     }
16113   %}
16114   ins_pipe(pipe_cmp_branch);
16115 %}
16116 
16117 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
16118   match(If cmp (CmpUL op1 op2));
16119   effect(USE labl);
16120 
16121   ins_cost(BRANCH_COST);
16122   format %{ "cb$cmp   $op1, $labl" %}
16123   ins_encode %{
16124     Label* L = $labl$$label;
16125     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16126     if (cond == Assembler::EQ || cond == Assembler::LS) {
16127       __ cbz($op1$$Register, *L);
16128     } else {
16129       assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16130       __ cbnz($op1$$Register, *L);
16131     }
16132   %}
16133   ins_pipe(pipe_cmp_branch);
16134 %}
16135 
16136 // Test bit and Branch
16137 
16138 // Patterns for short (< 32KiB) variants
16139 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16140   match(If cmp (CmpL op1 op2));
16141   effect(USE labl);
16142 
16143   ins_cost(BRANCH_COST);
16144   format %{ "cb$cmp   $op1, $labl # long" %}
16145   ins_encode %{
16146     Label* L = $labl$$label;
16147     Assembler::Condition cond =
16148       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16149     __ tbr(cond, $op1$$Register, 63, *L);
16150   %}
16151   ins_pipe(pipe_cmp_branch);
16152   ins_short_branch(1);
16153 %}
16154 
16155 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16156   match(If cmp (CmpI op1 op2));
16157   effect(USE labl);
16158 
16159   ins_cost(BRANCH_COST);
16160   format %{ "cb$cmp   $op1, $labl # int" %}
16161   ins_encode %{
16162     Label* L = $labl$$label;
16163     Assembler::Condition cond =
16164       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16165     __ tbr(cond, $op1$$Register, 31, *L);
16166   %}
16167   ins_pipe(pipe_cmp_branch);
16168   ins_short_branch(1);
16169 %}
16170 
16171 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16172   match(If cmp (CmpL (AndL op1 op2) op3));
16173   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16174   effect(USE labl);
16175 
16176   ins_cost(BRANCH_COST);
16177   format %{ "tb$cmp   $op1, $op2, $labl" %}
16178   ins_encode %{
16179     Label* L = $labl$$label;
16180     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16181     int bit = exact_log2_long($op2$$constant);
16182     __ tbr(cond, $op1$$Register, bit, *L);
16183   %}
16184   ins_pipe(pipe_cmp_branch);
16185   ins_short_branch(1);
16186 %}
16187 
16188 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16189   match(If cmp (CmpI (AndI op1 op2) op3));
16190   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16191   effect(USE labl);
16192 
16193   ins_cost(BRANCH_COST);
16194   format %{ "tb$cmp   $op1, $op2, $labl" %}
16195   ins_encode %{
16196     Label* L = $labl$$label;
16197     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16198     int bit = exact_log2((juint)$op2$$constant);
16199     __ tbr(cond, $op1$$Register, bit, *L);
16200   %}
16201   ins_pipe(pipe_cmp_branch);
16202   ins_short_branch(1);
16203 %}
16204 
16205 // And far variants
16206 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16207   match(If cmp (CmpL op1 op2));
16208   effect(USE labl);
16209 
16210   ins_cost(BRANCH_COST);
16211   format %{ "cb$cmp   $op1, $labl # long" %}
16212   ins_encode %{
16213     Label* L = $labl$$label;
16214     Assembler::Condition cond =
16215       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16216     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16217   %}
16218   ins_pipe(pipe_cmp_branch);
16219 %}
16220 
16221 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16222   match(If cmp (CmpI op1 op2));
16223   effect(USE labl);
16224 
16225   ins_cost(BRANCH_COST);
16226   format %{ "cb$cmp   $op1, $labl # int" %}
16227   ins_encode %{
16228     Label* L = $labl$$label;
16229     Assembler::Condition cond =
16230       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16231     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16232   %}
16233   ins_pipe(pipe_cmp_branch);
16234 %}
16235 
16236 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16237   match(If cmp (CmpL (AndL op1 op2) op3));
16238   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16239   effect(USE labl);
16240 
16241   ins_cost(BRANCH_COST);
16242   format %{ "tb$cmp   $op1, $op2, $labl" %}
16243   ins_encode %{
16244     Label* L = $labl$$label;
16245     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16246     int bit = exact_log2_long($op2$$constant);
16247     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16248   %}
16249   ins_pipe(pipe_cmp_branch);
16250 %}
16251 
16252 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16253   match(If cmp (CmpI (AndI op1 op2) op3));
16254   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16255   effect(USE labl);
16256 
16257   ins_cost(BRANCH_COST);
16258   format %{ "tb$cmp   $op1, $op2, $labl" %}
16259   ins_encode %{
16260     Label* L = $labl$$label;
16261     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16262     int bit = exact_log2((juint)$op2$$constant);
16263     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16264   %}
16265   ins_pipe(pipe_cmp_branch);
16266 %}
16267 
16268 // Test bits
16269 
16270 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16271   match(Set cr (CmpL (AndL op1 op2) op3));
16272   predicate(Assembler::operand_valid_for_logical_immediate
16273             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16274 
16275   ins_cost(INSN_COST);
16276   format %{ "tst $op1, $op2 # long" %}
16277   ins_encode %{
16278     __ tst($op1$$Register, $op2$$constant);
16279   %}
16280   ins_pipe(ialu_reg_reg);
16281 %}
16282 
16283 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16284   match(Set cr (CmpI (AndI op1 op2) op3));
16285   predicate(Assembler::operand_valid_for_logical_immediate
16286             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16287 
16288   ins_cost(INSN_COST);
16289   format %{ "tst $op1, $op2 # int" %}
16290   ins_encode %{
16291     __ tstw($op1$$Register, $op2$$constant);
16292   %}
16293   ins_pipe(ialu_reg_reg);
16294 %}
16295 
16296 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16297   match(Set cr (CmpL (AndL op1 op2) op3));
16298 
16299   ins_cost(INSN_COST);
16300   format %{ "tst $op1, $op2 # long" %}
16301   ins_encode %{
16302     __ tst($op1$$Register, $op2$$Register);
16303   %}
16304   ins_pipe(ialu_reg_reg);
16305 %}
16306 
16307 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16308   match(Set cr (CmpI (AndI op1 op2) op3));
16309 
16310   ins_cost(INSN_COST);
16311   format %{ "tstw $op1, $op2 # int" %}
16312   ins_encode %{
16313     __ tstw($op1$$Register, $op2$$Register);
16314   %}
16315   ins_pipe(ialu_reg_reg);
16316 %}
16317 
16318 
16319 // Conditional Far Branch
16320 // Conditional Far Branch Unsigned
16321 // TODO: fixme
16322 
16323 // counted loop end branch near
16324 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16325 %{
16326   match(CountedLoopEnd cmp cr);
16327 
16328   effect(USE lbl);
16329 
16330   ins_cost(BRANCH_COST);
16331   // short variant.
16332   // ins_short_branch(1);
16333   format %{ "b$cmp $lbl \t// counted loop end" %}
16334 
16335   ins_encode(aarch64_enc_br_con(cmp, lbl));
16336 
16337   ins_pipe(pipe_branch);
16338 %}
16339 
16340 // counted loop end branch far
16341 // TODO: fixme
16342 
16343 // ============================================================================
16344 // inlined locking and unlocking
16345 
16346 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16347 %{
16348   match(Set cr (FastLock object box));
16349   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16350 
16351   ins_cost(5 * INSN_COST);
16352   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16353 
16354   ins_encode %{
16355     __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16356   %}
16357 
16358   ins_pipe(pipe_serial);
16359 %}
16360 
16361 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16362 %{
16363   match(Set cr (FastUnlock object box));
16364   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16365 
16366   ins_cost(5 * INSN_COST);
16367   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16368 
16369   ins_encode %{
16370     __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16371   %}
16372 
16373   ins_pipe(pipe_serial);
16374 %}
16375 
16376 // ============================================================================
16377 // Safepoint Instructions
16378 
16379 // TODO
16380 // provide a near and far version of this code
16381 
16382 instruct safePoint(rFlagsReg cr, iRegP poll)
16383 %{
16384   match(SafePoint poll);
16385   effect(KILL cr);
16386 
16387   format %{
16388     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16389   %}
16390   ins_encode %{
16391     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16392   %}
16393   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16394 %}
16395 
16396 
16397 // ============================================================================
16398 // Procedure Call/Return Instructions
16399 
16400 // Call Java Static Instruction
16401 
16402 instruct CallStaticJavaDirect(method meth)
16403 %{
16404   match(CallStaticJava);
16405 
16406   effect(USE meth);
16407 
16408   ins_cost(CALL_COST);
16409 
16410   format %{ "call,static $meth \t// ==> " %}
16411 
16412   ins_encode(aarch64_enc_java_static_call(meth),
16413              aarch64_enc_call_epilog);
16414 
16415   ins_pipe(pipe_class_call);
16416 %}
16417 
16418 // TO HERE
16419 
16420 // Call Java Dynamic Instruction
16421 instruct CallDynamicJavaDirect(method meth)
16422 %{
16423   match(CallDynamicJava);
16424 
16425   effect(USE meth);
16426 
16427   ins_cost(CALL_COST);
16428 
16429   format %{ "CALL,dynamic $meth \t// ==> " %}
16430 
16431   ins_encode(aarch64_enc_java_dynamic_call(meth),
16432              aarch64_enc_call_epilog);
16433 
16434   ins_pipe(pipe_class_call);
16435 %}
16436 
16437 // Call Runtime Instruction
16438 
16439 instruct CallRuntimeDirect(method meth)
16440 %{
16441   match(CallRuntime);
16442 
16443   effect(USE meth);
16444 
16445   ins_cost(CALL_COST);
16446 
16447   format %{ "CALL, runtime $meth" %}
16448 
16449   ins_encode( aarch64_enc_java_to_runtime(meth) );
16450 
16451   ins_pipe(pipe_class_call);
16452 %}
16453 
16454 // Call Runtime Instruction
16455 
16456 instruct CallLeafDirect(method meth)
16457 %{
16458   match(CallLeaf);
16459 
16460   effect(USE meth);
16461 
16462   ins_cost(CALL_COST);
16463 
16464   format %{ "CALL, runtime leaf $meth" %}
16465 
16466   ins_encode( aarch64_enc_java_to_runtime(meth) );
16467 
16468   ins_pipe(pipe_class_call);
16469 %}
16470 
16471 // Call Runtime Instruction without safepoint and with vector arguments
16472 instruct CallLeafDirectVector(method meth)
16473 %{
16474   match(CallLeafVector);
16475 
16476   effect(USE meth);
16477 
16478   ins_cost(CALL_COST);
16479 
16480   format %{ "CALL, runtime leaf vector $meth" %}
16481 
16482   ins_encode(aarch64_enc_java_to_runtime(meth));
16483 
16484   ins_pipe(pipe_class_call);
16485 %}
16486 
16487 // Call Runtime Instruction
16488 
16489 instruct CallLeafNoFPDirect(method meth)
16490 %{
16491   match(CallLeafNoFP);
16492 
16493   effect(USE meth);
16494 
16495   ins_cost(CALL_COST);
16496 
16497   format %{ "CALL, runtime leaf nofp $meth" %}
16498 
16499   ins_encode( aarch64_enc_java_to_runtime(meth) );
16500 
16501   ins_pipe(pipe_class_call);
16502 %}
16503 
16504 // Tail Call; Jump from runtime stub to Java code.
16505 // Also known as an 'interprocedural jump'.
16506 // Target of jump will eventually return to caller.
16507 // TailJump below removes the return address.
16508 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16509 // emitted just above the TailCall which has reset rfp to the caller state.
16510 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16511 %{
16512   match(TailCall jump_target method_ptr);
16513 
16514   ins_cost(CALL_COST);
16515 
16516   format %{ "br $jump_target\t# $method_ptr holds method" %}
16517 
16518   ins_encode(aarch64_enc_tail_call(jump_target));
16519 
16520   ins_pipe(pipe_class_call);
16521 %}
16522 
16523 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16524 %{
16525   match(TailJump jump_target ex_oop);
16526 
16527   ins_cost(CALL_COST);
16528 
16529   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16530 
16531   ins_encode(aarch64_enc_tail_jmp(jump_target));
16532 
16533   ins_pipe(pipe_class_call);
16534 %}
16535 
16536 // Forward exception.
16537 instruct ForwardExceptionjmp()
16538 %{
16539   match(ForwardException);
16540   ins_cost(CALL_COST);
16541 
16542   format %{ "b forward_exception_stub" %}
16543   ins_encode %{
16544     __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16545   %}
16546   ins_pipe(pipe_class_call);
16547 %}
16548 
16549 // Create exception oop: created by stack-crawling runtime code.
16550 // Created exception is now available to this handler, and is setup
16551 // just prior to jumping to this handler. No code emitted.
16552 // TODO check
16553 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16554 instruct CreateException(iRegP_R0 ex_oop)
16555 %{
16556   match(Set ex_oop (CreateEx));
16557 
16558   format %{ " -- \t// exception oop; no code emitted" %}
16559 
16560   size(0);
16561 
16562   ins_encode( /*empty*/ );
16563 
16564   ins_pipe(pipe_class_empty);
16565 %}
16566 
16567 // Rethrow exception: The exception oop will come in the first
16568 // argument position. Then JUMP (not call) to the rethrow stub code.
16569 instruct RethrowException() %{
16570   match(Rethrow);
16571   ins_cost(CALL_COST);
16572 
16573   format %{ "b rethrow_stub" %}
16574 
16575   ins_encode( aarch64_enc_rethrow() );
16576 
16577   ins_pipe(pipe_class_call);
16578 %}
16579 
16580 
16581 // Return Instruction
16582 // epilog node loads ret address into lr as part of frame pop
16583 instruct Ret()
16584 %{
16585   match(Return);
16586 
16587   format %{ "ret\t// return register" %}
16588 
16589   ins_encode( aarch64_enc_ret() );
16590 
16591   ins_pipe(pipe_branch);
16592 %}
16593 
16594 // Die now.
16595 instruct ShouldNotReachHere() %{
16596   match(Halt);
16597 
16598   ins_cost(CALL_COST);
16599   format %{ "ShouldNotReachHere" %}
16600 
16601   ins_encode %{
16602     if (is_reachable()) {
16603       const char* str = __ code_string(_halt_reason);
16604       __ stop(str);
16605     }
16606   %}
16607 
16608   ins_pipe(pipe_class_default);
16609 %}
16610 
16611 // ============================================================================
16612 // Partial Subtype Check
16613 //
16614 // superklass array for an instance of the superklass.  Set a hidden
16615 // internal cache on a hit (cache is checked with exposed code in
16616 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16617 // encoding ALSO sets flags.
16618 
16619 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16620 %{
16621   match(Set result (PartialSubtypeCheck sub super));
16622   predicate(!UseSecondarySupersTable);
16623   effect(KILL cr, KILL temp);
16624 
16625   ins_cost(20 * INSN_COST);  // slightly larger than the next version
16626   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16627 
16628   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16629 
16630   opcode(0x1); // Force zero of result reg on hit
16631 
16632   ins_pipe(pipe_class_memory);
16633 %}
16634 
16635 // Two versions of partialSubtypeCheck, both used when we need to
16636 // search for a super class in the secondary supers array. The first
16637 // is used when we don't know _a priori_ the class being searched
16638 // for. The second, far more common, is used when we do know: this is
16639 // used for instanceof, checkcast, and any case where C2 can determine
16640 // it by constant propagation.
16641 
16642 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16643                                      iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16644                                      rFlagsReg cr)
16645 %{
16646   match(Set result (PartialSubtypeCheck sub super));
16647   predicate(UseSecondarySupersTable);
16648   effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16649 
16650   ins_cost(10 * INSN_COST);  // slightly larger than the next version
16651   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16652 
16653   ins_encode %{
16654     __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16655                                          $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16656                                          $vtemp$$FloatRegister,
16657                                          $result$$Register, /*L_success*/nullptr);
16658   %}
16659 
16660   ins_pipe(pipe_class_memory);
16661 %}
16662 
16663 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16664                                        iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16665                                        rFlagsReg cr)
16666 %{
16667   match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16668   predicate(UseSecondarySupersTable);
16669   effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16670 
16671   ins_cost(5 * INSN_COST);  // smaller than the next version
16672   format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16673 
16674   ins_encode %{
16675     bool success = false;
16676     u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16677     if (InlineSecondarySupersTest) {
16678       success =
16679         __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16680                                                $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16681                                                $vtemp$$FloatRegister,
16682                                                $result$$Register,
16683                                                super_klass_slot);
16684     } else {
16685       address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16686       success = (call != nullptr);
16687     }
16688     if (!success) {
16689       ciEnv::current()->record_failure("CodeCache is full");
16690       return;
16691     }
16692   %}
16693 
16694   ins_pipe(pipe_class_memory);
16695 %}
16696 
16697 // Intrisics for String.compareTo()
16698 
16699 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16700                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16701 %{
16702   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16703   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16704   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16705 
16706   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16707   ins_encode %{
16708     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16709     __ string_compare($str1$$Register, $str2$$Register,
16710                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16711                       $tmp1$$Register, $tmp2$$Register,
16712                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16713   %}
16714   ins_pipe(pipe_class_memory);
16715 %}
16716 
16717 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16718                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16719 %{
16720   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16721   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16722   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16723 
16724   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16725   ins_encode %{
16726     __ string_compare($str1$$Register, $str2$$Register,
16727                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16728                       $tmp1$$Register, $tmp2$$Register,
16729                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16730   %}
16731   ins_pipe(pipe_class_memory);
16732 %}
16733 
16734 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16735                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16736                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16737 %{
16738   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16739   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16740   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16741          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16742 
16743   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16744   ins_encode %{
16745     __ string_compare($str1$$Register, $str2$$Register,
16746                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16747                       $tmp1$$Register, $tmp2$$Register,
16748                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16749                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16750   %}
16751   ins_pipe(pipe_class_memory);
16752 %}
16753 
16754 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16755                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16756                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16757 %{
16758   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16759   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16760   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16761          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16762 
16763   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16764   ins_encode %{
16765     __ string_compare($str1$$Register, $str2$$Register,
16766                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16767                       $tmp1$$Register, $tmp2$$Register,
16768                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16769                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16770   %}
16771   ins_pipe(pipe_class_memory);
16772 %}
16773 
16774 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16775 // these string_compare variants as NEON register type for convenience so that the prototype of
16776 // string_compare can be shared with all variants.
16777 
16778 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16779                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16780                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16781                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16782 %{
16783   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16784   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16785   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16786          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16787 
16788   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16789   ins_encode %{
16790     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16791     __ string_compare($str1$$Register, $str2$$Register,
16792                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16793                       $tmp1$$Register, $tmp2$$Register,
16794                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16795                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16796                       StrIntrinsicNode::LL);
16797   %}
16798   ins_pipe(pipe_class_memory);
16799 %}
16800 
16801 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16802                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16803                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16804                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16805 %{
16806   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16807   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16808   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16809          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16810 
16811   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16812   ins_encode %{
16813     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16814     __ string_compare($str1$$Register, $str2$$Register,
16815                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16816                       $tmp1$$Register, $tmp2$$Register,
16817                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16818                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16819                       StrIntrinsicNode::LU);
16820   %}
16821   ins_pipe(pipe_class_memory);
16822 %}
16823 
16824 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16825                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16826                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16827                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16828 %{
16829   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16830   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16831   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16832          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16833 
16834   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16835   ins_encode %{
16836     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16837     __ string_compare($str1$$Register, $str2$$Register,
16838                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16839                       $tmp1$$Register, $tmp2$$Register,
16840                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16841                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16842                       StrIntrinsicNode::UL);
16843   %}
16844   ins_pipe(pipe_class_memory);
16845 %}
16846 
16847 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16848                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16849                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16850                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16851 %{
16852   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16853   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16854   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16855          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16856 
16857   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16858   ins_encode %{
16859     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16860     __ string_compare($str1$$Register, $str2$$Register,
16861                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16862                       $tmp1$$Register, $tmp2$$Register,
16863                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16864                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16865                       StrIntrinsicNode::UU);
16866   %}
16867   ins_pipe(pipe_class_memory);
16868 %}
16869 
16870 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16871                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16872                           iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16873                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16874 %{
16875   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16876   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16877   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16878          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16879          TEMP vtmp0, TEMP vtmp1, KILL cr);
16880   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16881             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16882 
16883   ins_encode %{
16884     __ string_indexof($str1$$Register, $str2$$Register,
16885                       $cnt1$$Register, $cnt2$$Register,
16886                       $tmp1$$Register, $tmp2$$Register,
16887                       $tmp3$$Register, $tmp4$$Register,
16888                       $tmp5$$Register, $tmp6$$Register,
16889                       -1, $result$$Register, StrIntrinsicNode::UU);
16890   %}
16891   ins_pipe(pipe_class_memory);
16892 %}
16893 
16894 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16895                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16896                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16897                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16898 %{
16899   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16900   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16901   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16902          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16903          TEMP vtmp0, TEMP vtmp1, KILL cr);
16904   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16905             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16906 
16907   ins_encode %{
16908     __ string_indexof($str1$$Register, $str2$$Register,
16909                       $cnt1$$Register, $cnt2$$Register,
16910                       $tmp1$$Register, $tmp2$$Register,
16911                       $tmp3$$Register, $tmp4$$Register,
16912                       $tmp5$$Register, $tmp6$$Register,
16913                       -1, $result$$Register, StrIntrinsicNode::LL);
16914   %}
16915   ins_pipe(pipe_class_memory);
16916 %}
16917 
16918 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16919                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16920                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16921                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16922 %{
16923   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16924   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16925   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16926          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16927          TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16928   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16929             "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16930 
16931   ins_encode %{
16932     __ string_indexof($str1$$Register, $str2$$Register,
16933                       $cnt1$$Register, $cnt2$$Register,
16934                       $tmp1$$Register, $tmp2$$Register,
16935                       $tmp3$$Register, $tmp4$$Register,
16936                       $tmp5$$Register, $tmp6$$Register,
16937                       -1, $result$$Register, StrIntrinsicNode::UL);
16938   %}
16939   ins_pipe(pipe_class_memory);
16940 %}
16941 
16942 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16943                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16944                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16945 %{
16946   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16947   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16948   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16949          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16950   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16951             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16952 
16953   ins_encode %{
16954     int icnt2 = (int)$int_cnt2$$constant;
16955     __ string_indexof($str1$$Register, $str2$$Register,
16956                       $cnt1$$Register, zr,
16957                       $tmp1$$Register, $tmp2$$Register,
16958                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16959                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16960   %}
16961   ins_pipe(pipe_class_memory);
16962 %}
16963 
16964 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16965                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16966                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16967 %{
16968   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16969   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16970   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16971          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16972   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16973             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16974 
16975   ins_encode %{
16976     int icnt2 = (int)$int_cnt2$$constant;
16977     __ string_indexof($str1$$Register, $str2$$Register,
16978                       $cnt1$$Register, zr,
16979                       $tmp1$$Register, $tmp2$$Register,
16980                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16981                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16982   %}
16983   ins_pipe(pipe_class_memory);
16984 %}
16985 
16986 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16987                               immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16988                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16989 %{
16990   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16991   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16992   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16993          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16994   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16995             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16996 
16997   ins_encode %{
16998     int icnt2 = (int)$int_cnt2$$constant;
16999     __ string_indexof($str1$$Register, $str2$$Register,
17000                       $cnt1$$Register, zr,
17001                       $tmp1$$Register, $tmp2$$Register,
17002                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17003                       icnt2, $result$$Register, StrIntrinsicNode::UL);
17004   %}
17005   ins_pipe(pipe_class_memory);
17006 %}
17007 
17008 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17009                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17010                              iRegINoSp tmp3, rFlagsReg cr)
17011 %{
17012   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17013   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17014   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17015          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17016 
17017   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17018 
17019   ins_encode %{
17020     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17021                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
17022                            $tmp3$$Register);
17023   %}
17024   ins_pipe(pipe_class_memory);
17025 %}
17026 
17027 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17028                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17029                               iRegINoSp tmp3, rFlagsReg cr)
17030 %{
17031   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17032   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17033   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17034          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17035 
17036   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17037 
17038   ins_encode %{
17039     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17040                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
17041                             $tmp3$$Register);
17042   %}
17043   ins_pipe(pipe_class_memory);
17044 %}
17045 
17046 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17047                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17048                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17049   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17050   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17051   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17052   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17053   ins_encode %{
17054     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17055                                $result$$Register, $ztmp1$$FloatRegister,
17056                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17057                                $ptmp$$PRegister, true /* isL */);
17058   %}
17059   ins_pipe(pipe_class_memory);
17060 %}
17061 
17062 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17063                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17064                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17065   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17066   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17067   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17068   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17069   ins_encode %{
17070     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17071                                $result$$Register, $ztmp1$$FloatRegister,
17072                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17073                                $ptmp$$PRegister, false /* isL */);
17074   %}
17075   ins_pipe(pipe_class_memory);
17076 %}
17077 
17078 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17079                         iRegI_R0 result, rFlagsReg cr)
17080 %{
17081   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17082   match(Set result (StrEquals (Binary str1 str2) cnt));
17083   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17084 
17085   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17086   ins_encode %{
17087     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17088     __ string_equals($str1$$Register, $str2$$Register,
17089                      $result$$Register, $cnt$$Register);
17090   %}
17091   ins_pipe(pipe_class_memory);
17092 %}
17093 
17094 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17095                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17096                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17097                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17098                        iRegP_R10 tmp, rFlagsReg cr)
17099 %{
17100   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17101   match(Set result (AryEq ary1 ary2));
17102   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17103          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17104          TEMP vtmp6, TEMP vtmp7, KILL cr);
17105 
17106   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17107   ins_encode %{
17108     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17109                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17110                                    $result$$Register, $tmp$$Register, 1);
17111     if (tpc == nullptr) {
17112       ciEnv::current()->record_failure("CodeCache is full");
17113       return;
17114     }
17115   %}
17116   ins_pipe(pipe_class_memory);
17117 %}
17118 
17119 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17120                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17121                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17122                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17123                        iRegP_R10 tmp, rFlagsReg cr)
17124 %{
17125   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17126   match(Set result (AryEq ary1 ary2));
17127   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17128          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17129          TEMP vtmp6, TEMP vtmp7, KILL cr);
17130 
17131   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17132   ins_encode %{
17133     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17134                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17135                                    $result$$Register, $tmp$$Register, 2);
17136     if (tpc == nullptr) {
17137       ciEnv::current()->record_failure("CodeCache is full");
17138       return;
17139     }
17140   %}
17141   ins_pipe(pipe_class_memory);
17142 %}
17143 
17144 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
17145                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17146                          vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17147                          vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
17148 %{
17149   match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
17150   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
17151          TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
17152 
17153   format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result   // KILL all" %}
17154   ins_encode %{
17155     address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
17156                                      $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
17157                                      $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
17158                                      $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
17159                                      $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
17160                                      $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
17161                                      (BasicType)$basic_type$$constant);
17162     if (tpc == nullptr) {
17163       ciEnv::current()->record_failure("CodeCache is full");
17164       return;
17165     }
17166   %}
17167   ins_pipe(pipe_class_memory);
17168 %}
17169 
17170 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17171 %{
17172   match(Set result (CountPositives ary1 len));
17173   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17174   format %{ "count positives byte[] $ary1,$len -> $result" %}
17175   ins_encode %{
17176     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17177     if (tpc == nullptr) {
17178       ciEnv::current()->record_failure("CodeCache is full");
17179       return;
17180     }
17181   %}
17182   ins_pipe( pipe_slow );
17183 %}
17184 
17185 // fast char[] to byte[] compression
17186 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17187                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17188                          vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17189                          iRegI_R0 result, rFlagsReg cr)
17190 %{
17191   match(Set result (StrCompressedCopy src (Binary dst len)));
17192   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17193          USE_KILL src, USE_KILL dst, USE len, KILL cr);
17194 
17195   format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17196   ins_encode %{
17197     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17198                            $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17199                            $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17200                            $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17201   %}
17202   ins_pipe(pipe_slow);
17203 %}
17204 
17205 // fast byte[] to char[] inflation
17206 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17207                         vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17208                         vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17209 %{
17210   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17211   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17212          TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17213          USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17214 
17215   format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17216   ins_encode %{
17217     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17218                                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17219                                         $vtmp2$$FloatRegister, $tmp$$Register);
17220     if (tpc == nullptr) {
17221       ciEnv::current()->record_failure("CodeCache is full");
17222       return;
17223     }
17224   %}
17225   ins_pipe(pipe_class_memory);
17226 %}
17227 
17228 // encode char[] to byte[] in ISO_8859_1
17229 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17230                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17231                           vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17232                           iRegI_R0 result, rFlagsReg cr)
17233 %{
17234   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17235   match(Set result (EncodeISOArray src (Binary dst len)));
17236   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17237          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17238 
17239   format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17240   ins_encode %{
17241     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17242                         $result$$Register, false,
17243                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17244                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17245                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17246   %}
17247   ins_pipe(pipe_class_memory);
17248 %}
17249 
17250 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17251                             vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17252                             vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17253                             iRegI_R0 result, rFlagsReg cr)
17254 %{
17255   predicate(((EncodeISOArrayNode*)n)->is_ascii());
17256   match(Set result (EncodeISOArray src (Binary dst len)));
17257   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17258          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17259 
17260   format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17261   ins_encode %{
17262     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17263                         $result$$Register, true,
17264                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17265                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17266                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17267   %}
17268   ins_pipe(pipe_class_memory);
17269 %}
17270 
17271 //----------------------------- CompressBits/ExpandBits ------------------------
17272 
17273 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17274                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17275   match(Set dst (CompressBits src mask));
17276   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17277   format %{ "mov    $tsrc, $src\n\t"
17278             "mov    $tmask, $mask\n\t"
17279             "bext   $tdst, $tsrc, $tmask\n\t"
17280             "mov    $dst, $tdst"
17281           %}
17282   ins_encode %{
17283     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17284     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17285     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17286     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17287   %}
17288   ins_pipe(pipe_slow);
17289 %}
17290 
17291 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17292                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17293   match(Set dst (CompressBits (LoadI mem) mask));
17294   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17295   format %{ "ldrs   $tsrc, $mem\n\t"
17296             "ldrs   $tmask, $mask\n\t"
17297             "bext   $tdst, $tsrc, $tmask\n\t"
17298             "mov    $dst, $tdst"
17299           %}
17300   ins_encode %{
17301     loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17302               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17303     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17304     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17305     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17306   %}
17307   ins_pipe(pipe_slow);
17308 %}
17309 
17310 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17311                            vRegD tdst, vRegD tsrc, vRegD tmask) %{
17312   match(Set dst (CompressBits src mask));
17313   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17314   format %{ "mov    $tsrc, $src\n\t"
17315             "mov    $tmask, $mask\n\t"
17316             "bext   $tdst, $tsrc, $tmask\n\t"
17317             "mov    $dst, $tdst"
17318           %}
17319   ins_encode %{
17320     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17321     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17322     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17323     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17324   %}
17325   ins_pipe(pipe_slow);
17326 %}
17327 
17328 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17329                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17330   match(Set dst (CompressBits (LoadL mem) mask));
17331   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17332   format %{ "ldrd   $tsrc, $mem\n\t"
17333             "ldrd   $tmask, $mask\n\t"
17334             "bext   $tdst, $tsrc, $tmask\n\t"
17335             "mov    $dst, $tdst"
17336           %}
17337   ins_encode %{
17338     loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17339               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17340     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17341     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17342     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17343   %}
17344   ins_pipe(pipe_slow);
17345 %}
17346 
17347 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17348                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17349   match(Set dst (ExpandBits src mask));
17350   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17351   format %{ "mov    $tsrc, $src\n\t"
17352             "mov    $tmask, $mask\n\t"
17353             "bdep   $tdst, $tsrc, $tmask\n\t"
17354             "mov    $dst, $tdst"
17355           %}
17356   ins_encode %{
17357     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17358     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17359     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17360     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17361   %}
17362   ins_pipe(pipe_slow);
17363 %}
17364 
17365 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17366                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17367   match(Set dst (ExpandBits (LoadI mem) mask));
17368   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17369   format %{ "ldrs   $tsrc, $mem\n\t"
17370             "ldrs   $tmask, $mask\n\t"
17371             "bdep   $tdst, $tsrc, $tmask\n\t"
17372             "mov    $dst, $tdst"
17373           %}
17374   ins_encode %{
17375     loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17376               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17377     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17378     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17379     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17380   %}
17381   ins_pipe(pipe_slow);
17382 %}
17383 
17384 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17385                          vRegD tdst, vRegD tsrc, vRegD tmask) %{
17386   match(Set dst (ExpandBits src mask));
17387   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17388   format %{ "mov    $tsrc, $src\n\t"
17389             "mov    $tmask, $mask\n\t"
17390             "bdep   $tdst, $tsrc, $tmask\n\t"
17391             "mov    $dst, $tdst"
17392           %}
17393   ins_encode %{
17394     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17395     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17396     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17397     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17398   %}
17399   ins_pipe(pipe_slow);
17400 %}
17401 
17402 
17403 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17404                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17405   match(Set dst (ExpandBits (LoadL mem) mask));
17406   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17407   format %{ "ldrd   $tsrc, $mem\n\t"
17408             "ldrd   $tmask, $mask\n\t"
17409             "bdep   $tdst, $tsrc, $tmask\n\t"
17410             "mov    $dst, $tdst"
17411           %}
17412   ins_encode %{
17413     loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17414               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17415     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17416     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17417     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17418   %}
17419   ins_pipe(pipe_slow);
17420 %}
17421 
17422 //----------------------------- Reinterpret ----------------------------------
17423 // Reinterpret a half-precision float value in a floating point register to a general purpose register
17424 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
17425   match(Set dst (ReinterpretHF2S src));
17426   format %{ "reinterpretHF2S $dst, $src" %}
17427   ins_encode %{
17428     __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
17429   %}
17430   ins_pipe(pipe_slow);
17431 %}
17432 
17433 // Reinterpret a half-precision float value in a general purpose register to a floating point register
17434 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
17435   match(Set dst (ReinterpretS2HF src));
17436   format %{ "reinterpretS2HF $dst, $src" %}
17437   ins_encode %{
17438     __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
17439   %}
17440   ins_pipe(pipe_slow);
17441 %}
17442 
17443 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
17444 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
17445 // fcvt $tmp1_fpr, $src_fpr    // Convert float to half-precision float
17446 // mov  $tmp2_gpr, $tmp1_fpr   // Move half-precision float in FPR to a GPR
17447 // mov  $dst_fpr,  $tmp2_gpr   // Move the result from a GPR to an FPR
17448 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
17449 // can be omitted in this pattern, resulting in -
17450 // fcvt $dst, $src  // Convert float to half-precision float
17451 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
17452 %{
17453   match(Set dst (ReinterpretS2HF (ConvF2HF src)));
17454   format %{ "convF2HFAndS2HF $dst, $src" %}
17455   ins_encode %{
17456     __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
17457   %}
17458   ins_pipe(pipe_slow);
17459 %}
17460 
17461 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
17462 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
17463 // mov  $tmp1_gpr, $src_fpr  // Move the half-precision float from an FPR to a GPR
17464 // mov  $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
17465 // fcvt $dst_fpr,  $tmp2_fpr // Convert the half-precision float to 32-bit float
17466 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
17467 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
17468 // resulting in -
17469 // fcvt $dst, $src  // Convert half-precision float to a 32-bit float
17470 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
17471 %{
17472   match(Set dst (ConvHF2F (ReinterpretHF2S src)));
17473   format %{ "convHF2SAndHF2F $dst, $src" %}
17474   ins_encode %{
17475     __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
17476   %}
17477   ins_pipe(pipe_slow);
17478 %}
17479 
17480 // ============================================================================
17481 // This name is KNOWN by the ADLC and cannot be changed.
17482 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17483 // for this guy.
17484 instruct tlsLoadP(thread_RegP dst)
17485 %{
17486   match(Set dst (ThreadLocal));
17487 
17488   ins_cost(0);
17489 
17490   format %{ " -- \t// $dst=Thread::current(), empty" %}
17491 
17492   size(0);
17493 
17494   ins_encode( /*empty*/ );
17495 
17496   ins_pipe(pipe_class_empty);
17497 %}
17498 
17499 //----------PEEPHOLE RULES-----------------------------------------------------
17500 // These must follow all instruction definitions as they use the names
17501 // defined in the instructions definitions.
17502 //
17503 // peepmatch ( root_instr_name [preceding_instruction]* );
17504 //
17505 // peepconstraint %{
17506 // (instruction_number.operand_name relational_op instruction_number.operand_name
17507 //  [, ...] );
17508 // // instruction numbers are zero-based using left to right order in peepmatch
17509 //
17510 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17511 // // provide an instruction_number.operand_name for each operand that appears
17512 // // in the replacement instruction's match rule
17513 //
17514 // ---------VM FLAGS---------------------------------------------------------
17515 //
17516 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17517 //
17518 // Each peephole rule is given an identifying number starting with zero and
17519 // increasing by one in the order seen by the parser.  An individual peephole
17520 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17521 // on the command-line.
17522 //
17523 // ---------CURRENT LIMITATIONS----------------------------------------------
17524 //
17525 // Only match adjacent instructions in same basic block
17526 // Only equality constraints
17527 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17528 // Only one replacement instruction
17529 //
17530 // ---------EXAMPLE----------------------------------------------------------
17531 //
17532 // // pertinent parts of existing instructions in architecture description
17533 // instruct movI(iRegINoSp dst, iRegI src)
17534 // %{
17535 //   match(Set dst (CopyI src));
17536 // %}
17537 //
17538 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17539 // %{
17540 //   match(Set dst (AddI dst src));
17541 //   effect(KILL cr);
17542 // %}
17543 //
17544 // // Change (inc mov) to lea
17545 // peephole %{
17546 //   // increment preceded by register-register move
17547 //   peepmatch ( incI_iReg movI );
17548 //   // require that the destination register of the increment
17549 //   // match the destination register of the move
17550 //   peepconstraint ( 0.dst == 1.dst );
17551 //   // construct a replacement instruction that sets
17552 //   // the destination to ( move's source register + one )
17553 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17554 // %}
17555 //
17556 
17557 // Implementation no longer uses movX instructions since
17558 // machine-independent system no longer uses CopyX nodes.
17559 //
17560 // peephole
17561 // %{
17562 //   peepmatch (incI_iReg movI);
17563 //   peepconstraint (0.dst == 1.dst);
17564 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17565 // %}
17566 
17567 // peephole
17568 // %{
17569 //   peepmatch (decI_iReg movI);
17570 //   peepconstraint (0.dst == 1.dst);
17571 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17572 // %}
17573 
17574 // peephole
17575 // %{
17576 //   peepmatch (addI_iReg_imm movI);
17577 //   peepconstraint (0.dst == 1.dst);
17578 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17579 // %}
17580 
17581 // peephole
17582 // %{
17583 //   peepmatch (incL_iReg movL);
17584 //   peepconstraint (0.dst == 1.dst);
17585 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17586 // %}
17587 
17588 // peephole
17589 // %{
17590 //   peepmatch (decL_iReg movL);
17591 //   peepconstraint (0.dst == 1.dst);
17592 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17593 // %}
17594 
17595 // peephole
17596 // %{
17597 //   peepmatch (addL_iReg_imm movL);
17598 //   peepconstraint (0.dst == 1.dst);
17599 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17600 // %}
17601 
17602 // peephole
17603 // %{
17604 //   peepmatch (addP_iReg_imm movP);
17605 //   peepconstraint (0.dst == 1.dst);
17606 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17607 // %}
17608 
17609 // // Change load of spilled value to only a spill
17610 // instruct storeI(memory mem, iRegI src)
17611 // %{
17612 //   match(Set mem (StoreI mem src));
17613 // %}
17614 //
17615 // instruct loadI(iRegINoSp dst, memory mem)
17616 // %{
17617 //   match(Set dst (LoadI mem));
17618 // %}
17619 //
17620 
17621 //----------SMARTSPILL RULES---------------------------------------------------
17622 // These must follow all instruction definitions as they use the names
17623 // defined in the instructions definitions.
17624 
17625 // Local Variables:
17626 // mode: c++
17627 // End: