1 //
    2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // architecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182 
  183   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  184   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  185   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  186   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  187 
  188   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  189   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  190   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  191   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  192 
  193   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  194   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  195   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  196   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  197 
  198   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  199   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  200   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  201   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  202 
  203   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  204   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  205   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  206   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  207 
  208   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  209   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  210   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  211   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  212 
  213   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  214   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  215   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  216   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  217 
  218   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  219   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  220   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  221   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  222 
  223   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  224   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  225   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  226   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  227 
  228   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  229   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  230   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  231   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  232 
  233   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  234   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  235   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  236   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  237 
  238   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  239   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  240   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  241   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  242 
  243   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  244   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  245   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  246   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  247 
  248   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  249   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  250   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  251   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  252 
  253   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  254   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  255   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  256   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  257 
  258   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  259   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  260   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  261   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  262 
  263   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  264   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  265   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  266   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  267 
  268   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  269   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  270   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  271   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  272 
  273   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  274   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  275   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  276   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  277 
  278   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  279   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  280   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  281   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  282 
  283   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  284   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  285   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  286   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  287 
  288   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  289   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  290   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  291   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  292 
  293   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  294   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  295   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  296   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  297 
  298   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  299   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  300   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  301   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  302 
  303   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  304   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  305   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  306   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  307 
  308   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  309   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  310   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  311   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  312 
  313   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  314   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  315   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  316   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  317 
  318   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  319   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  320   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  321   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  322 
  323   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  324   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  325   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  326   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  327 
  328   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  329   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  330   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  331   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  332 
  333   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  334   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  335   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  336   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  337 
  338 // ----------------------------
  339 // SVE Predicate Registers
  340 // ----------------------------
  341   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  342   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  343   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  344   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  345   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  346   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  347   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  348   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  349   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  350   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  351   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  352   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  353   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  354   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  355   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  356   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  357 
  358 // ----------------------------
  359 // Special Registers
  360 // ----------------------------
  361 
  362 // the AArch64 CSPR status flag register is not directly accessible as
  363 // instruction operand. the FPSR status flag register is a system
  364 // register which can be written/read using MSR/MRS but again does not
  365 // appear as an operand (a code identifying the FSPR occurs as an
  366 // immediate value in the instruction).
  367 
  368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  369 
  370 // Specify priority of register selection within phases of register
  371 // allocation.  Highest priority is first.  A useful heuristic is to
  372 // give registers a low priority when they are required by machine
  373 // instructions, like EAX and EDX on I486, and choose no-save registers
  374 // before save-on-call, & save-on-call before save-on-entry.  Registers
  375 // which participate in fixed calling sequences should come last.
  376 // Registers which are used as pairs must fall on an even boundary.
  377 
  378 alloc_class chunk0(
  379     // volatiles
  380     R10, R10_H,
  381     R11, R11_H,
  382     R12, R12_H,
  383     R13, R13_H,
  384     R14, R14_H,
  385     R15, R15_H,
  386     R16, R16_H,
  387     R17, R17_H,
  388     R18, R18_H,
  389 
  390     // arg registers
  391     R0, R0_H,
  392     R1, R1_H,
  393     R2, R2_H,
  394     R3, R3_H,
  395     R4, R4_H,
  396     R5, R5_H,
  397     R6, R6_H,
  398     R7, R7_H,
  399 
  400     // non-volatiles
  401     R19, R19_H,
  402     R20, R20_H,
  403     R21, R21_H,
  404     R22, R22_H,
  405     R23, R23_H,
  406     R24, R24_H,
  407     R25, R25_H,
  408     R26, R26_H,
  409 
  410     // non-allocatable registers
  411 
  412     R27, R27_H, // heapbase
  413     R28, R28_H, // thread
  414     R29, R29_H, // fp
  415     R30, R30_H, // lr
  416     R31, R31_H, // sp
  417     R8, R8_H,   // rscratch1
  418     R9, R9_H,   // rscratch2
  419 );
  420 
  421 alloc_class chunk1(
  422 
  423     // no save
  424     V16, V16_H, V16_J, V16_K,
  425     V17, V17_H, V17_J, V17_K,
  426     V18, V18_H, V18_J, V18_K,
  427     V19, V19_H, V19_J, V19_K,
  428     V20, V20_H, V20_J, V20_K,
  429     V21, V21_H, V21_J, V21_K,
  430     V22, V22_H, V22_J, V22_K,
  431     V23, V23_H, V23_J, V23_K,
  432     V24, V24_H, V24_J, V24_K,
  433     V25, V25_H, V25_J, V25_K,
  434     V26, V26_H, V26_J, V26_K,
  435     V27, V27_H, V27_J, V27_K,
  436     V28, V28_H, V28_J, V28_K,
  437     V29, V29_H, V29_J, V29_K,
  438     V30, V30_H, V30_J, V30_K,
  439     V31, V31_H, V31_J, V31_K,
  440 
  441     // arg registers
  442     V0, V0_H, V0_J, V0_K,
  443     V1, V1_H, V1_J, V1_K,
  444     V2, V2_H, V2_J, V2_K,
  445     V3, V3_H, V3_J, V3_K,
  446     V4, V4_H, V4_J, V4_K,
  447     V5, V5_H, V5_J, V5_K,
  448     V6, V6_H, V6_J, V6_K,
  449     V7, V7_H, V7_J, V7_K,
  450 
  451     // non-volatiles
  452     V8, V8_H, V8_J, V8_K,
  453     V9, V9_H, V9_J, V9_K,
  454     V10, V10_H, V10_J, V10_K,
  455     V11, V11_H, V11_J, V11_K,
  456     V12, V12_H, V12_J, V12_K,
  457     V13, V13_H, V13_J, V13_K,
  458     V14, V14_H, V14_J, V14_K,
  459     V15, V15_H, V15_J, V15_K,
  460 );
  461 
  462 alloc_class chunk2 (
  463     // Governing predicates for load/store and arithmetic
  464     P0,
  465     P1,
  466     P2,
  467     P3,
  468     P4,
  469     P5,
  470     P6,
  471 
  472     // Extra predicates
  473     P8,
  474     P9,
  475     P10,
  476     P11,
  477     P12,
  478     P13,
  479     P14,
  480     P15,
  481 
  482     // Preserved for all-true predicate
  483     P7,
  484 );
  485 
  486 alloc_class chunk3(RFLAGS);
  487 
  488 //----------Architecture Description Register Classes--------------------------
  489 // Several register classes are automatically defined based upon information in
  490 // this architecture description.
  491 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  493 //
  494 
  495 // Class for all 32 bit general purpose registers
  496 reg_class all_reg32(
  497     R0,
  498     R1,
  499     R2,
  500     R3,
  501     R4,
  502     R5,
  503     R6,
  504     R7,
  505     R10,
  506     R11,
  507     R12,
  508     R13,
  509     R14,
  510     R15,
  511     R16,
  512     R17,
  513     R18,
  514     R19,
  515     R20,
  516     R21,
  517     R22,
  518     R23,
  519     R24,
  520     R25,
  521     R26,
  522     R27,
  523     R28,
  524     R29,
  525     R30,
  526     R31
  527 );
  528 
  529 
  530 // Class for all 32 bit integer registers (excluding SP which
  531 // will never be used as an integer register)
  532 reg_class any_reg32 %{
  533   return _ANY_REG32_mask;
  534 %}
  535 
  536 // Singleton class for R0 int register
  537 reg_class int_r0_reg(R0);
  538 
  539 // Singleton class for R2 int register
  540 reg_class int_r2_reg(R2);
  541 
  542 // Singleton class for R3 int register
  543 reg_class int_r3_reg(R3);
  544 
  545 // Singleton class for R4 int register
  546 reg_class int_r4_reg(R4);
  547 
  548 // Singleton class for R31 int register
  549 reg_class int_r31_reg(R31);
  550 
  551 // Class for all 64 bit general purpose registers
  552 reg_class all_reg(
  553     R0, R0_H,
  554     R1, R1_H,
  555     R2, R2_H,
  556     R3, R3_H,
  557     R4, R4_H,
  558     R5, R5_H,
  559     R6, R6_H,
  560     R7, R7_H,
  561     R10, R10_H,
  562     R11, R11_H,
  563     R12, R12_H,
  564     R13, R13_H,
  565     R14, R14_H,
  566     R15, R15_H,
  567     R16, R16_H,
  568     R17, R17_H,
  569     R18, R18_H,
  570     R19, R19_H,
  571     R20, R20_H,
  572     R21, R21_H,
  573     R22, R22_H,
  574     R23, R23_H,
  575     R24, R24_H,
  576     R25, R25_H,
  577     R26, R26_H,
  578     R27, R27_H,
  579     R28, R28_H,
  580     R29, R29_H,
  581     R30, R30_H,
  582     R31, R31_H
  583 );
  584 
  585 // Class for all long integer registers (including SP)
  586 reg_class any_reg %{
  587   return _ANY_REG_mask;
  588 %}
  589 
  590 // Class for non-allocatable 32 bit registers
  591 reg_class non_allocatable_reg32(
  592 #ifdef R18_RESERVED
  593     // See comment in register_aarch64.hpp
  594     R18,                        // tls on Windows
  595 #endif
  596     R28,                        // thread
  597     R30,                        // lr
  598     R31                         // sp
  599 );
  600 
  601 // Class for non-allocatable 64 bit registers
  602 reg_class non_allocatable_reg(
  603 #ifdef R18_RESERVED
  604     // See comment in register_aarch64.hpp
  605     R18, R18_H,                 // tls on Windows, platform register on macOS
  606 #endif
  607     R28, R28_H,                 // thread
  608     R30, R30_H,                 // lr
  609     R31, R31_H                  // sp
  610 );
  611 
  612 // Class for all non-special integer registers
  613 reg_class no_special_reg32 %{
  614   return _NO_SPECIAL_REG32_mask;
  615 %}
  616 
  617 // Class for all non-special long integer registers
  618 reg_class no_special_reg %{
  619   return _NO_SPECIAL_REG_mask;
  620 %}
  621 
  622 // Class for 64 bit register r0
  623 reg_class r0_reg(
  624     R0, R0_H
  625 );
  626 
  627 // Class for 64 bit register r1
  628 reg_class r1_reg(
  629     R1, R1_H
  630 );
  631 
  632 // Class for 64 bit register r2
  633 reg_class r2_reg(
  634     R2, R2_H
  635 );
  636 
  637 // Class for 64 bit register r3
  638 reg_class r3_reg(
  639     R3, R3_H
  640 );
  641 
  642 // Class for 64 bit register r4
  643 reg_class r4_reg(
  644     R4, R4_H
  645 );
  646 
  647 // Class for 64 bit register r5
  648 reg_class r5_reg(
  649     R5, R5_H
  650 );
  651 
  652 // Class for 64 bit register r10
  653 reg_class r10_reg(
  654     R10, R10_H
  655 );
  656 
  657 // Class for 64 bit register r11
  658 reg_class r11_reg(
  659     R11, R11_H
  660 );
  661 
  662 // Class for method register
  663 reg_class method_reg(
  664     R12, R12_H
  665 );
  666 
  667 // Class for thread register
  668 reg_class thread_reg(
  669     R28, R28_H
  670 );
  671 
  672 // Class for frame pointer register
  673 reg_class fp_reg(
  674     R29, R29_H
  675 );
  676 
  677 // Class for link register
  678 reg_class lr_reg(
  679     R30, R30_H
  680 );
  681 
  682 // Class for long sp register
  683 reg_class sp_reg(
  684   R31, R31_H
  685 );
  686 
  687 // Class for all pointer registers
  688 reg_class ptr_reg %{
  689   return _PTR_REG_mask;
  690 %}
  691 
  692 // Class for all non_special pointer registers
  693 reg_class no_special_ptr_reg %{
  694   return _NO_SPECIAL_PTR_REG_mask;
  695 %}
  696 
  697 // Class for all float registers
  698 reg_class float_reg(
  699     V0,
  700     V1,
  701     V2,
  702     V3,
  703     V4,
  704     V5,
  705     V6,
  706     V7,
  707     V8,
  708     V9,
  709     V10,
  710     V11,
  711     V12,
  712     V13,
  713     V14,
  714     V15,
  715     V16,
  716     V17,
  717     V18,
  718     V19,
  719     V20,
  720     V21,
  721     V22,
  722     V23,
  723     V24,
  724     V25,
  725     V26,
  726     V27,
  727     V28,
  728     V29,
  729     V30,
  730     V31
  731 );
  732 
  733 // Double precision float registers have virtual `high halves' that
  734 // are needed by the allocator.
  735 // Class for all double registers
  736 reg_class double_reg(
  737     V0, V0_H,
  738     V1, V1_H,
  739     V2, V2_H,
  740     V3, V3_H,
  741     V4, V4_H,
  742     V5, V5_H,
  743     V6, V6_H,
  744     V7, V7_H,
  745     V8, V8_H,
  746     V9, V9_H,
  747     V10, V10_H,
  748     V11, V11_H,
  749     V12, V12_H,
  750     V13, V13_H,
  751     V14, V14_H,
  752     V15, V15_H,
  753     V16, V16_H,
  754     V17, V17_H,
  755     V18, V18_H,
  756     V19, V19_H,
  757     V20, V20_H,
  758     V21, V21_H,
  759     V22, V22_H,
  760     V23, V23_H,
  761     V24, V24_H,
  762     V25, V25_H,
  763     V26, V26_H,
  764     V27, V27_H,
  765     V28, V28_H,
  766     V29, V29_H,
  767     V30, V30_H,
  768     V31, V31_H
  769 );
  770 
  771 // Class for all SVE vector registers.
  772 reg_class vectora_reg (
  773     V0, V0_H, V0_J, V0_K,
  774     V1, V1_H, V1_J, V1_K,
  775     V2, V2_H, V2_J, V2_K,
  776     V3, V3_H, V3_J, V3_K,
  777     V4, V4_H, V4_J, V4_K,
  778     V5, V5_H, V5_J, V5_K,
  779     V6, V6_H, V6_J, V6_K,
  780     V7, V7_H, V7_J, V7_K,
  781     V8, V8_H, V8_J, V8_K,
  782     V9, V9_H, V9_J, V9_K,
  783     V10, V10_H, V10_J, V10_K,
  784     V11, V11_H, V11_J, V11_K,
  785     V12, V12_H, V12_J, V12_K,
  786     V13, V13_H, V13_J, V13_K,
  787     V14, V14_H, V14_J, V14_K,
  788     V15, V15_H, V15_J, V15_K,
  789     V16, V16_H, V16_J, V16_K,
  790     V17, V17_H, V17_J, V17_K,
  791     V18, V18_H, V18_J, V18_K,
  792     V19, V19_H, V19_J, V19_K,
  793     V20, V20_H, V20_J, V20_K,
  794     V21, V21_H, V21_J, V21_K,
  795     V22, V22_H, V22_J, V22_K,
  796     V23, V23_H, V23_J, V23_K,
  797     V24, V24_H, V24_J, V24_K,
  798     V25, V25_H, V25_J, V25_K,
  799     V26, V26_H, V26_J, V26_K,
  800     V27, V27_H, V27_J, V27_K,
  801     V28, V28_H, V28_J, V28_K,
  802     V29, V29_H, V29_J, V29_K,
  803     V30, V30_H, V30_J, V30_K,
  804     V31, V31_H, V31_J, V31_K,
  805 );
  806 
  807 // Class for all 64bit vector registers
  808 reg_class vectord_reg(
  809     V0, V0_H,
  810     V1, V1_H,
  811     V2, V2_H,
  812     V3, V3_H,
  813     V4, V4_H,
  814     V5, V5_H,
  815     V6, V6_H,
  816     V7, V7_H,
  817     V8, V8_H,
  818     V9, V9_H,
  819     V10, V10_H,
  820     V11, V11_H,
  821     V12, V12_H,
  822     V13, V13_H,
  823     V14, V14_H,
  824     V15, V15_H,
  825     V16, V16_H,
  826     V17, V17_H,
  827     V18, V18_H,
  828     V19, V19_H,
  829     V20, V20_H,
  830     V21, V21_H,
  831     V22, V22_H,
  832     V23, V23_H,
  833     V24, V24_H,
  834     V25, V25_H,
  835     V26, V26_H,
  836     V27, V27_H,
  837     V28, V28_H,
  838     V29, V29_H,
  839     V30, V30_H,
  840     V31, V31_H
  841 );
  842 
  843 // Class for all 128bit vector registers
  844 reg_class vectorx_reg(
  845     V0, V0_H, V0_J, V0_K,
  846     V1, V1_H, V1_J, V1_K,
  847     V2, V2_H, V2_J, V2_K,
  848     V3, V3_H, V3_J, V3_K,
  849     V4, V4_H, V4_J, V4_K,
  850     V5, V5_H, V5_J, V5_K,
  851     V6, V6_H, V6_J, V6_K,
  852     V7, V7_H, V7_J, V7_K,
  853     V8, V8_H, V8_J, V8_K,
  854     V9, V9_H, V9_J, V9_K,
  855     V10, V10_H, V10_J, V10_K,
  856     V11, V11_H, V11_J, V11_K,
  857     V12, V12_H, V12_J, V12_K,
  858     V13, V13_H, V13_J, V13_K,
  859     V14, V14_H, V14_J, V14_K,
  860     V15, V15_H, V15_J, V15_K,
  861     V16, V16_H, V16_J, V16_K,
  862     V17, V17_H, V17_J, V17_K,
  863     V18, V18_H, V18_J, V18_K,
  864     V19, V19_H, V19_J, V19_K,
  865     V20, V20_H, V20_J, V20_K,
  866     V21, V21_H, V21_J, V21_K,
  867     V22, V22_H, V22_J, V22_K,
  868     V23, V23_H, V23_J, V23_K,
  869     V24, V24_H, V24_J, V24_K,
  870     V25, V25_H, V25_J, V25_K,
  871     V26, V26_H, V26_J, V26_K,
  872     V27, V27_H, V27_J, V27_K,
  873     V28, V28_H, V28_J, V28_K,
  874     V29, V29_H, V29_J, V29_K,
  875     V30, V30_H, V30_J, V30_K,
  876     V31, V31_H, V31_J, V31_K
  877 );
  878 
  879 // Class for 128 bit register v0
  880 reg_class v0_reg(
  881     V0, V0_H
  882 );
  883 
  884 // Class for 128 bit register v1
  885 reg_class v1_reg(
  886     V1, V1_H
  887 );
  888 
  889 // Class for 128 bit register v2
  890 reg_class v2_reg(
  891     V2, V2_H
  892 );
  893 
  894 // Class for 128 bit register v3
  895 reg_class v3_reg(
  896     V3, V3_H
  897 );
  898 
  899 // Class for 128 bit register v4
  900 reg_class v4_reg(
  901     V4, V4_H
  902 );
  903 
  904 // Class for 128 bit register v5
  905 reg_class v5_reg(
  906     V5, V5_H
  907 );
  908 
  909 // Class for 128 bit register v6
  910 reg_class v6_reg(
  911     V6, V6_H
  912 );
  913 
  914 // Class for 128 bit register v7
  915 reg_class v7_reg(
  916     V7, V7_H
  917 );
  918 
  919 // Class for 128 bit register v8
  920 reg_class v8_reg(
  921     V8, V8_H
  922 );
  923 
  924 // Class for 128 bit register v9
  925 reg_class v9_reg(
  926     V9, V9_H
  927 );
  928 
  929 // Class for 128 bit register v10
  930 reg_class v10_reg(
  931     V10, V10_H
  932 );
  933 
  934 // Class for 128 bit register v11
  935 reg_class v11_reg(
  936     V11, V11_H
  937 );
  938 
  939 // Class for 128 bit register v12
  940 reg_class v12_reg(
  941     V12, V12_H
  942 );
  943 
  944 // Class for 128 bit register v13
  945 reg_class v13_reg(
  946     V13, V13_H
  947 );
  948 
  949 // Class for 128 bit register v14
  950 reg_class v14_reg(
  951     V14, V14_H
  952 );
  953 
  954 // Class for 128 bit register v15
  955 reg_class v15_reg(
  956     V15, V15_H
  957 );
  958 
  959 // Class for 128 bit register v16
  960 reg_class v16_reg(
  961     V16, V16_H
  962 );
  963 
  964 // Class for 128 bit register v17
  965 reg_class v17_reg(
  966     V17, V17_H
  967 );
  968 
  969 // Class for 128 bit register v18
  970 reg_class v18_reg(
  971     V18, V18_H
  972 );
  973 
  974 // Class for 128 bit register v19
  975 reg_class v19_reg(
  976     V19, V19_H
  977 );
  978 
  979 // Class for 128 bit register v20
  980 reg_class v20_reg(
  981     V20, V20_H
  982 );
  983 
  984 // Class for 128 bit register v21
  985 reg_class v21_reg(
  986     V21, V21_H
  987 );
  988 
  989 // Class for 128 bit register v22
  990 reg_class v22_reg(
  991     V22, V22_H
  992 );
  993 
  994 // Class for 128 bit register v23
  995 reg_class v23_reg(
  996     V23, V23_H
  997 );
  998 
  999 // Class for 128 bit register v24
 1000 reg_class v24_reg(
 1001     V24, V24_H
 1002 );
 1003 
 1004 // Class for 128 bit register v25
 1005 reg_class v25_reg(
 1006     V25, V25_H
 1007 );
 1008 
 1009 // Class for 128 bit register v26
 1010 reg_class v26_reg(
 1011     V26, V26_H
 1012 );
 1013 
 1014 // Class for 128 bit register v27
 1015 reg_class v27_reg(
 1016     V27, V27_H
 1017 );
 1018 
 1019 // Class for 128 bit register v28
 1020 reg_class v28_reg(
 1021     V28, V28_H
 1022 );
 1023 
 1024 // Class for 128 bit register v29
 1025 reg_class v29_reg(
 1026     V29, V29_H
 1027 );
 1028 
 1029 // Class for 128 bit register v30
 1030 reg_class v30_reg(
 1031     V30, V30_H
 1032 );
 1033 
 1034 // Class for 128 bit register v31
 1035 reg_class v31_reg(
 1036     V31, V31_H
 1037 );
 1038 
 1039 // Class for all SVE predicate registers.
 1040 reg_class pr_reg (
 1041     P0,
 1042     P1,
 1043     P2,
 1044     P3,
 1045     P4,
 1046     P5,
 1047     P6,
 1048     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1049     P8,
 1050     P9,
 1051     P10,
 1052     P11,
 1053     P12,
 1054     P13,
 1055     P14,
 1056     P15
 1057 );
 1058 
 1059 // Class for SVE governing predicate registers, which are used
 1060 // to determine the active elements of a predicated instruction.
 1061 reg_class gov_pr (
 1062     P0,
 1063     P1,
 1064     P2,
 1065     P3,
 1066     P4,
 1067     P5,
 1068     P6,
 1069     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1070 );
 1071 
 1072 reg_class p0_reg(P0);
 1073 reg_class p1_reg(P1);
 1074 
 1075 // Singleton class for condition codes
 1076 reg_class int_flags(RFLAGS);
 1077 
 1078 %}
 1079 
 1080 //----------DEFINITION BLOCK---------------------------------------------------
 1081 // Define name --> value mappings to inform the ADLC of an integer valued name
 1082 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1083 // Format:
 1084 //        int_def  <name>         ( <int_value>, <expression>);
 1085 // Generated Code in ad_<arch>.hpp
 1086 //        #define  <name>   (<expression>)
 1087 //        // value == <int_value>
 1088 // Generated code in ad_<arch>.cpp adlc_verification()
 1089 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1090 //
 1091 
 1092 // we follow the ppc-aix port in using a simple cost model which ranks
 1093 // register operations as cheap, memory ops as more expensive and
 1094 // branches as most expensive. the first two have a low as well as a
 1095 // normal cost. huge cost appears to be a way of saying don't do
 1096 // something
 1097 
 1098 definitions %{
 1099   // The default cost (of a register move instruction).
 1100   int_def INSN_COST            (    100,     100);
 1101   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1102   int_def CALL_COST            (    200,     2 * INSN_COST);
 1103   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1104 %}
 1105 
 1106 
 1107 //----------SOURCE BLOCK-------------------------------------------------------
 1108 // This is a block of C++ code which provides values, functions, and
 1109 // definitions necessary in the rest of the architecture description
 1110 
 1111 source_hpp %{
 1112 
 1113 #include "asm/macroAssembler.hpp"
 1114 #include "gc/shared/barrierSetAssembler.hpp"
 1115 #include "gc/shared/cardTable.hpp"
 1116 #include "gc/shared/cardTableBarrierSet.hpp"
 1117 #include "gc/shared/collectedHeap.hpp"
 1118 #include "opto/addnode.hpp"
 1119 #include "opto/convertnode.hpp"
 1120 #include "runtime/objectMonitor.hpp"
 1121 
 1122 extern RegMask _ANY_REG32_mask;
 1123 extern RegMask _ANY_REG_mask;
 1124 extern RegMask _PTR_REG_mask;
 1125 extern RegMask _NO_SPECIAL_REG32_mask;
 1126 extern RegMask _NO_SPECIAL_REG_mask;
 1127 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1128 
 1129 class CallStubImpl {
 1130 
 1131   //--------------------------------------------------------------
 1132   //---<  Used for optimization in Compile::shorten_branches  >---
 1133   //--------------------------------------------------------------
 1134 
 1135  public:
 1136   // Size of call trampoline stub.
 1137   static uint size_call_trampoline() {
 1138     return 0; // no call trampolines on this platform
 1139   }
 1140 
 1141   // number of relocations needed by a call trampoline stub
 1142   static uint reloc_call_trampoline() {
 1143     return 0; // no call trampolines on this platform
 1144   }
 1145 };
 1146 
 1147 class HandlerImpl {
 1148 
 1149  public:
 1150 
 1151   static int emit_exception_handler(CodeBuffer &cbuf);
 1152   static int emit_deopt_handler(CodeBuffer& cbuf);
 1153 
 1154   static uint size_exception_handler() {
 1155     return MacroAssembler::far_codestub_branch_size();
 1156   }
 1157 
 1158   static uint size_deopt_handler() {
 1159     // count one adr and one far branch instruction
 1160     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1161   }
 1162 };
 1163 
 1164 class Node::PD {
 1165 public:
 1166   enum NodeFlags {
 1167     _last_flag = Node::_last_flag
 1168   };
 1169 };
 1170 
 1171   bool is_CAS(int opcode, bool maybe_volatile);
 1172 
 1173   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1174 
 1175   bool unnecessary_acquire(const Node *barrier);
 1176   bool needs_acquiring_load(const Node *load);
 1177 
 1178   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1179 
 1180   bool unnecessary_release(const Node *barrier);
 1181   bool unnecessary_volatile(const Node *barrier);
 1182   bool needs_releasing_store(const Node *store);
 1183 
 1184   // predicate controlling translation of CompareAndSwapX
 1185   bool needs_acquiring_load_exclusive(const Node *load);
 1186 
 1187   // predicate controlling addressing modes
 1188   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1189 
 1190   // Convert BootTest condition to Assembler condition.
 1191   // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 1192   Assembler::Condition to_assembler_cond(BoolTest::mask cond);
 1193 %}
 1194 
 1195 source %{
 1196 
 1197   // Derived RegMask with conditionally allocatable registers
 1198 
 1199   void PhaseOutput::pd_perform_mach_node_analysis() {
 1200   }
 1201 
 1202   int MachNode::pd_alignment_required() const {
 1203     return 1;
 1204   }
 1205 
 1206   int MachNode::compute_padding(int current_offset) const {
 1207     return 0;
 1208   }
 1209 
 1210   RegMask _ANY_REG32_mask;
 1211   RegMask _ANY_REG_mask;
 1212   RegMask _PTR_REG_mask;
 1213   RegMask _NO_SPECIAL_REG32_mask;
 1214   RegMask _NO_SPECIAL_REG_mask;
 1215   RegMask _NO_SPECIAL_PTR_REG_mask;
 1216 
 1217   void reg_mask_init() {
 1218     // We derive below RegMask(s) from the ones which are auto-generated from
 1219     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1220     // registers conditionally reserved.
 1221 
 1222     _ANY_REG32_mask = _ALL_REG32_mask;
 1223     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1224 
 1225     _ANY_REG_mask = _ALL_REG_mask;
 1226 
 1227     _PTR_REG_mask = _ALL_REG_mask;
 1228 
 1229     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1230     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1231 
 1232     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1233     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1234 
 1235     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1236     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1237 
 1238     // r27 is not allocatable when compressed oops is on and heapbase is not
 1239     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1240     if (UseCompressedOops && (CompressedOops::ptrs_base() != nullptr)) {
 1241       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1242       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1243       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1244     }
 1245 
 1246     // r29 is not allocatable when PreserveFramePointer is on
 1247     if (PreserveFramePointer) {
 1248       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1249       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1250       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1251     }
 1252   }
 1253 
 1254   // Optimizaton of volatile gets and puts
 1255   // -------------------------------------
 1256   //
 1257   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1258   // use to implement volatile reads and writes. For a volatile read
 1259   // we simply need
 1260   //
 1261   //   ldar<x>
 1262   //
 1263   // and for a volatile write we need
 1264   //
 1265   //   stlr<x>
 1266   //
 1267   // Alternatively, we can implement them by pairing a normal
 1268   // load/store with a memory barrier. For a volatile read we need
 1269   //
 1270   //   ldr<x>
 1271   //   dmb ishld
 1272   //
 1273   // for a volatile write
 1274   //
 1275   //   dmb ish
 1276   //   str<x>
 1277   //   dmb ish
 1278   //
 1279   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1280   // sequences. These are normally translated to an instruction
 1281   // sequence like the following
 1282   //
 1283   //   dmb      ish
 1284   // retry:
 1285   //   ldxr<x>   rval raddr
 1286   //   cmp       rval rold
 1287   //   b.ne done
 1288   //   stlxr<x>  rval, rnew, rold
 1289   //   cbnz      rval retry
 1290   // done:
 1291   //   cset      r0, eq
 1292   //   dmb ishld
 1293   //
 1294   // Note that the exclusive store is already using an stlxr
 1295   // instruction. That is required to ensure visibility to other
 1296   // threads of the exclusive write (assuming it succeeds) before that
 1297   // of any subsequent writes.
 1298   //
 1299   // The following instruction sequence is an improvement on the above
 1300   //
 1301   // retry:
 1302   //   ldaxr<x>  rval raddr
 1303   //   cmp       rval rold
 1304   //   b.ne done
 1305   //   stlxr<x>  rval, rnew, rold
 1306   //   cbnz      rval retry
 1307   // done:
 1308   //   cset      r0, eq
 1309   //
 1310   // We don't need the leading dmb ish since the stlxr guarantees
 1311   // visibility of prior writes in the case that the swap is
 1312   // successful. Crucially we don't have to worry about the case where
 1313   // the swap is not successful since no valid program should be
 1314   // relying on visibility of prior changes by the attempting thread
 1315   // in the case where the CAS fails.
 1316   //
 1317   // Similarly, we don't need the trailing dmb ishld if we substitute
 1318   // an ldaxr instruction since that will provide all the guarantees we
 1319   // require regarding observation of changes made by other threads
 1320   // before any change to the CAS address observed by the load.
 1321   //
 1322   // In order to generate the desired instruction sequence we need to
 1323   // be able to identify specific 'signature' ideal graph node
 1324   // sequences which i) occur as a translation of a volatile reads or
 1325   // writes or CAS operations and ii) do not occur through any other
 1326   // translation or graph transformation. We can then provide
 1327   // alternative aldc matching rules which translate these node
 1328   // sequences to the desired machine code sequences. Selection of the
 1329   // alternative rules can be implemented by predicates which identify
 1330   // the relevant node sequences.
 1331   //
 1332   // The ideal graph generator translates a volatile read to the node
 1333   // sequence
 1334   //
 1335   //   LoadX[mo_acquire]
 1336   //   MemBarAcquire
 1337   //
 1338   // As a special case when using the compressed oops optimization we
 1339   // may also see this variant
 1340   //
 1341   //   LoadN[mo_acquire]
 1342   //   DecodeN
 1343   //   MemBarAcquire
 1344   //
 1345   // A volatile write is translated to the node sequence
 1346   //
 1347   //   MemBarRelease
 1348   //   StoreX[mo_release] {CardMark}-optional
 1349   //   MemBarVolatile
 1350   //
 1351   // n.b. the above node patterns are generated with a strict
 1352   // 'signature' configuration of input and output dependencies (see
 1353   // the predicates below for exact details). The card mark may be as
 1354   // simple as a few extra nodes or, in a few GC configurations, may
 1355   // include more complex control flow between the leading and
 1356   // trailing memory barriers. However, whatever the card mark
 1357   // configuration these signatures are unique to translated volatile
 1358   // reads/stores -- they will not appear as a result of any other
 1359   // bytecode translation or inlining nor as a consequence of
 1360   // optimizing transforms.
 1361   //
 1362   // We also want to catch inlined unsafe volatile gets and puts and
 1363   // be able to implement them using either ldar<x>/stlr<x> or some
 1364   // combination of ldr<x>/stlr<x> and dmb instructions.
 1365   //
 1366   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1367   // normal volatile put node sequence containing an extra cpuorder
 1368   // membar
 1369   //
 1370   //   MemBarRelease
 1371   //   MemBarCPUOrder
 1372   //   StoreX[mo_release] {CardMark}-optional
 1373   //   MemBarCPUOrder
 1374   //   MemBarVolatile
 1375   //
 1376   // n.b. as an aside, a cpuorder membar is not itself subject to
 1377   // matching and translation by adlc rules.  However, the rule
 1378   // predicates need to detect its presence in order to correctly
 1379   // select the desired adlc rules.
 1380   //
 1381   // Inlined unsafe volatile gets manifest as a slightly different
 1382   // node sequence to a normal volatile get because of the
 1383   // introduction of some CPUOrder memory barriers to bracket the
 1384   // Load. However, but the same basic skeleton of a LoadX feeding a
 1385   // MemBarAcquire, possibly through an optional DecodeN, is still
 1386   // present
 1387   //
 1388   //   MemBarCPUOrder
 1389   //        ||       \\
 1390   //   MemBarCPUOrder LoadX[mo_acquire]
 1391   //        ||            |
 1392   //        ||       {DecodeN} optional
 1393   //        ||       /
 1394   //     MemBarAcquire
 1395   //
 1396   // In this case the acquire membar does not directly depend on the
 1397   // load. However, we can be sure that the load is generated from an
 1398   // inlined unsafe volatile get if we see it dependent on this unique
 1399   // sequence of membar nodes. Similarly, given an acquire membar we
 1400   // can know that it was added because of an inlined unsafe volatile
 1401   // get if it is fed and feeds a cpuorder membar and if its feed
 1402   // membar also feeds an acquiring load.
 1403   //
 1404   // Finally an inlined (Unsafe) CAS operation is translated to the
 1405   // following ideal graph
 1406   //
 1407   //   MemBarRelease
 1408   //   MemBarCPUOrder
 1409   //   CompareAndSwapX {CardMark}-optional
 1410   //   MemBarCPUOrder
 1411   //   MemBarAcquire
 1412   //
 1413   // So, where we can identify these volatile read and write
 1414   // signatures we can choose to plant either of the above two code
 1415   // sequences. For a volatile read we can simply plant a normal
 1416   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1417   // also choose to inhibit translation of the MemBarAcquire and
 1418   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1419   //
 1420   // When we recognise a volatile store signature we can choose to
 1421   // plant at a dmb ish as a translation for the MemBarRelease, a
 1422   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1423   // Alternatively, we can inhibit translation of the MemBarRelease
 1424   // and MemBarVolatile and instead plant a simple stlr<x>
 1425   // instruction.
 1426   //
 1427   // when we recognise a CAS signature we can choose to plant a dmb
 1428   // ish as a translation for the MemBarRelease, the conventional
 1429   // macro-instruction sequence for the CompareAndSwap node (which
 1430   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1431   // Alternatively, we can elide generation of the dmb instructions
 1432   // and plant the alternative CompareAndSwap macro-instruction
 1433   // sequence (which uses ldaxr<x>).
 1434   //
 1435   // Of course, the above only applies when we see these signature
 1436   // configurations. We still want to plant dmb instructions in any
 1437   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1438   // MemBarVolatile. For example, at the end of a constructor which
 1439   // writes final/volatile fields we will see a MemBarRelease
 1440   // instruction and this needs a 'dmb ish' lest we risk the
 1441   // constructed object being visible without making the
 1442   // final/volatile field writes visible.
 1443   //
 1444   // n.b. the translation rules below which rely on detection of the
 1445   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1446   // If we see anything other than the signature configurations we
 1447   // always just translate the loads and stores to ldr<x> and str<x>
 1448   // and translate acquire, release and volatile membars to the
 1449   // relevant dmb instructions.
 1450   //
 1451 
 1452   // is_CAS(int opcode, bool maybe_volatile)
 1453   //
 1454   // return true if opcode is one of the possible CompareAndSwapX
 1455   // values otherwise false.
 1456 
 1457   bool is_CAS(int opcode, bool maybe_volatile)
 1458   {
 1459     switch(opcode) {
 1460       // We handle these
 1461     case Op_CompareAndSwapI:
 1462     case Op_CompareAndSwapL:
 1463     case Op_CompareAndSwapP:
 1464     case Op_CompareAndSwapN:
 1465     case Op_ShenandoahCompareAndSwapP:
 1466     case Op_ShenandoahCompareAndSwapN:
 1467     case Op_CompareAndSwapB:
 1468     case Op_CompareAndSwapS:
 1469     case Op_GetAndSetI:
 1470     case Op_GetAndSetL:
 1471     case Op_GetAndSetP:
 1472     case Op_GetAndSetN:
 1473     case Op_GetAndAddI:
 1474     case Op_GetAndAddL:
 1475       return true;
 1476     case Op_CompareAndExchangeI:
 1477     case Op_CompareAndExchangeN:
 1478     case Op_CompareAndExchangeB:
 1479     case Op_CompareAndExchangeS:
 1480     case Op_CompareAndExchangeL:
 1481     case Op_CompareAndExchangeP:
 1482     case Op_WeakCompareAndSwapB:
 1483     case Op_WeakCompareAndSwapS:
 1484     case Op_WeakCompareAndSwapI:
 1485     case Op_WeakCompareAndSwapL:
 1486     case Op_WeakCompareAndSwapP:
 1487     case Op_WeakCompareAndSwapN:
 1488     case Op_ShenandoahWeakCompareAndSwapP:
 1489     case Op_ShenandoahWeakCompareAndSwapN:
 1490     case Op_ShenandoahCompareAndExchangeP:
 1491     case Op_ShenandoahCompareAndExchangeN:
 1492       return maybe_volatile;
 1493     default:
 1494       return false;
 1495     }
 1496   }
 1497 
 1498   // helper to determine the maximum number of Phi nodes we may need to
 1499   // traverse when searching from a card mark membar for the merge mem
 1500   // feeding a trailing membar or vice versa
 1501 
 1502 // predicates controlling emit of ldr<x>/ldar<x>
 1503 
 1504 bool unnecessary_acquire(const Node *barrier)
 1505 {
 1506   assert(barrier->is_MemBar(), "expecting a membar");
 1507 
 1508   MemBarNode* mb = barrier->as_MemBar();
 1509 
 1510   if (mb->trailing_load()) {
 1511     return true;
 1512   }
 1513 
 1514   if (mb->trailing_load_store()) {
 1515     Node* load_store = mb->in(MemBarNode::Precedent);
 1516     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1517     return is_CAS(load_store->Opcode(), true);
 1518   }
 1519 
 1520   return false;
 1521 }
 1522 
 1523 bool needs_acquiring_load(const Node *n)
 1524 {
 1525   assert(n->is_Load(), "expecting a load");
 1526   LoadNode *ld = n->as_Load();
 1527   return ld->is_acquire();
 1528 }
 1529 
 1530 bool unnecessary_release(const Node *n)
 1531 {
 1532   assert((n->is_MemBar() &&
 1533           n->Opcode() == Op_MemBarRelease),
 1534          "expecting a release membar");
 1535 
 1536   MemBarNode *barrier = n->as_MemBar();
 1537   if (!barrier->leading()) {
 1538     return false;
 1539   } else {
 1540     Node* trailing = barrier->trailing_membar();
 1541     MemBarNode* trailing_mb = trailing->as_MemBar();
 1542     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1543     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1544 
 1545     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1546     if (mem->is_Store()) {
 1547       assert(mem->as_Store()->is_release(), "");
 1548       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1549       return true;
 1550     } else {
 1551       assert(mem->is_LoadStore(), "");
 1552       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1553       return is_CAS(mem->Opcode(), true);
 1554     }
 1555   }
 1556   return false;
 1557 }
 1558 
 1559 bool unnecessary_volatile(const Node *n)
 1560 {
 1561   // assert n->is_MemBar();
 1562   MemBarNode *mbvol = n->as_MemBar();
 1563 
 1564   bool release = mbvol->trailing_store();
 1565   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1566 #ifdef ASSERT
 1567   if (release) {
 1568     Node* leading = mbvol->leading_membar();
 1569     assert(leading->Opcode() == Op_MemBarRelease, "");
 1570     assert(leading->as_MemBar()->leading_store(), "");
 1571     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1572   }
 1573 #endif
 1574 
 1575   return release;
 1576 }
 1577 
 1578 // predicates controlling emit of str<x>/stlr<x>
 1579 
 1580 bool needs_releasing_store(const Node *n)
 1581 {
 1582   // assert n->is_Store();
 1583   StoreNode *st = n->as_Store();
 1584   return st->trailing_membar() != nullptr;
 1585 }
 1586 
 1587 // predicate controlling translation of CAS
 1588 //
 1589 // returns true if CAS needs to use an acquiring load otherwise false
 1590 
 1591 bool needs_acquiring_load_exclusive(const Node *n)
 1592 {
 1593   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1594   LoadStoreNode* ldst = n->as_LoadStore();
 1595   if (is_CAS(n->Opcode(), false)) {
 1596     assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
 1597   } else {
 1598     return ldst->trailing_membar() != nullptr;
 1599   }
 1600 
 1601   // so we can just return true here
 1602   return true;
 1603 }
 1604 
 1605 #define __ _masm.
 1606 
 1607 // advance declarations for helper functions to convert register
 1608 // indices to register objects
 1609 
 1610 // the ad file has to provide implementations of certain methods
 1611 // expected by the generic code
 1612 //
 1613 // REQUIRED FUNCTIONALITY
 1614 
 1615 //=============================================================================
 1616 
 1617 // !!!!! Special hack to get all types of calls to specify the byte offset
 1618 //       from the start of the call to the point where the return address
 1619 //       will point.
 1620 
 1621 int MachCallStaticJavaNode::ret_addr_offset()
 1622 {
 1623   // call should be a simple bl
 1624   int off = 4;
 1625   return off;
 1626 }
 1627 
 1628 int MachCallDynamicJavaNode::ret_addr_offset()
 1629 {
 1630   return 16; // movz, movk, movk, bl
 1631 }
 1632 
 1633 int MachCallRuntimeNode::ret_addr_offset() {
 1634   // for generated stubs the call will be
 1635   //   bl(addr)
 1636   // or with far branches
 1637   //   bl(trampoline_stub)
 1638   // for real runtime callouts it will be six instructions
 1639   // see aarch64_enc_java_to_runtime
 1640   //   adr(rscratch2, retaddr)
 1641   //   lea(rscratch1, RuntimeAddress(addr)
 1642   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1643   //   blr(rscratch1)
 1644   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1645   if (cb) {
 1646     return 1 * NativeInstruction::instruction_size;
 1647   } else {
 1648     return 6 * NativeInstruction::instruction_size;
 1649   }
 1650 }
 1651 
 1652 //=============================================================================
 1653 
 1654 #ifndef PRODUCT
 1655 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1656   st->print("BREAKPOINT");
 1657 }
 1658 #endif
 1659 
 1660 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1661   C2_MacroAssembler _masm(&cbuf);
 1662   __ brk(0);
 1663 }
 1664 
 1665 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1666   return MachNode::size(ra_);
 1667 }
 1668 
 1669 //=============================================================================
 1670 
 1671 #ifndef PRODUCT
 1672   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1673     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1674   }
 1675 #endif
 1676 
 1677   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1678     C2_MacroAssembler _masm(&cbuf);
 1679     for (int i = 0; i < _count; i++) {
 1680       __ nop();
 1681     }
 1682   }
 1683 
 1684   uint MachNopNode::size(PhaseRegAlloc*) const {
 1685     return _count * NativeInstruction::instruction_size;
 1686   }
 1687 
 1688 //=============================================================================
 1689 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1690 
 1691 int ConstantTable::calculate_table_base_offset() const {
 1692   return 0;  // absolute addressing, no offset
 1693 }
 1694 
 1695 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1696 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1697   ShouldNotReachHere();
 1698 }
 1699 
 1700 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1701   // Empty encoding
 1702 }
 1703 
 1704 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1705   return 0;
 1706 }
 1707 
 1708 #ifndef PRODUCT
 1709 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1710   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1711 }
 1712 #endif
 1713 
 1714 #ifndef PRODUCT
 1715 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1716   Compile* C = ra_->C;
 1717 
 1718   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1719 
 1720   if (C->output()->need_stack_bang(framesize))
 1721     st->print("# stack bang size=%d\n\t", framesize);
 1722 
 1723   if (VM_Version::use_rop_protection()) {
 1724     st->print("ldr  zr, [lr]\n\t");
 1725     st->print("paciaz\n\t");
 1726   }
 1727   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1728     st->print("sub  sp, sp, #%d\n\t", framesize);
 1729     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1730     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1731   } else {
 1732     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1733     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1734     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1735     st->print("sub  sp, sp, rscratch1");
 1736   }
 1737   if (C->stub_function() == nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
 1738     st->print("\n\t");
 1739     st->print("ldr  rscratch1, [guard]\n\t");
 1740     st->print("dmb ishld\n\t");
 1741     st->print("ldr  rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
 1742     st->print("cmp  rscratch1, rscratch2\n\t");
 1743     st->print("b.eq skip");
 1744     st->print("\n\t");
 1745     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1746     st->print("b skip\n\t");
 1747     st->print("guard: int\n\t");
 1748     st->print("\n\t");
 1749     st->print("skip:\n\t");
 1750   }
 1751 }
 1752 #endif
 1753 
 1754 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1755   Compile* C = ra_->C;
 1756   C2_MacroAssembler _masm(&cbuf);
 1757 
 1758   // n.b. frame size includes space for return pc and rfp
 1759   const int framesize = C->output()->frame_size_in_bytes();
 1760 
 1761   // insert a nop at the start of the prolog so we can patch in a
 1762   // branch if we need to invalidate the method later
 1763   __ nop();
 1764 
 1765   if (C->clinit_barrier_on_entry()) {
 1766     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1767 
 1768     Label L_skip_barrier;
 1769 
 1770     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1771     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1772     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1773     __ bind(L_skip_barrier);
 1774   }
 1775 
 1776   if (C->max_vector_size() > 0) {
 1777     __ reinitialize_ptrue();
 1778   }
 1779 
 1780   int bangsize = C->output()->bang_size_in_bytes();
 1781   if (C->output()->need_stack_bang(bangsize))
 1782     __ generate_stack_overflow_check(bangsize);
 1783 
 1784   __ build_frame(framesize);
 1785 
 1786   if (C->stub_function() == nullptr) {
 1787     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1788     if (BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
 1789       // Dummy labels for just measuring the code size
 1790       Label dummy_slow_path;
 1791       Label dummy_continuation;
 1792       Label dummy_guard;
 1793       Label* slow_path = &dummy_slow_path;
 1794       Label* continuation = &dummy_continuation;
 1795       Label* guard = &dummy_guard;
 1796       if (!Compile::current()->output()->in_scratch_emit_size()) {
 1797         // Use real labels from actual stub when not emitting code for the purpose of measuring its size
 1798         C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
 1799         Compile::current()->output()->add_stub(stub);
 1800         slow_path = &stub->entry();
 1801         continuation = &stub->continuation();
 1802         guard = &stub->guard();
 1803       }
 1804       // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
 1805       bs->nmethod_entry_barrier(&_masm, slow_path, continuation, guard);
 1806     }
 1807   }
 1808 
 1809   if (VerifyStackAtCalls) {
 1810     Unimplemented();
 1811   }
 1812 
 1813   C->output()->set_frame_complete(cbuf.insts_size());
 1814 
 1815   if (C->has_mach_constant_base_node()) {
 1816     // NOTE: We set the table base offset here because users might be
 1817     // emitted before MachConstantBaseNode.
 1818     ConstantTable& constant_table = C->output()->constant_table();
 1819     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1820   }
 1821 }
 1822 
 1823 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1824 {
 1825   return MachNode::size(ra_); // too many variables; just compute it
 1826                               // the hard way
 1827 }
 1828 
 1829 int MachPrologNode::reloc() const
 1830 {
 1831   return 0;
 1832 }
 1833 
 1834 //=============================================================================
 1835 
 1836 #ifndef PRODUCT
 1837 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1838   Compile* C = ra_->C;
 1839   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1840 
 1841   st->print("# pop frame %d\n\t",framesize);
 1842 
 1843   if (framesize == 0) {
 1844     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1845   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1846     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1847     st->print("add  sp, sp, #%d\n\t", framesize);
 1848   } else {
 1849     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1850     st->print("add  sp, sp, rscratch1\n\t");
 1851     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1852   }
 1853   if (VM_Version::use_rop_protection()) {
 1854     st->print("autiaz\n\t");
 1855     st->print("ldr  zr, [lr]\n\t");
 1856   }
 1857 
 1858   if (do_polling() && C->is_method_compilation()) {
 1859     st->print("# test polling word\n\t");
 1860     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1861     st->print("cmp  sp, rscratch1\n\t");
 1862     st->print("bhi #slow_path");
 1863   }
 1864 }
 1865 #endif
 1866 
 1867 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1868   Compile* C = ra_->C;
 1869   C2_MacroAssembler _masm(&cbuf);
 1870   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1871 
 1872   __ remove_frame(framesize);
 1873 
 1874   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1875     __ reserved_stack_check();
 1876   }
 1877 
 1878   if (do_polling() && C->is_method_compilation()) {
 1879     Label dummy_label;
 1880     Label* code_stub = &dummy_label;
 1881     if (!C->output()->in_scratch_emit_size()) {
 1882       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
 1883       C->output()->add_stub(stub);
 1884       code_stub = &stub->entry();
 1885     }
 1886     __ relocate(relocInfo::poll_return_type);
 1887     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1888   }
 1889 }
 1890 
 1891 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 1892   // Variable size. Determine dynamically.
 1893   return MachNode::size(ra_);
 1894 }
 1895 
 1896 int MachEpilogNode::reloc() const {
 1897   // Return number of relocatable values contained in this instruction.
 1898   return 1; // 1 for polling page.
 1899 }
 1900 
 1901 const Pipeline * MachEpilogNode::pipeline() const {
 1902   return MachNode::pipeline_class();
 1903 }
 1904 
 1905 //=============================================================================
 1906 
 1907 // Figure out which register class each belongs in: rc_int, rc_float or
 1908 // rc_stack.
 1909 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 1910 
 1911 static enum RC rc_class(OptoReg::Name reg) {
 1912 
 1913   if (reg == OptoReg::Bad) {
 1914     return rc_bad;
 1915   }
 1916 
 1917   // we have 32 int registers * 2 halves
 1918   int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
 1919 
 1920   if (reg < slots_of_int_registers) {
 1921     return rc_int;
 1922   }
 1923 
 1924   // we have 32 float register * 8 halves
 1925   int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
 1926   if (reg < slots_of_int_registers + slots_of_float_registers) {
 1927     return rc_float;
 1928   }
 1929 
 1930   int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
 1931   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 1932     return rc_predicate;
 1933   }
 1934 
 1935   // Between predicate regs & stack is the flags.
 1936   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 1937 
 1938   return rc_stack;
 1939 }
 1940 
 1941 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 1942   Compile* C = ra_->C;
 1943 
 1944   // Get registers to move.
 1945   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 1946   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 1947   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 1948   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 1949 
 1950   enum RC src_hi_rc = rc_class(src_hi);
 1951   enum RC src_lo_rc = rc_class(src_lo);
 1952   enum RC dst_hi_rc = rc_class(dst_hi);
 1953   enum RC dst_lo_rc = rc_class(dst_lo);
 1954 
 1955   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 1956 
 1957   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 1958     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 1959            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 1960            "expected aligned-adjacent pairs");
 1961   }
 1962 
 1963   if (src_lo == dst_lo && src_hi == dst_hi) {
 1964     return 0;            // Self copy, no move.
 1965   }
 1966 
 1967   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 1968               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 1969   int src_offset = ra_->reg2offset(src_lo);
 1970   int dst_offset = ra_->reg2offset(dst_lo);
 1971 
 1972   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 1973     uint ireg = ideal_reg();
 1974     if (ireg == Op_VecA && cbuf) {
 1975       C2_MacroAssembler _masm(cbuf);
 1976       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 1977       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1978         // stack->stack
 1979         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 1980                                                 sve_vector_reg_size_in_bytes);
 1981       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 1982         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 1983                             sve_vector_reg_size_in_bytes);
 1984       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 1985         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 1986                               sve_vector_reg_size_in_bytes);
 1987       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 1988         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1989                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 1990                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 1991       } else {
 1992         ShouldNotReachHere();
 1993       }
 1994     } else if (cbuf) {
 1995       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 1996       C2_MacroAssembler _masm(cbuf);
 1997       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 1998       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1999         // stack->stack
 2000         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2001         if (ireg == Op_VecD) {
 2002           __ unspill(rscratch1, true, src_offset);
 2003           __ spill(rscratch1, true, dst_offset);
 2004         } else {
 2005           __ spill_copy128(src_offset, dst_offset);
 2006         }
 2007       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2008         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2009                ireg == Op_VecD ? __ T8B : __ T16B,
 2010                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2011       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2012         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2013                  ireg == Op_VecD ? __ D : __ Q,
 2014                  ra_->reg2offset(dst_lo));
 2015       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2016         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2017                    ireg == Op_VecD ? __ D : __ Q,
 2018                    ra_->reg2offset(src_lo));
 2019       } else {
 2020         ShouldNotReachHere();
 2021       }
 2022     }
 2023   } else if (cbuf) {
 2024     C2_MacroAssembler _masm(cbuf);
 2025     switch (src_lo_rc) {
 2026     case rc_int:
 2027       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2028         if (is64) {
 2029             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2030                    as_Register(Matcher::_regEncode[src_lo]));
 2031         } else {
 2032             C2_MacroAssembler _masm(cbuf);
 2033             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2034                     as_Register(Matcher::_regEncode[src_lo]));
 2035         }
 2036       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2037         if (is64) {
 2038             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2039                      as_Register(Matcher::_regEncode[src_lo]));
 2040         } else {
 2041             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2042                      as_Register(Matcher::_regEncode[src_lo]));
 2043         }
 2044       } else {                    // gpr --> stack spill
 2045         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2046         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2047       }
 2048       break;
 2049     case rc_float:
 2050       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2051         if (is64) {
 2052             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2053                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2054         } else {
 2055             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2056                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2057         }
 2058       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2059         if (is64) {
 2060             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2061                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2062         } else {
 2063             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2064                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2065         }
 2066       } else {                    // fpr --> stack spill
 2067         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2068         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2069                  is64 ? __ D : __ S, dst_offset);
 2070       }
 2071       break;
 2072     case rc_stack:
 2073       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2074         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2075       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2076         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2077                    is64 ? __ D : __ S, src_offset);
 2078       } else if (dst_lo_rc == rc_predicate) {
 2079         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2080                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2081       } else {                    // stack --> stack copy
 2082         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2083         if (ideal_reg() == Op_RegVectMask) {
 2084           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2085                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2086         } else {
 2087           __ unspill(rscratch1, is64, src_offset);
 2088           __ spill(rscratch1, is64, dst_offset);
 2089         }
 2090       }
 2091       break;
 2092     case rc_predicate:
 2093       if (dst_lo_rc == rc_predicate) {
 2094         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2095       } else if (dst_lo_rc == rc_stack) {
 2096         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2097                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2098       } else {
 2099         assert(false, "bad src and dst rc_class combination.");
 2100         ShouldNotReachHere();
 2101       }
 2102       break;
 2103     default:
 2104       assert(false, "bad rc_class for spill");
 2105       ShouldNotReachHere();
 2106     }
 2107   }
 2108 
 2109   if (st) {
 2110     st->print("spill ");
 2111     if (src_lo_rc == rc_stack) {
 2112       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2113     } else {
 2114       st->print("%s -> ", Matcher::regName[src_lo]);
 2115     }
 2116     if (dst_lo_rc == rc_stack) {
 2117       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2118     } else {
 2119       st->print("%s", Matcher::regName[dst_lo]);
 2120     }
 2121     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2122       int vsize = 0;
 2123       switch (ideal_reg()) {
 2124       case Op_VecD:
 2125         vsize = 64;
 2126         break;
 2127       case Op_VecX:
 2128         vsize = 128;
 2129         break;
 2130       case Op_VecA:
 2131         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2132         break;
 2133       default:
 2134         assert(false, "bad register type for spill");
 2135         ShouldNotReachHere();
 2136       }
 2137       st->print("\t# vector spill size = %d", vsize);
 2138     } else if (ideal_reg() == Op_RegVectMask) {
 2139       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2140       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2141       st->print("\t# predicate spill size = %d", vsize);
 2142     } else {
 2143       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2144     }
 2145   }
 2146 
 2147   return 0;
 2148 
 2149 }
 2150 
 2151 #ifndef PRODUCT
 2152 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2153   if (!ra_)
 2154     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2155   else
 2156     implementation(nullptr, ra_, false, st);
 2157 }
 2158 #endif
 2159 
 2160 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2161   implementation(&cbuf, ra_, false, nullptr);
 2162 }
 2163 
 2164 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2165   return MachNode::size(ra_);
 2166 }
 2167 
 2168 //=============================================================================
 2169 
 2170 #ifndef PRODUCT
 2171 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2172   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2173   int reg = ra_->get_reg_first(this);
 2174   st->print("add %s, rsp, #%d]\t# box lock",
 2175             Matcher::regName[reg], offset);
 2176 }
 2177 #endif
 2178 
 2179 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2180   C2_MacroAssembler _masm(&cbuf);
 2181 
 2182   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2183   int reg    = ra_->get_encode(this);
 2184 
 2185   // This add will handle any 24-bit signed offset. 24 bits allows an
 2186   // 8 megabyte stack frame.
 2187   __ add(as_Register(reg), sp, offset);
 2188 }
 2189 
 2190 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2191   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2192   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2193 
 2194   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2195     return NativeInstruction::instruction_size;
 2196   } else {
 2197     return 2 * NativeInstruction::instruction_size;
 2198   }
 2199 }
 2200 
 2201 //=============================================================================
 2202 
 2203 #ifndef PRODUCT
 2204 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2205 {
 2206   st->print_cr("# MachUEPNode");
 2207   if (UseCompressedClassPointers) {
 2208     st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2209     st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
 2210     st->print_cr("\tcmpw rscratch1, r10");
 2211   } else {
 2212     st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2213     st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
 2214     st->print_cr("\tcmp rscratch1, r10");
 2215   }
 2216   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2217 }
 2218 #endif
 2219 
 2220 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2221 {
 2222   // This is the unverified entry point.
 2223   C2_MacroAssembler _masm(&cbuf);
 2224   __ ic_check(InteriorEntryAlignment);
 2225 }
 2226 
 2227 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2228 {
 2229   return MachNode::size(ra_);
 2230 }
 2231 
 2232 // REQUIRED EMIT CODE
 2233 
 2234 //=============================================================================
 2235 
 2236 // Emit exception handler code.
 2237 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2238 {
 2239   // mov rscratch1 #exception_blob_entry_point
 2240   // br rscratch1
 2241   // Note that the code buffer's insts_mark is always relative to insts.
 2242   // That's why we must use the macroassembler to generate a handler.
 2243   C2_MacroAssembler _masm(&cbuf);
 2244   address base = __ start_a_stub(size_exception_handler());
 2245   if (base == nullptr) {
 2246     ciEnv::current()->record_failure("CodeCache is full");
 2247     return 0;  // CodeBuffer::expand failed
 2248   }
 2249   int offset = __ offset();
 2250   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2251   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2252   __ end_a_stub();
 2253   return offset;
 2254 }
 2255 
 2256 // Emit deopt handler code.
 2257 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2258 {
 2259   // Note that the code buffer's insts_mark is always relative to insts.
 2260   // That's why we must use the macroassembler to generate a handler.
 2261   C2_MacroAssembler _masm(&cbuf);
 2262   address base = __ start_a_stub(size_deopt_handler());
 2263   if (base == nullptr) {
 2264     ciEnv::current()->record_failure("CodeCache is full");
 2265     return 0;  // CodeBuffer::expand failed
 2266   }
 2267   int offset = __ offset();
 2268 
 2269   __ adr(lr, __ pc());
 2270   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2271 
 2272   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2273   __ end_a_stub();
 2274   return offset;
 2275 }
 2276 
 2277 // REQUIRED MATCHER CODE
 2278 
 2279 //=============================================================================
 2280 
 2281 bool Matcher::match_rule_supported(int opcode) {
 2282   if (!has_match_rule(opcode))
 2283     return false;
 2284 
 2285   switch (opcode) {
 2286     case Op_OnSpinWait:
 2287       return VM_Version::supports_on_spin_wait();
 2288     case Op_CacheWB:
 2289     case Op_CacheWBPreSync:
 2290     case Op_CacheWBPostSync:
 2291       if (!VM_Version::supports_data_cache_line_flush()) {
 2292         return false;
 2293       }
 2294       break;
 2295     case Op_ExpandBits:
 2296     case Op_CompressBits:
 2297       if (!VM_Version::supports_svebitperm()) {
 2298         return false;
 2299       }
 2300       break;
 2301     case Op_FmaF:
 2302     case Op_FmaD:
 2303     case Op_FmaVF:
 2304     case Op_FmaVD:
 2305       if (!UseFMA) {
 2306         return false;
 2307       }
 2308       break;
 2309   }
 2310 
 2311   return true; // Per default match rules are supported.
 2312 }
 2313 
 2314 const RegMask* Matcher::predicate_reg_mask(void) {
 2315   return &_PR_REG_mask;
 2316 }
 2317 
 2318 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2319   return new TypeVectMask(elemTy, length);
 2320 }
 2321 
 2322 // Vector calling convention not yet implemented.
 2323 bool Matcher::supports_vector_calling_convention(void) {
 2324   return false;
 2325 }
 2326 
 2327 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2328   Unimplemented();
 2329   return OptoRegPair(0, 0);
 2330 }
 2331 
 2332 // Is this branch offset short enough that a short branch can be used?
 2333 //
 2334 // NOTE: If the platform does not provide any short branch variants, then
 2335 //       this method should return false for offset 0.
 2336 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2337   // The passed offset is relative to address of the branch.
 2338 
 2339   return (-32768 <= offset && offset < 32768);
 2340 }
 2341 
 2342 // Vector width in bytes.
 2343 int Matcher::vector_width_in_bytes(BasicType bt) {
 2344   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2345   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2346   // Minimum 2 values in vector
 2347   if (size < 2*type2aelembytes(bt)) size = 0;
 2348   // But never < 4
 2349   if (size < 4) size = 0;
 2350   return size;
 2351 }
 2352 
 2353 // Limits on vector size (number of elements) loaded into vector.
 2354 int Matcher::max_vector_size(const BasicType bt) {
 2355   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2356 }
 2357 
 2358 int Matcher::min_vector_size(const BasicType bt) {
 2359   int max_size = max_vector_size(bt);
 2360   // Limit the min vector size to 8 bytes.
 2361   int size = 8 / type2aelembytes(bt);
 2362   if (bt == T_BYTE) {
 2363     // To support vector api shuffle/rearrange.
 2364     size = 4;
 2365   } else if (bt == T_BOOLEAN) {
 2366     // To support vector api load/store mask.
 2367     size = 2;
 2368   }
 2369   if (size < 2) size = 2;
 2370   return MIN2(size, max_size);
 2371 }
 2372 
 2373 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
 2374   return Matcher::max_vector_size(bt);
 2375 }
 2376 
 2377 // Actual max scalable vector register length.
 2378 int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2379   return Matcher::max_vector_size(bt);
 2380 }
 2381 
 2382 // Vector ideal reg.
 2383 uint Matcher::vector_ideal_reg(int len) {
 2384   if (UseSVE > 0 && 16 < len && len <= 256) {
 2385     return Op_VecA;
 2386   }
 2387   switch(len) {
 2388     // For 16-bit/32-bit mask vector, reuse VecD.
 2389     case  2:
 2390     case  4:
 2391     case  8: return Op_VecD;
 2392     case 16: return Op_VecX;
 2393   }
 2394   ShouldNotReachHere();
 2395   return 0;
 2396 }
 2397 
 2398 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
 2399   assert(Matcher::is_generic_vector(generic_opnd), "not generic");
 2400   switch (ideal_reg) {
 2401     case Op_VecA: return new vecAOper();
 2402     case Op_VecD: return new vecDOper();
 2403     case Op_VecX: return new vecXOper();
 2404   }
 2405   ShouldNotReachHere();
 2406   return nullptr;
 2407 }
 2408 
 2409 bool Matcher::is_reg2reg_move(MachNode* m) {
 2410   return false;
 2411 }
 2412 
 2413 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2414   return opnd->opcode() == VREG;
 2415 }
 2416 
 2417 // Return whether or not this register is ever used as an argument.
 2418 // This function is used on startup to build the trampoline stubs in
 2419 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2420 // call in the trampoline, and arguments in those registers not be
 2421 // available to the callee.
 2422 bool Matcher::can_be_java_arg(int reg)
 2423 {
 2424   return
 2425     reg ==  R0_num || reg == R0_H_num ||
 2426     reg ==  R1_num || reg == R1_H_num ||
 2427     reg ==  R2_num || reg == R2_H_num ||
 2428     reg ==  R3_num || reg == R3_H_num ||
 2429     reg ==  R4_num || reg == R4_H_num ||
 2430     reg ==  R5_num || reg == R5_H_num ||
 2431     reg ==  R6_num || reg == R6_H_num ||
 2432     reg ==  R7_num || reg == R7_H_num ||
 2433     reg ==  V0_num || reg == V0_H_num ||
 2434     reg ==  V1_num || reg == V1_H_num ||
 2435     reg ==  V2_num || reg == V2_H_num ||
 2436     reg ==  V3_num || reg == V3_H_num ||
 2437     reg ==  V4_num || reg == V4_H_num ||
 2438     reg ==  V5_num || reg == V5_H_num ||
 2439     reg ==  V6_num || reg == V6_H_num ||
 2440     reg ==  V7_num || reg == V7_H_num;
 2441 }
 2442 
 2443 bool Matcher::is_spillable_arg(int reg)
 2444 {
 2445   return can_be_java_arg(reg);
 2446 }
 2447 
 2448 uint Matcher::int_pressure_limit()
 2449 {
 2450   // JDK-8183543: When taking the number of available registers as int
 2451   // register pressure threshold, the jtreg test:
 2452   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2453   // failed due to C2 compilation failure with
 2454   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2455   //
 2456   // A derived pointer is live at CallNode and then is flagged by RA
 2457   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2458   // derived pointers and lastly fail to spill after reaching maximum
 2459   // number of iterations. Lowering the default pressure threshold to
 2460   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2461   // a high register pressure area of the code so that split_DEF can
 2462   // generate DefinitionSpillCopy for the derived pointer.
 2463   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2464   if (!PreserveFramePointer) {
 2465     // When PreserveFramePointer is off, frame pointer is allocatable,
 2466     // but different from other SOC registers, it is excluded from
 2467     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2468     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2469     // See check_pressure_at_fatproj().
 2470     default_int_pressure_threshold--;
 2471   }
 2472   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2473 }
 2474 
 2475 uint Matcher::float_pressure_limit()
 2476 {
 2477   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2478   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2479 }
 2480 
 2481 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2482   return false;
 2483 }
 2484 
 2485 RegMask Matcher::divI_proj_mask() {
 2486   ShouldNotReachHere();
 2487   return RegMask();
 2488 }
 2489 
 2490 // Register for MODI projection of divmodI.
 2491 RegMask Matcher::modI_proj_mask() {
 2492   ShouldNotReachHere();
 2493   return RegMask();
 2494 }
 2495 
 2496 // Register for DIVL projection of divmodL.
 2497 RegMask Matcher::divL_proj_mask() {
 2498   ShouldNotReachHere();
 2499   return RegMask();
 2500 }
 2501 
 2502 // Register for MODL projection of divmodL.
 2503 RegMask Matcher::modL_proj_mask() {
 2504   ShouldNotReachHere();
 2505   return RegMask();
 2506 }
 2507 
 2508 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2509   return FP_REG_mask();
 2510 }
 2511 
 2512 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2513   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2514     Node* u = addp->fast_out(i);
 2515     if (u->is_LoadStore()) {
 2516       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2517       // instructions) only take register indirect as an operand, so
 2518       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2519       // must fail.
 2520       return false;
 2521     }
 2522     if (u->is_Mem()) {
 2523       int opsize = u->as_Mem()->memory_size();
 2524       assert(opsize > 0, "unexpected memory operand size");
 2525       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2526         return false;
 2527       }
 2528     }
 2529   }
 2530   return true;
 2531 }
 2532 
 2533 // Convert BootTest condition to Assembler condition.
 2534 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 2535 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
 2536   Assembler::Condition result;
 2537   switch(cond) {
 2538     case BoolTest::eq:
 2539       result = Assembler::EQ; break;
 2540     case BoolTest::ne:
 2541       result = Assembler::NE; break;
 2542     case BoolTest::le:
 2543       result = Assembler::LE; break;
 2544     case BoolTest::ge:
 2545       result = Assembler::GE; break;
 2546     case BoolTest::lt:
 2547       result = Assembler::LT; break;
 2548     case BoolTest::gt:
 2549       result = Assembler::GT; break;
 2550     case BoolTest::ule:
 2551       result = Assembler::LS; break;
 2552     case BoolTest::uge:
 2553       result = Assembler::HS; break;
 2554     case BoolTest::ult:
 2555       result = Assembler::LO; break;
 2556     case BoolTest::ugt:
 2557       result = Assembler::HI; break;
 2558     case BoolTest::overflow:
 2559       result = Assembler::VS; break;
 2560     case BoolTest::no_overflow:
 2561       result = Assembler::VC; break;
 2562     default:
 2563       ShouldNotReachHere();
 2564       return Assembler::Condition(-1);
 2565   }
 2566 
 2567   // Check conversion
 2568   if (cond & BoolTest::unsigned_compare) {
 2569     assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
 2570   } else {
 2571     assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
 2572   }
 2573 
 2574   return result;
 2575 }
 2576 
 2577 // Binary src (Replicate con)
 2578 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
 2579   if (n == nullptr || m == nullptr) {
 2580     return false;
 2581   }
 2582 
 2583   if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
 2584     return false;
 2585   }
 2586 
 2587   Node* imm_node = m->in(1);
 2588   if (!imm_node->is_Con()) {
 2589     return false;
 2590   }
 2591 
 2592   const Type* t = imm_node->bottom_type();
 2593   if (!(t->isa_int() || t->isa_long())) {
 2594     return false;
 2595   }
 2596 
 2597   switch (n->Opcode()) {
 2598   case Op_AndV:
 2599   case Op_OrV:
 2600   case Op_XorV: {
 2601     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
 2602     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2603     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2604   }
 2605   case Op_AddVB:
 2606     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2607   case Op_AddVS:
 2608   case Op_AddVI:
 2609     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2610   case Op_AddVL:
 2611     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2612   default:
 2613     return false;
 2614   }
 2615 }
 2616 
 2617 // (XorV src (Replicate m1))
 2618 // (XorVMask src (MaskAll m1))
 2619 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
 2620   if (n != nullptr && m != nullptr) {
 2621     return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
 2622            VectorNode::is_all_ones_vector(m);
 2623   }
 2624   return false;
 2625 }
 2626 
 2627 // Should the matcher clone input 'm' of node 'n'?
 2628 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2629   if (is_vshift_con_pattern(n, m) ||
 2630       is_vector_bitwise_not_pattern(n, m) ||
 2631       is_valid_sve_arith_imm_pattern(n, m)) {
 2632     mstack.push(m, Visit);
 2633     return true;
 2634   }
 2635   return false;
 2636 }
 2637 
 2638 // Should the Matcher clone shifts on addressing modes, expecting them
 2639 // to be subsumed into complex addressing expressions or compute them
 2640 // into registers?
 2641 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2642   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2643     return true;
 2644   }
 2645 
 2646   Node *off = m->in(AddPNode::Offset);
 2647   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2648       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2649       // Are there other uses besides address expressions?
 2650       !is_visited(off)) {
 2651     address_visited.set(off->_idx); // Flag as address_visited
 2652     mstack.push(off->in(2), Visit);
 2653     Node *conv = off->in(1);
 2654     if (conv->Opcode() == Op_ConvI2L &&
 2655         // Are there other uses besides address expressions?
 2656         !is_visited(conv)) {
 2657       address_visited.set(conv->_idx); // Flag as address_visited
 2658       mstack.push(conv->in(1), Pre_Visit);
 2659     } else {
 2660       mstack.push(conv, Pre_Visit);
 2661     }
 2662     address_visited.test_set(m->_idx); // Flag as address_visited
 2663     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2664     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2665     return true;
 2666   } else if (off->Opcode() == Op_ConvI2L &&
 2667              // Are there other uses besides address expressions?
 2668              !is_visited(off)) {
 2669     address_visited.test_set(m->_idx); // Flag as address_visited
 2670     address_visited.set(off->_idx); // Flag as address_visited
 2671     mstack.push(off->in(1), Pre_Visit);
 2672     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2673     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2674     return true;
 2675   }
 2676   return false;
 2677 }
 2678 
 2679 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2680   C2_MacroAssembler _masm(&cbuf);                                       \
 2681   {                                                                     \
 2682     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2683     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2684     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2685     __ INSN(REG, as_Register(BASE));                                    \
 2686   }
 2687 
 2688 
 2689 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2690   {
 2691     Address::extend scale;
 2692 
 2693     // Hooboy, this is fugly.  We need a way to communicate to the
 2694     // encoder that the index needs to be sign extended, so we have to
 2695     // enumerate all the cases.
 2696     switch (opcode) {
 2697     case INDINDEXSCALEDI2L:
 2698     case INDINDEXSCALEDI2LN:
 2699     case INDINDEXI2L:
 2700     case INDINDEXI2LN:
 2701       scale = Address::sxtw(size);
 2702       break;
 2703     default:
 2704       scale = Address::lsl(size);
 2705     }
 2706 
 2707     if (index == -1) {
 2708       return Address(base, disp);
 2709     } else {
 2710       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2711       return Address(base, as_Register(index), scale);
 2712     }
 2713   }
 2714 
 2715 
 2716 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2717 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2718 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2719 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2720                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2721 
 2722   // Used for all non-volatile memory accesses.  The use of
 2723   // $mem->opcode() to discover whether this pattern uses sign-extended
 2724   // offsets is something of a kludge.
 2725   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2726                         Register reg, int opcode,
 2727                         Register base, int index, int scale, int disp,
 2728                         int size_in_memory)
 2729   {
 2730     Address addr = mem2address(opcode, base, index, scale, disp);
 2731     if (addr.getMode() == Address::base_plus_offset) {
 2732       /* If we get an out-of-range offset it is a bug in the compiler,
 2733          so we assert here. */
 2734       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2735              "c2 compiler bug");
 2736       /* Fix up any out-of-range offsets. */
 2737       assert_different_registers(rscratch1, base);
 2738       assert_different_registers(rscratch1, reg);
 2739       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2740     }
 2741     (masm.*insn)(reg, addr);
 2742   }
 2743 
 2744   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2745                         FloatRegister reg, int opcode,
 2746                         Register base, int index, int size, int disp,
 2747                         int size_in_memory)
 2748   {
 2749     Address::extend scale;
 2750 
 2751     switch (opcode) {
 2752     case INDINDEXSCALEDI2L:
 2753     case INDINDEXSCALEDI2LN:
 2754       scale = Address::sxtw(size);
 2755       break;
 2756     default:
 2757       scale = Address::lsl(size);
 2758     }
 2759 
 2760     if (index == -1) {
 2761       /* If we get an out-of-range offset it is a bug in the compiler,
 2762          so we assert here. */
 2763       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2764       /* Fix up any out-of-range offsets. */
 2765       assert_different_registers(rscratch1, base);
 2766       Address addr = Address(base, disp);
 2767       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2768       (masm.*insn)(reg, addr);
 2769     } else {
 2770       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2771       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2772     }
 2773   }
 2774 
 2775   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2776                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2777                         int opcode, Register base, int index, int size, int disp)
 2778   {
 2779     if (index == -1) {
 2780       (masm.*insn)(reg, T, Address(base, disp));
 2781     } else {
 2782       assert(disp == 0, "unsupported address mode");
 2783       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2784     }
 2785   }
 2786 
 2787 %}
 2788 
 2789 
 2790 
 2791 //----------ENCODING BLOCK-----------------------------------------------------
 2792 // This block specifies the encoding classes used by the compiler to
 2793 // output byte streams.  Encoding classes are parameterized macros
 2794 // used by Machine Instruction Nodes in order to generate the bit
 2795 // encoding of the instruction.  Operands specify their base encoding
 2796 // interface with the interface keyword.  There are currently
 2797 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2798 // COND_INTER.  REG_INTER causes an operand to generate a function
 2799 // which returns its register number when queried.  CONST_INTER causes
 2800 // an operand to generate a function which returns the value of the
 2801 // constant when queried.  MEMORY_INTER causes an operand to generate
 2802 // four functions which return the Base Register, the Index Register,
 2803 // the Scale Value, and the Offset Value of the operand when queried.
 2804 // COND_INTER causes an operand to generate six functions which return
 2805 // the encoding code (ie - encoding bits for the instruction)
 2806 // associated with each basic boolean condition for a conditional
 2807 // instruction.
 2808 //
 2809 // Instructions specify two basic values for encoding.  Again, a
 2810 // function is available to check if the constant displacement is an
 2811 // oop. They use the ins_encode keyword to specify their encoding
 2812 // classes (which must be a sequence of enc_class names, and their
 2813 // parameters, specified in the encoding block), and they use the
 2814 // opcode keyword to specify, in order, their primary, secondary, and
 2815 // tertiary opcode.  Only the opcode sections which a particular
 2816 // instruction needs for encoding need to be specified.
 2817 encode %{
 2818   // Build emit functions for each basic byte or larger field in the
 2819   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2820   // from C++ code in the enc_class source block.  Emit functions will
 2821   // live in the main source block for now.  In future, we can
 2822   // generalize this by adding a syntax that specifies the sizes of
 2823   // fields in an order, so that the adlc can build the emit functions
 2824   // automagically
 2825 
 2826   // catch all for unimplemented encodings
 2827   enc_class enc_unimplemented %{
 2828     C2_MacroAssembler _masm(&cbuf);
 2829     __ unimplemented("C2 catch all");
 2830   %}
 2831 
 2832   // BEGIN Non-volatile memory access
 2833 
 2834   // This encoding class is generated automatically from ad_encode.m4.
 2835   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2836   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2837     Register dst_reg = as_Register($dst$$reg);
 2838     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2839                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2840   %}
 2841 
 2842   // This encoding class is generated automatically from ad_encode.m4.
 2843   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2844   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2845     Register dst_reg = as_Register($dst$$reg);
 2846     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2847                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2848   %}
 2849 
 2850   // This encoding class is generated automatically from ad_encode.m4.
 2851   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2852   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2853     Register dst_reg = as_Register($dst$$reg);
 2854     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2855                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2856   %}
 2857 
 2858   // This encoding class is generated automatically from ad_encode.m4.
 2859   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2860   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2861     Register dst_reg = as_Register($dst$$reg);
 2862     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2863                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2864   %}
 2865 
 2866   // This encoding class is generated automatically from ad_encode.m4.
 2867   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2868   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2869     Register dst_reg = as_Register($dst$$reg);
 2870     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2871                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2872   %}
 2873 
 2874   // This encoding class is generated automatically from ad_encode.m4.
 2875   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2876   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2877     Register dst_reg = as_Register($dst$$reg);
 2878     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2879                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2880   %}
 2881 
 2882   // This encoding class is generated automatically from ad_encode.m4.
 2883   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2884   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2885     Register dst_reg = as_Register($dst$$reg);
 2886     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2887                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2888   %}
 2889 
 2890   // This encoding class is generated automatically from ad_encode.m4.
 2891   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2892   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2893     Register dst_reg = as_Register($dst$$reg);
 2894     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2895                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2896   %}
 2897 
 2898   // This encoding class is generated automatically from ad_encode.m4.
 2899   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2900   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2901     Register dst_reg = as_Register($dst$$reg);
 2902     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2903                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2904   %}
 2905 
 2906   // This encoding class is generated automatically from ad_encode.m4.
 2907   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2908   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2909     Register dst_reg = as_Register($dst$$reg);
 2910     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2911                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2912   %}
 2913 
 2914   // This encoding class is generated automatically from ad_encode.m4.
 2915   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2916   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2917     Register dst_reg = as_Register($dst$$reg);
 2918     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2919                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2920   %}
 2921 
 2922   // This encoding class is generated automatically from ad_encode.m4.
 2923   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2924   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2925     Register dst_reg = as_Register($dst$$reg);
 2926     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2927                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2928   %}
 2929 
 2930   // This encoding class is generated automatically from ad_encode.m4.
 2931   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2932   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2933     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2934     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2935                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2936   %}
 2937 
 2938   // This encoding class is generated automatically from ad_encode.m4.
 2939   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2940   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 2941     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2942     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2943                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2944   %}
 2945 
 2946   // This encoding class is generated automatically from ad_encode.m4.
 2947   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2948   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 2949     Register src_reg = as_Register($src$$reg);
 2950     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 2951                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2952   %}
 2953 
 2954   // This encoding class is generated automatically from ad_encode.m4.
 2955   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2956   enc_class aarch64_enc_strb0(memory1 mem) %{
 2957     C2_MacroAssembler _masm(&cbuf);
 2958     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2959                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2960   %}
 2961 
 2962   // This encoding class is generated automatically from ad_encode.m4.
 2963   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2964   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 2965     Register src_reg = as_Register($src$$reg);
 2966     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 2967                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2968   %}
 2969 
 2970   // This encoding class is generated automatically from ad_encode.m4.
 2971   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2972   enc_class aarch64_enc_strh0(memory2 mem) %{
 2973     C2_MacroAssembler _masm(&cbuf);
 2974     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 2975                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2976   %}
 2977 
 2978   // This encoding class is generated automatically from ad_encode.m4.
 2979   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2980   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 2981     Register src_reg = as_Register($src$$reg);
 2982     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 2983                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2984   %}
 2985 
 2986   // This encoding class is generated automatically from ad_encode.m4.
 2987   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2988   enc_class aarch64_enc_strw0(memory4 mem) %{
 2989     C2_MacroAssembler _masm(&cbuf);
 2990     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 2991                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2992   %}
 2993 
 2994   // This encoding class is generated automatically from ad_encode.m4.
 2995   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2996   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 2997     Register src_reg = as_Register($src$$reg);
 2998     // we sometimes get asked to store the stack pointer into the
 2999     // current thread -- we cannot do that directly on AArch64
 3000     if (src_reg == r31_sp) {
 3001       C2_MacroAssembler _masm(&cbuf);
 3002       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3003       __ mov(rscratch2, sp);
 3004       src_reg = rscratch2;
 3005     }
 3006     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 3007                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3008   %}
 3009 
 3010   // This encoding class is generated automatically from ad_encode.m4.
 3011   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3012   enc_class aarch64_enc_str0(memory8 mem) %{
 3013     C2_MacroAssembler _masm(&cbuf);
 3014     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 3015                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3016   %}
 3017 
 3018   // This encoding class is generated automatically from ad_encode.m4.
 3019   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3020   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3021     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3022     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 3023                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3024   %}
 3025 
 3026   // This encoding class is generated automatically from ad_encode.m4.
 3027   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3028   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3029     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3030     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 3031                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3032   %}
 3033 
 3034   // This encoding class is generated automatically from ad_encode.m4.
 3035   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3036   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3037       C2_MacroAssembler _masm(&cbuf);
 3038       __ membar(Assembler::StoreStore);
 3039       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3040                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3041   %}
 3042 
 3043   // END Non-volatile memory access
 3044 
 3045   // Vector loads and stores
 3046   enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
 3047     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3048     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3049        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3050   %}
 3051 
 3052   enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
 3053     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3054     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3055        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3056   %}
 3057 
 3058   enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
 3059     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3060     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3061        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3062   %}
 3063 
 3064   enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
 3065     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3066     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3067        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3068   %}
 3069 
 3070   enc_class aarch64_enc_strvH(vReg src, memory mem) %{
 3071     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3072     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3073        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3074   %}
 3075 
 3076   enc_class aarch64_enc_strvS(vReg src, memory mem) %{
 3077     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3078     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3079        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3080   %}
 3081 
 3082   enc_class aarch64_enc_strvD(vReg src, memory mem) %{
 3083     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3084     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3085        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3086   %}
 3087 
 3088   enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
 3089     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3090     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3091        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3092   %}
 3093 
 3094   // volatile loads and stores
 3095 
 3096   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3097     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3098                  rscratch1, stlrb);
 3099   %}
 3100 
 3101   enc_class aarch64_enc_stlrb0(memory mem) %{
 3102     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3103                  rscratch1, stlrb);
 3104   %}
 3105 
 3106   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3107     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3108                  rscratch1, stlrh);
 3109   %}
 3110 
 3111   enc_class aarch64_enc_stlrh0(memory mem) %{
 3112     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3113                  rscratch1, stlrh);
 3114   %}
 3115 
 3116   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3117     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3118                  rscratch1, stlrw);
 3119   %}
 3120 
 3121   enc_class aarch64_enc_stlrw0(memory mem) %{
 3122     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3123                  rscratch1, stlrw);
 3124   %}
 3125 
 3126   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3127     Register dst_reg = as_Register($dst$$reg);
 3128     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3129              rscratch1, ldarb);
 3130     __ sxtbw(dst_reg, dst_reg);
 3131   %}
 3132 
 3133   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3134     Register dst_reg = as_Register($dst$$reg);
 3135     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3136              rscratch1, ldarb);
 3137     __ sxtb(dst_reg, dst_reg);
 3138   %}
 3139 
 3140   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3141     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3142              rscratch1, ldarb);
 3143   %}
 3144 
 3145   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3146     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3147              rscratch1, ldarb);
 3148   %}
 3149 
 3150   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3151     Register dst_reg = as_Register($dst$$reg);
 3152     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3153              rscratch1, ldarh);
 3154     __ sxthw(dst_reg, dst_reg);
 3155   %}
 3156 
 3157   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3158     Register dst_reg = as_Register($dst$$reg);
 3159     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3160              rscratch1, ldarh);
 3161     __ sxth(dst_reg, dst_reg);
 3162   %}
 3163 
 3164   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3165     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3166              rscratch1, ldarh);
 3167   %}
 3168 
 3169   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3170     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3171              rscratch1, ldarh);
 3172   %}
 3173 
 3174   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3175     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3176              rscratch1, ldarw);
 3177   %}
 3178 
 3179   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3180     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3181              rscratch1, ldarw);
 3182   %}
 3183 
 3184   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3185     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3186              rscratch1, ldar);
 3187   %}
 3188 
 3189   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3190     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3191              rscratch1, ldarw);
 3192     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3193   %}
 3194 
 3195   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3196     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3197              rscratch1, ldar);
 3198     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3199   %}
 3200 
 3201   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3202     Register src_reg = as_Register($src$$reg);
 3203     // we sometimes get asked to store the stack pointer into the
 3204     // current thread -- we cannot do that directly on AArch64
 3205     if (src_reg == r31_sp) {
 3206       C2_MacroAssembler _masm(&cbuf);
 3207       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3208       __ mov(rscratch2, sp);
 3209       src_reg = rscratch2;
 3210     }
 3211     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3212                  rscratch1, stlr);
 3213   %}
 3214 
 3215   enc_class aarch64_enc_stlr0(memory mem) %{
 3216     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3217                  rscratch1, stlr);
 3218   %}
 3219 
 3220   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3221     {
 3222       C2_MacroAssembler _masm(&cbuf);
 3223       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3224       __ fmovs(rscratch2, src_reg);
 3225     }
 3226     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3227                  rscratch1, stlrw);
 3228   %}
 3229 
 3230   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3231     {
 3232       C2_MacroAssembler _masm(&cbuf);
 3233       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3234       __ fmovd(rscratch2, src_reg);
 3235     }
 3236     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3237                  rscratch1, stlr);
 3238   %}
 3239 
 3240   // synchronized read/update encodings
 3241 
 3242   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3243     C2_MacroAssembler _masm(&cbuf);
 3244     Register dst_reg = as_Register($dst$$reg);
 3245     Register base = as_Register($mem$$base);
 3246     int index = $mem$$index;
 3247     int scale = $mem$$scale;
 3248     int disp = $mem$$disp;
 3249     if (index == -1) {
 3250        if (disp != 0) {
 3251         __ lea(rscratch1, Address(base, disp));
 3252         __ ldaxr(dst_reg, rscratch1);
 3253       } else {
 3254         // TODO
 3255         // should we ever get anything other than this case?
 3256         __ ldaxr(dst_reg, base);
 3257       }
 3258     } else {
 3259       Register index_reg = as_Register(index);
 3260       if (disp == 0) {
 3261         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3262         __ ldaxr(dst_reg, rscratch1);
 3263       } else {
 3264         __ lea(rscratch1, Address(base, disp));
 3265         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3266         __ ldaxr(dst_reg, rscratch1);
 3267       }
 3268     }
 3269   %}
 3270 
 3271   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3272     C2_MacroAssembler _masm(&cbuf);
 3273     Register src_reg = as_Register($src$$reg);
 3274     Register base = as_Register($mem$$base);
 3275     int index = $mem$$index;
 3276     int scale = $mem$$scale;
 3277     int disp = $mem$$disp;
 3278     if (index == -1) {
 3279        if (disp != 0) {
 3280         __ lea(rscratch2, Address(base, disp));
 3281         __ stlxr(rscratch1, src_reg, rscratch2);
 3282       } else {
 3283         // TODO
 3284         // should we ever get anything other than this case?
 3285         __ stlxr(rscratch1, src_reg, base);
 3286       }
 3287     } else {
 3288       Register index_reg = as_Register(index);
 3289       if (disp == 0) {
 3290         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3291         __ stlxr(rscratch1, src_reg, rscratch2);
 3292       } else {
 3293         __ lea(rscratch2, Address(base, disp));
 3294         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3295         __ stlxr(rscratch1, src_reg, rscratch2);
 3296       }
 3297     }
 3298     __ cmpw(rscratch1, zr);
 3299   %}
 3300 
 3301   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3302     C2_MacroAssembler _masm(&cbuf);
 3303     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3304     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3305                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3306                /*weak*/ false, noreg);
 3307   %}
 3308 
 3309   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3310     C2_MacroAssembler _masm(&cbuf);
 3311     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3312     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3313                Assembler::word, /*acquire*/ false, /*release*/ true,
 3314                /*weak*/ false, noreg);
 3315   %}
 3316 
 3317   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3318     C2_MacroAssembler _masm(&cbuf);
 3319     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3320     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3321                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3322                /*weak*/ false, noreg);
 3323   %}
 3324 
 3325   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3326     C2_MacroAssembler _masm(&cbuf);
 3327     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3328     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3329                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3330                /*weak*/ false, noreg);
 3331   %}
 3332 
 3333 
 3334   // The only difference between aarch64_enc_cmpxchg and
 3335   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3336   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3337   // lock.
 3338   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3339     C2_MacroAssembler _masm(&cbuf);
 3340     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3341     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3342                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3343                /*weak*/ false, noreg);
 3344   %}
 3345 
 3346   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3347     C2_MacroAssembler _masm(&cbuf);
 3348     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3349     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3350                Assembler::word, /*acquire*/ true, /*release*/ true,
 3351                /*weak*/ false, noreg);
 3352   %}
 3353 
 3354   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3355     C2_MacroAssembler _masm(&cbuf);
 3356     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3357     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3358                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3359                /*weak*/ false, noreg);
 3360   %}
 3361 
 3362   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3363     C2_MacroAssembler _masm(&cbuf);
 3364     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3365     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3366                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3367                /*weak*/ false, noreg);
 3368   %}
 3369 
 3370   // auxiliary used for CompareAndSwapX to set result register
 3371   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3372     C2_MacroAssembler _masm(&cbuf);
 3373     Register res_reg = as_Register($res$$reg);
 3374     __ cset(res_reg, Assembler::EQ);
 3375   %}
 3376 
 3377   // prefetch encodings
 3378 
 3379   enc_class aarch64_enc_prefetchw(memory mem) %{
 3380     C2_MacroAssembler _masm(&cbuf);
 3381     Register base = as_Register($mem$$base);
 3382     int index = $mem$$index;
 3383     int scale = $mem$$scale;
 3384     int disp = $mem$$disp;
 3385     if (index == -1) {
 3386       __ prfm(Address(base, disp), PSTL1KEEP);
 3387     } else {
 3388       Register index_reg = as_Register(index);
 3389       if (disp == 0) {
 3390         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3391       } else {
 3392         __ lea(rscratch1, Address(base, disp));
 3393 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3394       }
 3395     }
 3396   %}
 3397 
 3398   /// mov envcodings
 3399 
 3400   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3401     C2_MacroAssembler _masm(&cbuf);
 3402     uint32_t con = (uint32_t)$src$$constant;
 3403     Register dst_reg = as_Register($dst$$reg);
 3404     if (con == 0) {
 3405       __ movw(dst_reg, zr);
 3406     } else {
 3407       __ movw(dst_reg, con);
 3408     }
 3409   %}
 3410 
 3411   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3412     C2_MacroAssembler _masm(&cbuf);
 3413     Register dst_reg = as_Register($dst$$reg);
 3414     uint64_t con = (uint64_t)$src$$constant;
 3415     if (con == 0) {
 3416       __ mov(dst_reg, zr);
 3417     } else {
 3418       __ mov(dst_reg, con);
 3419     }
 3420   %}
 3421 
 3422   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3423     C2_MacroAssembler _masm(&cbuf);
 3424     Register dst_reg = as_Register($dst$$reg);
 3425     address con = (address)$src$$constant;
 3426     if (con == nullptr || con == (address)1) {
 3427       ShouldNotReachHere();
 3428     } else {
 3429       relocInfo::relocType rtype = $src->constant_reloc();
 3430       if (rtype == relocInfo::oop_type) {
 3431         __ movoop(dst_reg, (jobject)con);
 3432       } else if (rtype == relocInfo::metadata_type) {
 3433         __ mov_metadata(dst_reg, (Metadata*)con);
 3434       } else {
 3435         assert(rtype == relocInfo::none, "unexpected reloc type");
 3436         if (! __ is_valid_AArch64_address(con) ||
 3437             con < (address)(uintptr_t)os::vm_page_size()) {
 3438           __ mov(dst_reg, con);
 3439         } else {
 3440           uint64_t offset;
 3441           __ adrp(dst_reg, con, offset);
 3442           __ add(dst_reg, dst_reg, offset);
 3443         }
 3444       }
 3445     }
 3446   %}
 3447 
 3448   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3449     C2_MacroAssembler _masm(&cbuf);
 3450     Register dst_reg = as_Register($dst$$reg);
 3451     __ mov(dst_reg, zr);
 3452   %}
 3453 
 3454   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3455     C2_MacroAssembler _masm(&cbuf);
 3456     Register dst_reg = as_Register($dst$$reg);
 3457     __ mov(dst_reg, (uint64_t)1);
 3458   %}
 3459 
 3460   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3461     C2_MacroAssembler _masm(&cbuf);
 3462     __ load_byte_map_base($dst$$Register);
 3463   %}
 3464 
 3465   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3466     C2_MacroAssembler _masm(&cbuf);
 3467     Register dst_reg = as_Register($dst$$reg);
 3468     address con = (address)$src$$constant;
 3469     if (con == nullptr) {
 3470       ShouldNotReachHere();
 3471     } else {
 3472       relocInfo::relocType rtype = $src->constant_reloc();
 3473       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3474       __ set_narrow_oop(dst_reg, (jobject)con);
 3475     }
 3476   %}
 3477 
 3478   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3479     C2_MacroAssembler _masm(&cbuf);
 3480     Register dst_reg = as_Register($dst$$reg);
 3481     __ mov(dst_reg, zr);
 3482   %}
 3483 
 3484   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3485     C2_MacroAssembler _masm(&cbuf);
 3486     Register dst_reg = as_Register($dst$$reg);
 3487     address con = (address)$src$$constant;
 3488     if (con == nullptr) {
 3489       ShouldNotReachHere();
 3490     } else {
 3491       relocInfo::relocType rtype = $src->constant_reloc();
 3492       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3493       __ set_narrow_klass(dst_reg, (Klass *)con);
 3494     }
 3495   %}
 3496 
 3497   // arithmetic encodings
 3498 
 3499   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3500     C2_MacroAssembler _masm(&cbuf);
 3501     Register dst_reg = as_Register($dst$$reg);
 3502     Register src_reg = as_Register($src1$$reg);
 3503     int32_t con = (int32_t)$src2$$constant;
 3504     // add has primary == 0, subtract has primary == 1
 3505     if ($primary) { con = -con; }
 3506     if (con < 0) {
 3507       __ subw(dst_reg, src_reg, -con);
 3508     } else {
 3509       __ addw(dst_reg, src_reg, con);
 3510     }
 3511   %}
 3512 
 3513   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3514     C2_MacroAssembler _masm(&cbuf);
 3515     Register dst_reg = as_Register($dst$$reg);
 3516     Register src_reg = as_Register($src1$$reg);
 3517     int32_t con = (int32_t)$src2$$constant;
 3518     // add has primary == 0, subtract has primary == 1
 3519     if ($primary) { con = -con; }
 3520     if (con < 0) {
 3521       __ sub(dst_reg, src_reg, -con);
 3522     } else {
 3523       __ add(dst_reg, src_reg, con);
 3524     }
 3525   %}
 3526 
 3527   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3528     C2_MacroAssembler _masm(&cbuf);
 3529    Register dst_reg = as_Register($dst$$reg);
 3530    Register src1_reg = as_Register($src1$$reg);
 3531    Register src2_reg = as_Register($src2$$reg);
 3532     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3533   %}
 3534 
 3535   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3536     C2_MacroAssembler _masm(&cbuf);
 3537    Register dst_reg = as_Register($dst$$reg);
 3538    Register src1_reg = as_Register($src1$$reg);
 3539    Register src2_reg = as_Register($src2$$reg);
 3540     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3541   %}
 3542 
 3543   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3544     C2_MacroAssembler _masm(&cbuf);
 3545    Register dst_reg = as_Register($dst$$reg);
 3546    Register src1_reg = as_Register($src1$$reg);
 3547    Register src2_reg = as_Register($src2$$reg);
 3548     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3549   %}
 3550 
 3551   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3552     C2_MacroAssembler _masm(&cbuf);
 3553    Register dst_reg = as_Register($dst$$reg);
 3554    Register src1_reg = as_Register($src1$$reg);
 3555    Register src2_reg = as_Register($src2$$reg);
 3556     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3557   %}
 3558 
 3559   // compare instruction encodings
 3560 
 3561   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3562     C2_MacroAssembler _masm(&cbuf);
 3563     Register reg1 = as_Register($src1$$reg);
 3564     Register reg2 = as_Register($src2$$reg);
 3565     __ cmpw(reg1, reg2);
 3566   %}
 3567 
 3568   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3569     C2_MacroAssembler _masm(&cbuf);
 3570     Register reg = as_Register($src1$$reg);
 3571     int32_t val = $src2$$constant;
 3572     if (val >= 0) {
 3573       __ subsw(zr, reg, val);
 3574     } else {
 3575       __ addsw(zr, reg, -val);
 3576     }
 3577   %}
 3578 
 3579   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3580     C2_MacroAssembler _masm(&cbuf);
 3581     Register reg1 = as_Register($src1$$reg);
 3582     uint32_t val = (uint32_t)$src2$$constant;
 3583     __ movw(rscratch1, val);
 3584     __ cmpw(reg1, rscratch1);
 3585   %}
 3586 
 3587   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3588     C2_MacroAssembler _masm(&cbuf);
 3589     Register reg1 = as_Register($src1$$reg);
 3590     Register reg2 = as_Register($src2$$reg);
 3591     __ cmp(reg1, reg2);
 3592   %}
 3593 
 3594   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3595     C2_MacroAssembler _masm(&cbuf);
 3596     Register reg = as_Register($src1$$reg);
 3597     int64_t val = $src2$$constant;
 3598     if (val >= 0) {
 3599       __ subs(zr, reg, val);
 3600     } else if (val != -val) {
 3601       __ adds(zr, reg, -val);
 3602     } else {
 3603     // aargh, Long.MIN_VALUE is a special case
 3604       __ orr(rscratch1, zr, (uint64_t)val);
 3605       __ subs(zr, reg, rscratch1);
 3606     }
 3607   %}
 3608 
 3609   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3610     C2_MacroAssembler _masm(&cbuf);
 3611     Register reg1 = as_Register($src1$$reg);
 3612     uint64_t val = (uint64_t)$src2$$constant;
 3613     __ mov(rscratch1, val);
 3614     __ cmp(reg1, rscratch1);
 3615   %}
 3616 
 3617   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3618     C2_MacroAssembler _masm(&cbuf);
 3619     Register reg1 = as_Register($src1$$reg);
 3620     Register reg2 = as_Register($src2$$reg);
 3621     __ cmp(reg1, reg2);
 3622   %}
 3623 
 3624   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3625     C2_MacroAssembler _masm(&cbuf);
 3626     Register reg1 = as_Register($src1$$reg);
 3627     Register reg2 = as_Register($src2$$reg);
 3628     __ cmpw(reg1, reg2);
 3629   %}
 3630 
 3631   enc_class aarch64_enc_testp(iRegP src) %{
 3632     C2_MacroAssembler _masm(&cbuf);
 3633     Register reg = as_Register($src$$reg);
 3634     __ cmp(reg, zr);
 3635   %}
 3636 
 3637   enc_class aarch64_enc_testn(iRegN src) %{
 3638     C2_MacroAssembler _masm(&cbuf);
 3639     Register reg = as_Register($src$$reg);
 3640     __ cmpw(reg, zr);
 3641   %}
 3642 
 3643   enc_class aarch64_enc_b(label lbl) %{
 3644     C2_MacroAssembler _masm(&cbuf);
 3645     Label *L = $lbl$$label;
 3646     __ b(*L);
 3647   %}
 3648 
 3649   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3650     C2_MacroAssembler _masm(&cbuf);
 3651     Label *L = $lbl$$label;
 3652     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3653   %}
 3654 
 3655   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3656     C2_MacroAssembler _masm(&cbuf);
 3657     Label *L = $lbl$$label;
 3658     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3659   %}
 3660 
 3661   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3662   %{
 3663      Register sub_reg = as_Register($sub$$reg);
 3664      Register super_reg = as_Register($super$$reg);
 3665      Register temp_reg = as_Register($temp$$reg);
 3666      Register result_reg = as_Register($result$$reg);
 3667 
 3668      Label miss;
 3669      C2_MacroAssembler _masm(&cbuf);
 3670      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3671                                      nullptr, &miss,
 3672                                      /*set_cond_codes:*/ true);
 3673      if ($primary) {
 3674        __ mov(result_reg, zr);
 3675      }
 3676      __ bind(miss);
 3677   %}
 3678 
 3679   enc_class aarch64_enc_java_static_call(method meth) %{
 3680     C2_MacroAssembler _masm(&cbuf);
 3681 
 3682     address addr = (address)$meth$$method;
 3683     address call;
 3684     if (!_method) {
 3685       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3686       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
 3687       if (call == nullptr) {
 3688         ciEnv::current()->record_failure("CodeCache is full");
 3689         return;
 3690       }
 3691     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 3692       // The NOP here is purely to ensure that eliding a call to
 3693       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 3694       __ nop();
 3695       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 3696     } else {
 3697       int method_index = resolved_method_index(cbuf);
 3698       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3699                                                   : static_call_Relocation::spec(method_index);
 3700       call = __ trampoline_call(Address(addr, rspec));
 3701       if (call == nullptr) {
 3702         ciEnv::current()->record_failure("CodeCache is full");
 3703         return;
 3704       }
 3705       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 3706         // Calls of the same statically bound method can share
 3707         // a stub to the interpreter.
 3708         cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
 3709       } else {
 3710         // Emit stub for static call
 3711         address stub = CompiledDirectCall::emit_to_interp_stub(cbuf, call);
 3712         if (stub == nullptr) {
 3713           ciEnv::current()->record_failure("CodeCache is full");
 3714           return;
 3715         }
 3716       }
 3717     }
 3718 
 3719     __ post_call_nop();
 3720 
 3721     // Only non uncommon_trap calls need to reinitialize ptrue.
 3722     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3723       __ reinitialize_ptrue();
 3724     }
 3725   %}
 3726 
 3727   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3728     C2_MacroAssembler _masm(&cbuf);
 3729     int method_index = resolved_method_index(cbuf);
 3730     address call = __ ic_call((address)$meth$$method, method_index);
 3731     if (call == nullptr) {
 3732       ciEnv::current()->record_failure("CodeCache is full");
 3733       return;
 3734     }
 3735     __ post_call_nop();
 3736     if (Compile::current()->max_vector_size() > 0) {
 3737       __ reinitialize_ptrue();
 3738     }
 3739   %}
 3740 
 3741   enc_class aarch64_enc_call_epilog() %{
 3742     C2_MacroAssembler _masm(&cbuf);
 3743     if (VerifyStackAtCalls) {
 3744       // Check that stack depth is unchanged: find majik cookie on stack
 3745       __ call_Unimplemented();
 3746     }
 3747   %}
 3748 
 3749   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3750     C2_MacroAssembler _masm(&cbuf);
 3751 
 3752     // some calls to generated routines (arraycopy code) are scheduled
 3753     // by C2 as runtime calls. if so we can call them using a br (they
 3754     // will be in a reachable segment) otherwise we have to use a blr
 3755     // which loads the absolute address into a register.
 3756     address entry = (address)$meth$$method;
 3757     CodeBlob *cb = CodeCache::find_blob(entry);
 3758     if (cb) {
 3759       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3760       if (call == nullptr) {
 3761         ciEnv::current()->record_failure("CodeCache is full");
 3762         return;
 3763       }
 3764       __ post_call_nop();
 3765     } else {
 3766       Label retaddr;
 3767       __ adr(rscratch2, retaddr);
 3768       __ lea(rscratch1, RuntimeAddress(entry));
 3769       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3770       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3771       __ blr(rscratch1);
 3772       __ bind(retaddr);
 3773       __ post_call_nop();
 3774       __ add(sp, sp, 2 * wordSize);
 3775     }
 3776     if (Compile::current()->max_vector_size() > 0) {
 3777       __ reinitialize_ptrue();
 3778     }
 3779   %}
 3780 
 3781   enc_class aarch64_enc_rethrow() %{
 3782     C2_MacroAssembler _masm(&cbuf);
 3783     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3784   %}
 3785 
 3786   enc_class aarch64_enc_ret() %{
 3787     C2_MacroAssembler _masm(&cbuf);
 3788 #ifdef ASSERT
 3789     if (Compile::current()->max_vector_size() > 0) {
 3790       __ verify_ptrue();
 3791     }
 3792 #endif
 3793     __ ret(lr);
 3794   %}
 3795 
 3796   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3797     C2_MacroAssembler _masm(&cbuf);
 3798     Register target_reg = as_Register($jump_target$$reg);
 3799     __ br(target_reg);
 3800   %}
 3801 
 3802   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3803     C2_MacroAssembler _masm(&cbuf);
 3804     Register target_reg = as_Register($jump_target$$reg);
 3805     // exception oop should be in r0
 3806     // ret addr has been popped into lr
 3807     // callee expects it in r3
 3808     __ mov(r3, lr);
 3809     __ br(target_reg);
 3810   %}
 3811 
 3812 %}
 3813 
 3814 //----------FRAME--------------------------------------------------------------
 3815 // Definition of frame structure and management information.
 3816 //
 3817 //  S T A C K   L A Y O U T    Allocators stack-slot number
 3818 //                             |   (to get allocators register number
 3819 //  G  Owned by    |        |  v    add OptoReg::stack0())
 3820 //  r   CALLER     |        |
 3821 //  o     |        +--------+      pad to even-align allocators stack-slot
 3822 //  w     V        |  pad0  |        numbers; owned by CALLER
 3823 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 3824 //  h     ^        |   in   |  5
 3825 //        |        |  args  |  4   Holes in incoming args owned by SELF
 3826 //  |     |        |        |  3
 3827 //  |     |        +--------+
 3828 //  V     |        | old out|      Empty on Intel, window on Sparc
 3829 //        |    old |preserve|      Must be even aligned.
 3830 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 3831 //        |        |   in   |  3   area for Intel ret address
 3832 //     Owned by    |preserve|      Empty on Sparc.
 3833 //       SELF      +--------+
 3834 //        |        |  pad2  |  2   pad to align old SP
 3835 //        |        +--------+  1
 3836 //        |        | locks  |  0
 3837 //        |        +--------+----> OptoReg::stack0(), even aligned
 3838 //        |        |  pad1  | 11   pad to align new SP
 3839 //        |        +--------+
 3840 //        |        |        | 10
 3841 //        |        | spills |  9   spills
 3842 //        V        |        |  8   (pad0 slot for callee)
 3843 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 3844 //        ^        |  out   |  7
 3845 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 3846 //     Owned by    +--------+
 3847 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 3848 //        |    new |preserve|      Must be even-aligned.
 3849 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 3850 //        |        |        |
 3851 //
 3852 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 3853 //         known from SELF's arguments and the Java calling convention.
 3854 //         Region 6-7 is determined per call site.
 3855 // Note 2: If the calling convention leaves holes in the incoming argument
 3856 //         area, those holes are owned by SELF.  Holes in the outgoing area
 3857 //         are owned by the CALLEE.  Holes should not be necessary in the
 3858 //         incoming area, as the Java calling convention is completely under
 3859 //         the control of the AD file.  Doubles can be sorted and packed to
 3860 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 3861 //         varargs C calling conventions.
 3862 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 3863 //         even aligned with pad0 as needed.
 3864 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 3865 //           (the latter is true on Intel but is it false on AArch64?)
 3866 //         region 6-11 is even aligned; it may be padded out more so that
 3867 //         the region from SP to FP meets the minimum stack alignment.
 3868 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 3869 //         alignment.  Region 11, pad1, may be dynamically extended so that
 3870 //         SP meets the minimum alignment.
 3871 
 3872 frame %{
 3873   // These three registers define part of the calling convention
 3874   // between compiled code and the interpreter.
 3875 
 3876   // Inline Cache Register or Method for I2C.
 3877   inline_cache_reg(R12);
 3878 
 3879   // Number of stack slots consumed by locking an object
 3880   sync_stack_slots(2);
 3881 
 3882   // Compiled code's Frame Pointer
 3883   frame_pointer(R31);
 3884 
 3885   // Interpreter stores its frame pointer in a register which is
 3886   // stored to the stack by I2CAdaptors.
 3887   // I2CAdaptors convert from interpreted java to compiled java.
 3888   interpreter_frame_pointer(R29);
 3889 
 3890   // Stack alignment requirement
 3891   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 3892 
 3893   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 3894   // for calls to C.  Supports the var-args backing area for register parms.
 3895   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 3896 
 3897   // The after-PROLOG location of the return address.  Location of
 3898   // return address specifies a type (REG or STACK) and a number
 3899   // representing the register number (i.e. - use a register name) or
 3900   // stack slot.
 3901   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 3902   // Otherwise, it is above the locks and verification slot and alignment word
 3903   // TODO this may well be correct but need to check why that - 2 is there
 3904   // ppc port uses 0 but we definitely need to allow for fixed_slots
 3905   // which folds in the space used for monitors
 3906   return_addr(STACK - 2 +
 3907               align_up((Compile::current()->in_preserve_stack_slots() +
 3908                         Compile::current()->fixed_slots()),
 3909                        stack_alignment_in_slots()));
 3910 
 3911   // Location of compiled Java return values.  Same as C for now.
 3912   return_value
 3913   %{
 3914     // TODO do we allow ideal_reg == Op_RegN???
 3915     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 3916            "only return normal values");
 3917 
 3918     static const int lo[Op_RegL + 1] = { // enum name
 3919       0,                                 // Op_Node
 3920       0,                                 // Op_Set
 3921       R0_num,                            // Op_RegN
 3922       R0_num,                            // Op_RegI
 3923       R0_num,                            // Op_RegP
 3924       V0_num,                            // Op_RegF
 3925       V0_num,                            // Op_RegD
 3926       R0_num                             // Op_RegL
 3927     };
 3928 
 3929     static const int hi[Op_RegL + 1] = { // enum name
 3930       0,                                 // Op_Node
 3931       0,                                 // Op_Set
 3932       OptoReg::Bad,                      // Op_RegN
 3933       OptoReg::Bad,                      // Op_RegI
 3934       R0_H_num,                          // Op_RegP
 3935       OptoReg::Bad,                      // Op_RegF
 3936       V0_H_num,                          // Op_RegD
 3937       R0_H_num                           // Op_RegL
 3938     };
 3939 
 3940     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 3941   %}
 3942 %}
 3943 
 3944 //----------ATTRIBUTES---------------------------------------------------------
 3945 //----------Operand Attributes-------------------------------------------------
 3946 op_attrib op_cost(1);        // Required cost attribute
 3947 
 3948 //----------Instruction Attributes---------------------------------------------
 3949 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 3950 ins_attrib ins_size(32);        // Required size attribute (in bits)
 3951 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 3952                                 // a non-matching short branch variant
 3953                                 // of some long branch?
 3954 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 3955                                 // be a power of 2) specifies the
 3956                                 // alignment that some part of the
 3957                                 // instruction (not necessarily the
 3958                                 // start) requires.  If > 1, a
 3959                                 // compute_padding() function must be
 3960                                 // provided for the instruction
 3961 
 3962 //----------OPERANDS-----------------------------------------------------------
 3963 // Operand definitions must precede instruction definitions for correct parsing
 3964 // in the ADLC because operands constitute user defined types which are used in
 3965 // instruction definitions.
 3966 
 3967 //----------Simple Operands----------------------------------------------------
 3968 
 3969 // Integer operands 32 bit
 3970 // 32 bit immediate
 3971 operand immI()
 3972 %{
 3973   match(ConI);
 3974 
 3975   op_cost(0);
 3976   format %{ %}
 3977   interface(CONST_INTER);
 3978 %}
 3979 
 3980 // 32 bit zero
 3981 operand immI0()
 3982 %{
 3983   predicate(n->get_int() == 0);
 3984   match(ConI);
 3985 
 3986   op_cost(0);
 3987   format %{ %}
 3988   interface(CONST_INTER);
 3989 %}
 3990 
 3991 // 32 bit unit increment
 3992 operand immI_1()
 3993 %{
 3994   predicate(n->get_int() == 1);
 3995   match(ConI);
 3996 
 3997   op_cost(0);
 3998   format %{ %}
 3999   interface(CONST_INTER);
 4000 %}
 4001 
 4002 // 32 bit unit decrement
 4003 operand immI_M1()
 4004 %{
 4005   predicate(n->get_int() == -1);
 4006   match(ConI);
 4007 
 4008   op_cost(0);
 4009   format %{ %}
 4010   interface(CONST_INTER);
 4011 %}
 4012 
 4013 // Shift values for add/sub extension shift
 4014 operand immIExt()
 4015 %{
 4016   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4017   match(ConI);
 4018 
 4019   op_cost(0);
 4020   format %{ %}
 4021   interface(CONST_INTER);
 4022 %}
 4023 
 4024 operand immI_gt_1()
 4025 %{
 4026   predicate(n->get_int() > 1);
 4027   match(ConI);
 4028 
 4029   op_cost(0);
 4030   format %{ %}
 4031   interface(CONST_INTER);
 4032 %}
 4033 
 4034 operand immI_le_4()
 4035 %{
 4036   predicate(n->get_int() <= 4);
 4037   match(ConI);
 4038 
 4039   op_cost(0);
 4040   format %{ %}
 4041   interface(CONST_INTER);
 4042 %}
 4043 
 4044 operand immI_16()
 4045 %{
 4046   predicate(n->get_int() == 16);
 4047   match(ConI);
 4048 
 4049   op_cost(0);
 4050   format %{ %}
 4051   interface(CONST_INTER);
 4052 %}
 4053 
 4054 operand immI_24()
 4055 %{
 4056   predicate(n->get_int() == 24);
 4057   match(ConI);
 4058 
 4059   op_cost(0);
 4060   format %{ %}
 4061   interface(CONST_INTER);
 4062 %}
 4063 
 4064 operand immI_32()
 4065 %{
 4066   predicate(n->get_int() == 32);
 4067   match(ConI);
 4068 
 4069   op_cost(0);
 4070   format %{ %}
 4071   interface(CONST_INTER);
 4072 %}
 4073 
 4074 operand immI_48()
 4075 %{
 4076   predicate(n->get_int() == 48);
 4077   match(ConI);
 4078 
 4079   op_cost(0);
 4080   format %{ %}
 4081   interface(CONST_INTER);
 4082 %}
 4083 
 4084 operand immI_56()
 4085 %{
 4086   predicate(n->get_int() == 56);
 4087   match(ConI);
 4088 
 4089   op_cost(0);
 4090   format %{ %}
 4091   interface(CONST_INTER);
 4092 %}
 4093 
 4094 operand immI_63()
 4095 %{
 4096   predicate(n->get_int() == 63);
 4097   match(ConI);
 4098 
 4099   op_cost(0);
 4100   format %{ %}
 4101   interface(CONST_INTER);
 4102 %}
 4103 
 4104 operand immI_64()
 4105 %{
 4106   predicate(n->get_int() == 64);
 4107   match(ConI);
 4108 
 4109   op_cost(0);
 4110   format %{ %}
 4111   interface(CONST_INTER);
 4112 %}
 4113 
 4114 operand immI_255()
 4115 %{
 4116   predicate(n->get_int() == 255);
 4117   match(ConI);
 4118 
 4119   op_cost(0);
 4120   format %{ %}
 4121   interface(CONST_INTER);
 4122 %}
 4123 
 4124 operand immI_65535()
 4125 %{
 4126   predicate(n->get_int() == 65535);
 4127   match(ConI);
 4128 
 4129   op_cost(0);
 4130   format %{ %}
 4131   interface(CONST_INTER);
 4132 %}
 4133 
 4134 operand immI_positive()
 4135 %{
 4136   predicate(n->get_int() > 0);
 4137   match(ConI);
 4138 
 4139   op_cost(0);
 4140   format %{ %}
 4141   interface(CONST_INTER);
 4142 %}
 4143 
 4144 // BoolTest condition for signed compare
 4145 operand immI_cmp_cond()
 4146 %{
 4147   predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
 4148   match(ConI);
 4149 
 4150   op_cost(0);
 4151   format %{ %}
 4152   interface(CONST_INTER);
 4153 %}
 4154 
 4155 // BoolTest condition for unsigned compare
 4156 operand immI_cmpU_cond()
 4157 %{
 4158   predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
 4159   match(ConI);
 4160 
 4161   op_cost(0);
 4162   format %{ %}
 4163   interface(CONST_INTER);
 4164 %}
 4165 
 4166 operand immL_255()
 4167 %{
 4168   predicate(n->get_long() == 255L);
 4169   match(ConL);
 4170 
 4171   op_cost(0);
 4172   format %{ %}
 4173   interface(CONST_INTER);
 4174 %}
 4175 
 4176 operand immL_65535()
 4177 %{
 4178   predicate(n->get_long() == 65535L);
 4179   match(ConL);
 4180 
 4181   op_cost(0);
 4182   format %{ %}
 4183   interface(CONST_INTER);
 4184 %}
 4185 
 4186 operand immL_4294967295()
 4187 %{
 4188   predicate(n->get_long() == 4294967295L);
 4189   match(ConL);
 4190 
 4191   op_cost(0);
 4192   format %{ %}
 4193   interface(CONST_INTER);
 4194 %}
 4195 
 4196 operand immL_bitmask()
 4197 %{
 4198   predicate((n->get_long() != 0)
 4199             && ((n->get_long() & 0xc000000000000000l) == 0)
 4200             && is_power_of_2(n->get_long() + 1));
 4201   match(ConL);
 4202 
 4203   op_cost(0);
 4204   format %{ %}
 4205   interface(CONST_INTER);
 4206 %}
 4207 
 4208 operand immI_bitmask()
 4209 %{
 4210   predicate((n->get_int() != 0)
 4211             && ((n->get_int() & 0xc0000000) == 0)
 4212             && is_power_of_2(n->get_int() + 1));
 4213   match(ConI);
 4214 
 4215   op_cost(0);
 4216   format %{ %}
 4217   interface(CONST_INTER);
 4218 %}
 4219 
 4220 operand immL_positive_bitmaskI()
 4221 %{
 4222   predicate((n->get_long() != 0)
 4223             && ((julong)n->get_long() < 0x80000000ULL)
 4224             && is_power_of_2(n->get_long() + 1));
 4225   match(ConL);
 4226 
 4227   op_cost(0);
 4228   format %{ %}
 4229   interface(CONST_INTER);
 4230 %}
 4231 
 4232 // Scale values for scaled offset addressing modes (up to long but not quad)
 4233 operand immIScale()
 4234 %{
 4235   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4236   match(ConI);
 4237 
 4238   op_cost(0);
 4239   format %{ %}
 4240   interface(CONST_INTER);
 4241 %}
 4242 
 4243 // 26 bit signed offset -- for pc-relative branches
 4244 operand immI26()
 4245 %{
 4246   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4247   match(ConI);
 4248 
 4249   op_cost(0);
 4250   format %{ %}
 4251   interface(CONST_INTER);
 4252 %}
 4253 
 4254 // 19 bit signed offset -- for pc-relative loads
 4255 operand immI19()
 4256 %{
 4257   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4258   match(ConI);
 4259 
 4260   op_cost(0);
 4261   format %{ %}
 4262   interface(CONST_INTER);
 4263 %}
 4264 
 4265 // 5 bit signed integer
 4266 operand immI5()
 4267 %{
 4268   predicate(Assembler::is_simm(n->get_int(), 5));
 4269   match(ConI);
 4270 
 4271   op_cost(0);
 4272   format %{ %}
 4273   interface(CONST_INTER);
 4274 %}
 4275 
 4276 // 7 bit unsigned integer
 4277 operand immIU7()
 4278 %{
 4279   predicate(Assembler::is_uimm(n->get_int(), 7));
 4280   match(ConI);
 4281 
 4282   op_cost(0);
 4283   format %{ %}
 4284   interface(CONST_INTER);
 4285 %}
 4286 
 4287 // 12 bit unsigned offset -- for base plus immediate loads
 4288 operand immIU12()
 4289 %{
 4290   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4291   match(ConI);
 4292 
 4293   op_cost(0);
 4294   format %{ %}
 4295   interface(CONST_INTER);
 4296 %}
 4297 
 4298 operand immLU12()
 4299 %{
 4300   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4301   match(ConL);
 4302 
 4303   op_cost(0);
 4304   format %{ %}
 4305   interface(CONST_INTER);
 4306 %}
 4307 
 4308 // Offset for scaled or unscaled immediate loads and stores
 4309 operand immIOffset()
 4310 %{
 4311   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4312   match(ConI);
 4313 
 4314   op_cost(0);
 4315   format %{ %}
 4316   interface(CONST_INTER);
 4317 %}
 4318 
 4319 operand immIOffset1()
 4320 %{
 4321   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4322   match(ConI);
 4323 
 4324   op_cost(0);
 4325   format %{ %}
 4326   interface(CONST_INTER);
 4327 %}
 4328 
 4329 operand immIOffset2()
 4330 %{
 4331   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4332   match(ConI);
 4333 
 4334   op_cost(0);
 4335   format %{ %}
 4336   interface(CONST_INTER);
 4337 %}
 4338 
 4339 operand immIOffset4()
 4340 %{
 4341   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4342   match(ConI);
 4343 
 4344   op_cost(0);
 4345   format %{ %}
 4346   interface(CONST_INTER);
 4347 %}
 4348 
 4349 operand immIOffset8()
 4350 %{
 4351   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4352   match(ConI);
 4353 
 4354   op_cost(0);
 4355   format %{ %}
 4356   interface(CONST_INTER);
 4357 %}
 4358 
 4359 operand immIOffset16()
 4360 %{
 4361   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4362   match(ConI);
 4363 
 4364   op_cost(0);
 4365   format %{ %}
 4366   interface(CONST_INTER);
 4367 %}
 4368 
 4369 operand immLoffset()
 4370 %{
 4371   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4372   match(ConL);
 4373 
 4374   op_cost(0);
 4375   format %{ %}
 4376   interface(CONST_INTER);
 4377 %}
 4378 
 4379 operand immLoffset1()
 4380 %{
 4381   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4382   match(ConL);
 4383 
 4384   op_cost(0);
 4385   format %{ %}
 4386   interface(CONST_INTER);
 4387 %}
 4388 
 4389 operand immLoffset2()
 4390 %{
 4391   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4392   match(ConL);
 4393 
 4394   op_cost(0);
 4395   format %{ %}
 4396   interface(CONST_INTER);
 4397 %}
 4398 
 4399 operand immLoffset4()
 4400 %{
 4401   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4402   match(ConL);
 4403 
 4404   op_cost(0);
 4405   format %{ %}
 4406   interface(CONST_INTER);
 4407 %}
 4408 
 4409 operand immLoffset8()
 4410 %{
 4411   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4412   match(ConL);
 4413 
 4414   op_cost(0);
 4415   format %{ %}
 4416   interface(CONST_INTER);
 4417 %}
 4418 
 4419 operand immLoffset16()
 4420 %{
 4421   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4422   match(ConL);
 4423 
 4424   op_cost(0);
 4425   format %{ %}
 4426   interface(CONST_INTER);
 4427 %}
 4428 
 4429 // 5 bit signed long integer
 4430 operand immL5()
 4431 %{
 4432   predicate(Assembler::is_simm(n->get_long(), 5));
 4433   match(ConL);
 4434 
 4435   op_cost(0);
 4436   format %{ %}
 4437   interface(CONST_INTER);
 4438 %}
 4439 
 4440 // 7 bit unsigned long integer
 4441 operand immLU7()
 4442 %{
 4443   predicate(Assembler::is_uimm(n->get_long(), 7));
 4444   match(ConL);
 4445 
 4446   op_cost(0);
 4447   format %{ %}
 4448   interface(CONST_INTER);
 4449 %}
 4450 
 4451 // 8 bit signed value.
 4452 operand immI8()
 4453 %{
 4454   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4455   match(ConI);
 4456 
 4457   op_cost(0);
 4458   format %{ %}
 4459   interface(CONST_INTER);
 4460 %}
 4461 
 4462 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4463 operand immI8_shift8()
 4464 %{
 4465   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4466             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4467   match(ConI);
 4468 
 4469   op_cost(0);
 4470   format %{ %}
 4471   interface(CONST_INTER);
 4472 %}
 4473 
 4474 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4475 operand immL8_shift8()
 4476 %{
 4477   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4478             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4479   match(ConL);
 4480 
 4481   op_cost(0);
 4482   format %{ %}
 4483   interface(CONST_INTER);
 4484 %}
 4485 
 4486 // 8 bit integer valid for vector add sub immediate
 4487 operand immBAddSubV()
 4488 %{
 4489   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4490   match(ConI);
 4491 
 4492   op_cost(0);
 4493   format %{ %}
 4494   interface(CONST_INTER);
 4495 %}
 4496 
 4497 // 32 bit integer valid for add sub immediate
 4498 operand immIAddSub()
 4499 %{
 4500   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4501   match(ConI);
 4502   op_cost(0);
 4503   format %{ %}
 4504   interface(CONST_INTER);
 4505 %}
 4506 
 4507 // 32 bit integer valid for vector add sub immediate
 4508 operand immIAddSubV()
 4509 %{
 4510   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4511   match(ConI);
 4512 
 4513   op_cost(0);
 4514   format %{ %}
 4515   interface(CONST_INTER);
 4516 %}
 4517 
 4518 // 32 bit unsigned integer valid for logical immediate
 4519 
 4520 operand immBLog()
 4521 %{
 4522   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4523   match(ConI);
 4524 
 4525   op_cost(0);
 4526   format %{ %}
 4527   interface(CONST_INTER);
 4528 %}
 4529 
 4530 operand immSLog()
 4531 %{
 4532   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4533   match(ConI);
 4534 
 4535   op_cost(0);
 4536   format %{ %}
 4537   interface(CONST_INTER);
 4538 %}
 4539 
 4540 operand immILog()
 4541 %{
 4542   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4543   match(ConI);
 4544 
 4545   op_cost(0);
 4546   format %{ %}
 4547   interface(CONST_INTER);
 4548 %}
 4549 
 4550 // Integer operands 64 bit
 4551 // 64 bit immediate
 4552 operand immL()
 4553 %{
 4554   match(ConL);
 4555 
 4556   op_cost(0);
 4557   format %{ %}
 4558   interface(CONST_INTER);
 4559 %}
 4560 
 4561 // 64 bit zero
 4562 operand immL0()
 4563 %{
 4564   predicate(n->get_long() == 0);
 4565   match(ConL);
 4566 
 4567   op_cost(0);
 4568   format %{ %}
 4569   interface(CONST_INTER);
 4570 %}
 4571 
 4572 // 64 bit unit increment
 4573 operand immL_1()
 4574 %{
 4575   predicate(n->get_long() == 1);
 4576   match(ConL);
 4577 
 4578   op_cost(0);
 4579   format %{ %}
 4580   interface(CONST_INTER);
 4581 %}
 4582 
 4583 // 64 bit unit decrement
 4584 operand immL_M1()
 4585 %{
 4586   predicate(n->get_long() == -1);
 4587   match(ConL);
 4588 
 4589   op_cost(0);
 4590   format %{ %}
 4591   interface(CONST_INTER);
 4592 %}
 4593 
 4594 // 32 bit offset of pc in thread anchor
 4595 
 4596 operand immL_pc_off()
 4597 %{
 4598   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4599                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4600   match(ConL);
 4601 
 4602   op_cost(0);
 4603   format %{ %}
 4604   interface(CONST_INTER);
 4605 %}
 4606 
 4607 // 64 bit integer valid for add sub immediate
 4608 operand immLAddSub()
 4609 %{
 4610   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4611   match(ConL);
 4612   op_cost(0);
 4613   format %{ %}
 4614   interface(CONST_INTER);
 4615 %}
 4616 
 4617 // 64 bit integer valid for addv subv immediate
 4618 operand immLAddSubV()
 4619 %{
 4620   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4621   match(ConL);
 4622 
 4623   op_cost(0);
 4624   format %{ %}
 4625   interface(CONST_INTER);
 4626 %}
 4627 
 4628 // 64 bit integer valid for logical immediate
 4629 operand immLLog()
 4630 %{
 4631   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4632   match(ConL);
 4633   op_cost(0);
 4634   format %{ %}
 4635   interface(CONST_INTER);
 4636 %}
 4637 
 4638 // Long Immediate: low 32-bit mask
 4639 operand immL_32bits()
 4640 %{
 4641   predicate(n->get_long() == 0xFFFFFFFFL);
 4642   match(ConL);
 4643   op_cost(0);
 4644   format %{ %}
 4645   interface(CONST_INTER);
 4646 %}
 4647 
 4648 // Pointer operands
 4649 // Pointer Immediate
 4650 operand immP()
 4651 %{
 4652   match(ConP);
 4653 
 4654   op_cost(0);
 4655   format %{ %}
 4656   interface(CONST_INTER);
 4657 %}
 4658 
 4659 // Null Pointer Immediate
 4660 operand immP0()
 4661 %{
 4662   predicate(n->get_ptr() == 0);
 4663   match(ConP);
 4664 
 4665   op_cost(0);
 4666   format %{ %}
 4667   interface(CONST_INTER);
 4668 %}
 4669 
 4670 // Pointer Immediate One
 4671 // this is used in object initialization (initial object header)
 4672 operand immP_1()
 4673 %{
 4674   predicate(n->get_ptr() == 1);
 4675   match(ConP);
 4676 
 4677   op_cost(0);
 4678   format %{ %}
 4679   interface(CONST_INTER);
 4680 %}
 4681 
 4682 // Card Table Byte Map Base
 4683 operand immByteMapBase()
 4684 %{
 4685   // Get base of card map
 4686   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4687             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4688   match(ConP);
 4689 
 4690   op_cost(0);
 4691   format %{ %}
 4692   interface(CONST_INTER);
 4693 %}
 4694 
 4695 // Pointer Immediate Minus One
 4696 // this is used when we want to write the current PC to the thread anchor
 4697 operand immP_M1()
 4698 %{
 4699   predicate(n->get_ptr() == -1);
 4700   match(ConP);
 4701 
 4702   op_cost(0);
 4703   format %{ %}
 4704   interface(CONST_INTER);
 4705 %}
 4706 
 4707 // Pointer Immediate Minus Two
 4708 // this is used when we want to write the current PC to the thread anchor
 4709 operand immP_M2()
 4710 %{
 4711   predicate(n->get_ptr() == -2);
 4712   match(ConP);
 4713 
 4714   op_cost(0);
 4715   format %{ %}
 4716   interface(CONST_INTER);
 4717 %}
 4718 
 4719 // Float and Double operands
 4720 // Double Immediate
 4721 operand immD()
 4722 %{
 4723   match(ConD);
 4724   op_cost(0);
 4725   format %{ %}
 4726   interface(CONST_INTER);
 4727 %}
 4728 
 4729 // Double Immediate: +0.0d
 4730 operand immD0()
 4731 %{
 4732   predicate(jlong_cast(n->getd()) == 0);
 4733   match(ConD);
 4734 
 4735   op_cost(0);
 4736   format %{ %}
 4737   interface(CONST_INTER);
 4738 %}
 4739 
 4740 // constant 'double +0.0'.
 4741 operand immDPacked()
 4742 %{
 4743   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4744   match(ConD);
 4745   op_cost(0);
 4746   format %{ %}
 4747   interface(CONST_INTER);
 4748 %}
 4749 
 4750 // Float Immediate
 4751 operand immF()
 4752 %{
 4753   match(ConF);
 4754   op_cost(0);
 4755   format %{ %}
 4756   interface(CONST_INTER);
 4757 %}
 4758 
 4759 // Float Immediate: +0.0f.
 4760 operand immF0()
 4761 %{
 4762   predicate(jint_cast(n->getf()) == 0);
 4763   match(ConF);
 4764 
 4765   op_cost(0);
 4766   format %{ %}
 4767   interface(CONST_INTER);
 4768 %}
 4769 
 4770 //
 4771 operand immFPacked()
 4772 %{
 4773   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4774   match(ConF);
 4775   op_cost(0);
 4776   format %{ %}
 4777   interface(CONST_INTER);
 4778 %}
 4779 
 4780 // Narrow pointer operands
 4781 // Narrow Pointer Immediate
 4782 operand immN()
 4783 %{
 4784   match(ConN);
 4785 
 4786   op_cost(0);
 4787   format %{ %}
 4788   interface(CONST_INTER);
 4789 %}
 4790 
 4791 // Narrow Null Pointer Immediate
 4792 operand immN0()
 4793 %{
 4794   predicate(n->get_narrowcon() == 0);
 4795   match(ConN);
 4796 
 4797   op_cost(0);
 4798   format %{ %}
 4799   interface(CONST_INTER);
 4800 %}
 4801 
 4802 operand immNKlass()
 4803 %{
 4804   match(ConNKlass);
 4805 
 4806   op_cost(0);
 4807   format %{ %}
 4808   interface(CONST_INTER);
 4809 %}
 4810 
 4811 // Integer 32 bit Register Operands
 4812 // Integer 32 bitRegister (excludes SP)
 4813 operand iRegI()
 4814 %{
 4815   constraint(ALLOC_IN_RC(any_reg32));
 4816   match(RegI);
 4817   match(iRegINoSp);
 4818   op_cost(0);
 4819   format %{ %}
 4820   interface(REG_INTER);
 4821 %}
 4822 
 4823 // Integer 32 bit Register not Special
 4824 operand iRegINoSp()
 4825 %{
 4826   constraint(ALLOC_IN_RC(no_special_reg32));
 4827   match(RegI);
 4828   op_cost(0);
 4829   format %{ %}
 4830   interface(REG_INTER);
 4831 %}
 4832 
 4833 // Integer 64 bit Register Operands
 4834 // Integer 64 bit Register (includes SP)
 4835 operand iRegL()
 4836 %{
 4837   constraint(ALLOC_IN_RC(any_reg));
 4838   match(RegL);
 4839   match(iRegLNoSp);
 4840   op_cost(0);
 4841   format %{ %}
 4842   interface(REG_INTER);
 4843 %}
 4844 
 4845 // Integer 64 bit Register not Special
 4846 operand iRegLNoSp()
 4847 %{
 4848   constraint(ALLOC_IN_RC(no_special_reg));
 4849   match(RegL);
 4850   match(iRegL_R0);
 4851   format %{ %}
 4852   interface(REG_INTER);
 4853 %}
 4854 
 4855 // Pointer Register Operands
 4856 // Pointer Register
 4857 operand iRegP()
 4858 %{
 4859   constraint(ALLOC_IN_RC(ptr_reg));
 4860   match(RegP);
 4861   match(iRegPNoSp);
 4862   match(iRegP_R0);
 4863   //match(iRegP_R2);
 4864   //match(iRegP_R4);
 4865   match(iRegP_R5);
 4866   match(thread_RegP);
 4867   op_cost(0);
 4868   format %{ %}
 4869   interface(REG_INTER);
 4870 %}
 4871 
 4872 // Pointer 64 bit Register not Special
 4873 operand iRegPNoSp()
 4874 %{
 4875   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4876   match(RegP);
 4877   // match(iRegP);
 4878   // match(iRegP_R0);
 4879   // match(iRegP_R2);
 4880   // match(iRegP_R4);
 4881   // match(iRegP_R5);
 4882   // match(thread_RegP);
 4883   op_cost(0);
 4884   format %{ %}
 4885   interface(REG_INTER);
 4886 %}
 4887 
 4888 // Pointer 64 bit Register R0 only
 4889 operand iRegP_R0()
 4890 %{
 4891   constraint(ALLOC_IN_RC(r0_reg));
 4892   match(RegP);
 4893   // match(iRegP);
 4894   match(iRegPNoSp);
 4895   op_cost(0);
 4896   format %{ %}
 4897   interface(REG_INTER);
 4898 %}
 4899 
 4900 // Pointer 64 bit Register R1 only
 4901 operand iRegP_R1()
 4902 %{
 4903   constraint(ALLOC_IN_RC(r1_reg));
 4904   match(RegP);
 4905   // match(iRegP);
 4906   match(iRegPNoSp);
 4907   op_cost(0);
 4908   format %{ %}
 4909   interface(REG_INTER);
 4910 %}
 4911 
 4912 // Pointer 64 bit Register R2 only
 4913 operand iRegP_R2()
 4914 %{
 4915   constraint(ALLOC_IN_RC(r2_reg));
 4916   match(RegP);
 4917   // match(iRegP);
 4918   match(iRegPNoSp);
 4919   op_cost(0);
 4920   format %{ %}
 4921   interface(REG_INTER);
 4922 %}
 4923 
 4924 // Pointer 64 bit Register R3 only
 4925 operand iRegP_R3()
 4926 %{
 4927   constraint(ALLOC_IN_RC(r3_reg));
 4928   match(RegP);
 4929   // match(iRegP);
 4930   match(iRegPNoSp);
 4931   op_cost(0);
 4932   format %{ %}
 4933   interface(REG_INTER);
 4934 %}
 4935 
 4936 // Pointer 64 bit Register R4 only
 4937 operand iRegP_R4()
 4938 %{
 4939   constraint(ALLOC_IN_RC(r4_reg));
 4940   match(RegP);
 4941   // match(iRegP);
 4942   match(iRegPNoSp);
 4943   op_cost(0);
 4944   format %{ %}
 4945   interface(REG_INTER);
 4946 %}
 4947 
 4948 // Pointer 64 bit Register R5 only
 4949 operand iRegP_R5()
 4950 %{
 4951   constraint(ALLOC_IN_RC(r5_reg));
 4952   match(RegP);
 4953   // match(iRegP);
 4954   match(iRegPNoSp);
 4955   op_cost(0);
 4956   format %{ %}
 4957   interface(REG_INTER);
 4958 %}
 4959 
 4960 // Pointer 64 bit Register R10 only
 4961 operand iRegP_R10()
 4962 %{
 4963   constraint(ALLOC_IN_RC(r10_reg));
 4964   match(RegP);
 4965   // match(iRegP);
 4966   match(iRegPNoSp);
 4967   op_cost(0);
 4968   format %{ %}
 4969   interface(REG_INTER);
 4970 %}
 4971 
 4972 // Long 64 bit Register R0 only
 4973 operand iRegL_R0()
 4974 %{
 4975   constraint(ALLOC_IN_RC(r0_reg));
 4976   match(RegL);
 4977   match(iRegLNoSp);
 4978   op_cost(0);
 4979   format %{ %}
 4980   interface(REG_INTER);
 4981 %}
 4982 
 4983 // Long 64 bit Register R2 only
 4984 operand iRegL_R2()
 4985 %{
 4986   constraint(ALLOC_IN_RC(r2_reg));
 4987   match(RegL);
 4988   match(iRegLNoSp);
 4989   op_cost(0);
 4990   format %{ %}
 4991   interface(REG_INTER);
 4992 %}
 4993 
 4994 // Long 64 bit Register R3 only
 4995 operand iRegL_R3()
 4996 %{
 4997   constraint(ALLOC_IN_RC(r3_reg));
 4998   match(RegL);
 4999   match(iRegLNoSp);
 5000   op_cost(0);
 5001   format %{ %}
 5002   interface(REG_INTER);
 5003 %}
 5004 
 5005 // Long 64 bit Register R11 only
 5006 operand iRegL_R11()
 5007 %{
 5008   constraint(ALLOC_IN_RC(r11_reg));
 5009   match(RegL);
 5010   match(iRegLNoSp);
 5011   op_cost(0);
 5012   format %{ %}
 5013   interface(REG_INTER);
 5014 %}
 5015 
 5016 // Pointer 64 bit Register FP only
 5017 operand iRegP_FP()
 5018 %{
 5019   constraint(ALLOC_IN_RC(fp_reg));
 5020   match(RegP);
 5021   // match(iRegP);
 5022   op_cost(0);
 5023   format %{ %}
 5024   interface(REG_INTER);
 5025 %}
 5026 
 5027 // Register R0 only
 5028 operand iRegI_R0()
 5029 %{
 5030   constraint(ALLOC_IN_RC(int_r0_reg));
 5031   match(RegI);
 5032   match(iRegINoSp);
 5033   op_cost(0);
 5034   format %{ %}
 5035   interface(REG_INTER);
 5036 %}
 5037 
 5038 // Register R2 only
 5039 operand iRegI_R2()
 5040 %{
 5041   constraint(ALLOC_IN_RC(int_r2_reg));
 5042   match(RegI);
 5043   match(iRegINoSp);
 5044   op_cost(0);
 5045   format %{ %}
 5046   interface(REG_INTER);
 5047 %}
 5048 
 5049 // Register R3 only
 5050 operand iRegI_R3()
 5051 %{
 5052   constraint(ALLOC_IN_RC(int_r3_reg));
 5053   match(RegI);
 5054   match(iRegINoSp);
 5055   op_cost(0);
 5056   format %{ %}
 5057   interface(REG_INTER);
 5058 %}
 5059 
 5060 
 5061 // Register R4 only
 5062 operand iRegI_R4()
 5063 %{
 5064   constraint(ALLOC_IN_RC(int_r4_reg));
 5065   match(RegI);
 5066   match(iRegINoSp);
 5067   op_cost(0);
 5068   format %{ %}
 5069   interface(REG_INTER);
 5070 %}
 5071 
 5072 
 5073 // Pointer Register Operands
 5074 // Narrow Pointer Register
 5075 operand iRegN()
 5076 %{
 5077   constraint(ALLOC_IN_RC(any_reg32));
 5078   match(RegN);
 5079   match(iRegNNoSp);
 5080   op_cost(0);
 5081   format %{ %}
 5082   interface(REG_INTER);
 5083 %}
 5084 
 5085 operand iRegN_R0()
 5086 %{
 5087   constraint(ALLOC_IN_RC(r0_reg));
 5088   match(iRegN);
 5089   op_cost(0);
 5090   format %{ %}
 5091   interface(REG_INTER);
 5092 %}
 5093 
 5094 operand iRegN_R2()
 5095 %{
 5096   constraint(ALLOC_IN_RC(r2_reg));
 5097   match(iRegN);
 5098   op_cost(0);
 5099   format %{ %}
 5100   interface(REG_INTER);
 5101 %}
 5102 
 5103 operand iRegN_R3()
 5104 %{
 5105   constraint(ALLOC_IN_RC(r3_reg));
 5106   match(iRegN);
 5107   op_cost(0);
 5108   format %{ %}
 5109   interface(REG_INTER);
 5110 %}
 5111 
 5112 // Integer 64 bit Register not Special
 5113 operand iRegNNoSp()
 5114 %{
 5115   constraint(ALLOC_IN_RC(no_special_reg32));
 5116   match(RegN);
 5117   op_cost(0);
 5118   format %{ %}
 5119   interface(REG_INTER);
 5120 %}
 5121 
 5122 // Float Register
 5123 // Float register operands
 5124 operand vRegF()
 5125 %{
 5126   constraint(ALLOC_IN_RC(float_reg));
 5127   match(RegF);
 5128 
 5129   op_cost(0);
 5130   format %{ %}
 5131   interface(REG_INTER);
 5132 %}
 5133 
 5134 // Double Register
 5135 // Double register operands
 5136 operand vRegD()
 5137 %{
 5138   constraint(ALLOC_IN_RC(double_reg));
 5139   match(RegD);
 5140 
 5141   op_cost(0);
 5142   format %{ %}
 5143   interface(REG_INTER);
 5144 %}
 5145 
 5146 // Generic vector class. This will be used for
 5147 // all vector operands, including NEON and SVE.
 5148 operand vReg()
 5149 %{
 5150   constraint(ALLOC_IN_RC(dynamic));
 5151   match(VecA);
 5152   match(VecD);
 5153   match(VecX);
 5154 
 5155   op_cost(0);
 5156   format %{ %}
 5157   interface(REG_INTER);
 5158 %}
 5159 
 5160 operand vecA()
 5161 %{
 5162   constraint(ALLOC_IN_RC(vectora_reg));
 5163   match(VecA);
 5164 
 5165   op_cost(0);
 5166   format %{ %}
 5167   interface(REG_INTER);
 5168 %}
 5169 
 5170 operand vecD()
 5171 %{
 5172   constraint(ALLOC_IN_RC(vectord_reg));
 5173   match(VecD);
 5174 
 5175   op_cost(0);
 5176   format %{ %}
 5177   interface(REG_INTER);
 5178 %}
 5179 
 5180 operand vecX()
 5181 %{
 5182   constraint(ALLOC_IN_RC(vectorx_reg));
 5183   match(VecX);
 5184 
 5185   op_cost(0);
 5186   format %{ %}
 5187   interface(REG_INTER);
 5188 %}
 5189 
 5190 operand vRegD_V0()
 5191 %{
 5192   constraint(ALLOC_IN_RC(v0_reg));
 5193   match(RegD);
 5194   op_cost(0);
 5195   format %{ %}
 5196   interface(REG_INTER);
 5197 %}
 5198 
 5199 operand vRegD_V1()
 5200 %{
 5201   constraint(ALLOC_IN_RC(v1_reg));
 5202   match(RegD);
 5203   op_cost(0);
 5204   format %{ %}
 5205   interface(REG_INTER);
 5206 %}
 5207 
 5208 operand vRegD_V2()
 5209 %{
 5210   constraint(ALLOC_IN_RC(v2_reg));
 5211   match(RegD);
 5212   op_cost(0);
 5213   format %{ %}
 5214   interface(REG_INTER);
 5215 %}
 5216 
 5217 operand vRegD_V3()
 5218 %{
 5219   constraint(ALLOC_IN_RC(v3_reg));
 5220   match(RegD);
 5221   op_cost(0);
 5222   format %{ %}
 5223   interface(REG_INTER);
 5224 %}
 5225 
 5226 operand vRegD_V4()
 5227 %{
 5228   constraint(ALLOC_IN_RC(v4_reg));
 5229   match(RegD);
 5230   op_cost(0);
 5231   format %{ %}
 5232   interface(REG_INTER);
 5233 %}
 5234 
 5235 operand vRegD_V5()
 5236 %{
 5237   constraint(ALLOC_IN_RC(v5_reg));
 5238   match(RegD);
 5239   op_cost(0);
 5240   format %{ %}
 5241   interface(REG_INTER);
 5242 %}
 5243 
 5244 operand vRegD_V6()
 5245 %{
 5246   constraint(ALLOC_IN_RC(v6_reg));
 5247   match(RegD);
 5248   op_cost(0);
 5249   format %{ %}
 5250   interface(REG_INTER);
 5251 %}
 5252 
 5253 operand vRegD_V7()
 5254 %{
 5255   constraint(ALLOC_IN_RC(v7_reg));
 5256   match(RegD);
 5257   op_cost(0);
 5258   format %{ %}
 5259   interface(REG_INTER);
 5260 %}
 5261 
 5262 operand vRegD_V8()
 5263 %{
 5264   constraint(ALLOC_IN_RC(v8_reg));
 5265   match(RegD);
 5266   op_cost(0);
 5267   format %{ %}
 5268   interface(REG_INTER);
 5269 %}
 5270 
 5271 operand vRegD_V9()
 5272 %{
 5273   constraint(ALLOC_IN_RC(v9_reg));
 5274   match(RegD);
 5275   op_cost(0);
 5276   format %{ %}
 5277   interface(REG_INTER);
 5278 %}
 5279 
 5280 operand vRegD_V10()
 5281 %{
 5282   constraint(ALLOC_IN_RC(v10_reg));
 5283   match(RegD);
 5284   op_cost(0);
 5285   format %{ %}
 5286   interface(REG_INTER);
 5287 %}
 5288 
 5289 operand vRegD_V11()
 5290 %{
 5291   constraint(ALLOC_IN_RC(v11_reg));
 5292   match(RegD);
 5293   op_cost(0);
 5294   format %{ %}
 5295   interface(REG_INTER);
 5296 %}
 5297 
 5298 operand vRegD_V12()
 5299 %{
 5300   constraint(ALLOC_IN_RC(v12_reg));
 5301   match(RegD);
 5302   op_cost(0);
 5303   format %{ %}
 5304   interface(REG_INTER);
 5305 %}
 5306 
 5307 operand vRegD_V13()
 5308 %{
 5309   constraint(ALLOC_IN_RC(v13_reg));
 5310   match(RegD);
 5311   op_cost(0);
 5312   format %{ %}
 5313   interface(REG_INTER);
 5314 %}
 5315 
 5316 operand vRegD_V14()
 5317 %{
 5318   constraint(ALLOC_IN_RC(v14_reg));
 5319   match(RegD);
 5320   op_cost(0);
 5321   format %{ %}
 5322   interface(REG_INTER);
 5323 %}
 5324 
 5325 operand vRegD_V15()
 5326 %{
 5327   constraint(ALLOC_IN_RC(v15_reg));
 5328   match(RegD);
 5329   op_cost(0);
 5330   format %{ %}
 5331   interface(REG_INTER);
 5332 %}
 5333 
 5334 operand vRegD_V16()
 5335 %{
 5336   constraint(ALLOC_IN_RC(v16_reg));
 5337   match(RegD);
 5338   op_cost(0);
 5339   format %{ %}
 5340   interface(REG_INTER);
 5341 %}
 5342 
 5343 operand vRegD_V17()
 5344 %{
 5345   constraint(ALLOC_IN_RC(v17_reg));
 5346   match(RegD);
 5347   op_cost(0);
 5348   format %{ %}
 5349   interface(REG_INTER);
 5350 %}
 5351 
 5352 operand vRegD_V18()
 5353 %{
 5354   constraint(ALLOC_IN_RC(v18_reg));
 5355   match(RegD);
 5356   op_cost(0);
 5357   format %{ %}
 5358   interface(REG_INTER);
 5359 %}
 5360 
 5361 operand vRegD_V19()
 5362 %{
 5363   constraint(ALLOC_IN_RC(v19_reg));
 5364   match(RegD);
 5365   op_cost(0);
 5366   format %{ %}
 5367   interface(REG_INTER);
 5368 %}
 5369 
 5370 operand vRegD_V20()
 5371 %{
 5372   constraint(ALLOC_IN_RC(v20_reg));
 5373   match(RegD);
 5374   op_cost(0);
 5375   format %{ %}
 5376   interface(REG_INTER);
 5377 %}
 5378 
 5379 operand vRegD_V21()
 5380 %{
 5381   constraint(ALLOC_IN_RC(v21_reg));
 5382   match(RegD);
 5383   op_cost(0);
 5384   format %{ %}
 5385   interface(REG_INTER);
 5386 %}
 5387 
 5388 operand vRegD_V22()
 5389 %{
 5390   constraint(ALLOC_IN_RC(v22_reg));
 5391   match(RegD);
 5392   op_cost(0);
 5393   format %{ %}
 5394   interface(REG_INTER);
 5395 %}
 5396 
 5397 operand vRegD_V23()
 5398 %{
 5399   constraint(ALLOC_IN_RC(v23_reg));
 5400   match(RegD);
 5401   op_cost(0);
 5402   format %{ %}
 5403   interface(REG_INTER);
 5404 %}
 5405 
 5406 operand vRegD_V24()
 5407 %{
 5408   constraint(ALLOC_IN_RC(v24_reg));
 5409   match(RegD);
 5410   op_cost(0);
 5411   format %{ %}
 5412   interface(REG_INTER);
 5413 %}
 5414 
 5415 operand vRegD_V25()
 5416 %{
 5417   constraint(ALLOC_IN_RC(v25_reg));
 5418   match(RegD);
 5419   op_cost(0);
 5420   format %{ %}
 5421   interface(REG_INTER);
 5422 %}
 5423 
 5424 operand vRegD_V26()
 5425 %{
 5426   constraint(ALLOC_IN_RC(v26_reg));
 5427   match(RegD);
 5428   op_cost(0);
 5429   format %{ %}
 5430   interface(REG_INTER);
 5431 %}
 5432 
 5433 operand vRegD_V27()
 5434 %{
 5435   constraint(ALLOC_IN_RC(v27_reg));
 5436   match(RegD);
 5437   op_cost(0);
 5438   format %{ %}
 5439   interface(REG_INTER);
 5440 %}
 5441 
 5442 operand vRegD_V28()
 5443 %{
 5444   constraint(ALLOC_IN_RC(v28_reg));
 5445   match(RegD);
 5446   op_cost(0);
 5447   format %{ %}
 5448   interface(REG_INTER);
 5449 %}
 5450 
 5451 operand vRegD_V29()
 5452 %{
 5453   constraint(ALLOC_IN_RC(v29_reg));
 5454   match(RegD);
 5455   op_cost(0);
 5456   format %{ %}
 5457   interface(REG_INTER);
 5458 %}
 5459 
 5460 operand vRegD_V30()
 5461 %{
 5462   constraint(ALLOC_IN_RC(v30_reg));
 5463   match(RegD);
 5464   op_cost(0);
 5465   format %{ %}
 5466   interface(REG_INTER);
 5467 %}
 5468 
 5469 operand vRegD_V31()
 5470 %{
 5471   constraint(ALLOC_IN_RC(v31_reg));
 5472   match(RegD);
 5473   op_cost(0);
 5474   format %{ %}
 5475   interface(REG_INTER);
 5476 %}
 5477 
 5478 operand pReg()
 5479 %{
 5480   constraint(ALLOC_IN_RC(pr_reg));
 5481   match(RegVectMask);
 5482   match(pRegGov);
 5483   op_cost(0);
 5484   format %{ %}
 5485   interface(REG_INTER);
 5486 %}
 5487 
 5488 operand pRegGov()
 5489 %{
 5490   constraint(ALLOC_IN_RC(gov_pr));
 5491   match(RegVectMask);
 5492   match(pReg);
 5493   op_cost(0);
 5494   format %{ %}
 5495   interface(REG_INTER);
 5496 %}
 5497 
 5498 operand pRegGov_P0()
 5499 %{
 5500   constraint(ALLOC_IN_RC(p0_reg));
 5501   match(RegVectMask);
 5502   op_cost(0);
 5503   format %{ %}
 5504   interface(REG_INTER);
 5505 %}
 5506 
 5507 operand pRegGov_P1()
 5508 %{
 5509   constraint(ALLOC_IN_RC(p1_reg));
 5510   match(RegVectMask);
 5511   op_cost(0);
 5512   format %{ %}
 5513   interface(REG_INTER);
 5514 %}
 5515 
 5516 // Flags register, used as output of signed compare instructions
 5517 
 5518 // note that on AArch64 we also use this register as the output for
 5519 // for floating point compare instructions (CmpF CmpD). this ensures
 5520 // that ordered inequality tests use GT, GE, LT or LE none of which
 5521 // pass through cases where the result is unordered i.e. one or both
 5522 // inputs to the compare is a NaN. this means that the ideal code can
 5523 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5524 // (where the comparison should always fail). EQ and NE tests are
 5525 // always generated in ideal code so that unordered folds into the NE
 5526 // case, matching the behaviour of AArch64 NE.
 5527 //
 5528 // This differs from x86 where the outputs of FP compares use a
 5529 // special FP flags registers and where compares based on this
 5530 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5531 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5532 // to explicitly handle the unordered case in branches. x86 also has
 5533 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5534 
 5535 operand rFlagsReg()
 5536 %{
 5537   constraint(ALLOC_IN_RC(int_flags));
 5538   match(RegFlags);
 5539 
 5540   op_cost(0);
 5541   format %{ "RFLAGS" %}
 5542   interface(REG_INTER);
 5543 %}
 5544 
 5545 // Flags register, used as output of unsigned compare instructions
 5546 operand rFlagsRegU()
 5547 %{
 5548   constraint(ALLOC_IN_RC(int_flags));
 5549   match(RegFlags);
 5550 
 5551   op_cost(0);
 5552   format %{ "RFLAGSU" %}
 5553   interface(REG_INTER);
 5554 %}
 5555 
 5556 // Special Registers
 5557 
 5558 // Method Register
 5559 operand inline_cache_RegP(iRegP reg)
 5560 %{
 5561   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5562   match(reg);
 5563   match(iRegPNoSp);
 5564   op_cost(0);
 5565   format %{ %}
 5566   interface(REG_INTER);
 5567 %}
 5568 
 5569 // Thread Register
 5570 operand thread_RegP(iRegP reg)
 5571 %{
 5572   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5573   match(reg);
 5574   op_cost(0);
 5575   format %{ %}
 5576   interface(REG_INTER);
 5577 %}
 5578 
 5579 operand lr_RegP(iRegP reg)
 5580 %{
 5581   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5582   match(reg);
 5583   op_cost(0);
 5584   format %{ %}
 5585   interface(REG_INTER);
 5586 %}
 5587 
 5588 //----------Memory Operands----------------------------------------------------
 5589 
 5590 operand indirect(iRegP reg)
 5591 %{
 5592   constraint(ALLOC_IN_RC(ptr_reg));
 5593   match(reg);
 5594   op_cost(0);
 5595   format %{ "[$reg]" %}
 5596   interface(MEMORY_INTER) %{
 5597     base($reg);
 5598     index(0xffffffff);
 5599     scale(0x0);
 5600     disp(0x0);
 5601   %}
 5602 %}
 5603 
 5604 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5605 %{
 5606   constraint(ALLOC_IN_RC(ptr_reg));
 5607   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5608   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5609   op_cost(0);
 5610   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5611   interface(MEMORY_INTER) %{
 5612     base($reg);
 5613     index($ireg);
 5614     scale($scale);
 5615     disp(0x0);
 5616   %}
 5617 %}
 5618 
 5619 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5620 %{
 5621   constraint(ALLOC_IN_RC(ptr_reg));
 5622   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5623   match(AddP reg (LShiftL lreg scale));
 5624   op_cost(0);
 5625   format %{ "$reg, $lreg lsl($scale)" %}
 5626   interface(MEMORY_INTER) %{
 5627     base($reg);
 5628     index($lreg);
 5629     scale($scale);
 5630     disp(0x0);
 5631   %}
 5632 %}
 5633 
 5634 operand indIndexI2L(iRegP reg, iRegI ireg)
 5635 %{
 5636   constraint(ALLOC_IN_RC(ptr_reg));
 5637   match(AddP reg (ConvI2L ireg));
 5638   op_cost(0);
 5639   format %{ "$reg, $ireg, 0, I2L" %}
 5640   interface(MEMORY_INTER) %{
 5641     base($reg);
 5642     index($ireg);
 5643     scale(0x0);
 5644     disp(0x0);
 5645   %}
 5646 %}
 5647 
 5648 operand indIndex(iRegP reg, iRegL lreg)
 5649 %{
 5650   constraint(ALLOC_IN_RC(ptr_reg));
 5651   match(AddP reg lreg);
 5652   op_cost(0);
 5653   format %{ "$reg, $lreg" %}
 5654   interface(MEMORY_INTER) %{
 5655     base($reg);
 5656     index($lreg);
 5657     scale(0x0);
 5658     disp(0x0);
 5659   %}
 5660 %}
 5661 
 5662 operand indOffI(iRegP reg, immIOffset off)
 5663 %{
 5664   constraint(ALLOC_IN_RC(ptr_reg));
 5665   match(AddP reg off);
 5666   op_cost(0);
 5667   format %{ "[$reg, $off]" %}
 5668   interface(MEMORY_INTER) %{
 5669     base($reg);
 5670     index(0xffffffff);
 5671     scale(0x0);
 5672     disp($off);
 5673   %}
 5674 %}
 5675 
 5676 operand indOffI1(iRegP reg, immIOffset1 off)
 5677 %{
 5678   constraint(ALLOC_IN_RC(ptr_reg));
 5679   match(AddP reg off);
 5680   op_cost(0);
 5681   format %{ "[$reg, $off]" %}
 5682   interface(MEMORY_INTER) %{
 5683     base($reg);
 5684     index(0xffffffff);
 5685     scale(0x0);
 5686     disp($off);
 5687   %}
 5688 %}
 5689 
 5690 operand indOffI2(iRegP reg, immIOffset2 off)
 5691 %{
 5692   constraint(ALLOC_IN_RC(ptr_reg));
 5693   match(AddP reg off);
 5694   op_cost(0);
 5695   format %{ "[$reg, $off]" %}
 5696   interface(MEMORY_INTER) %{
 5697     base($reg);
 5698     index(0xffffffff);
 5699     scale(0x0);
 5700     disp($off);
 5701   %}
 5702 %}
 5703 
 5704 operand indOffI4(iRegP reg, immIOffset4 off)
 5705 %{
 5706   constraint(ALLOC_IN_RC(ptr_reg));
 5707   match(AddP reg off);
 5708   op_cost(0);
 5709   format %{ "[$reg, $off]" %}
 5710   interface(MEMORY_INTER) %{
 5711     base($reg);
 5712     index(0xffffffff);
 5713     scale(0x0);
 5714     disp($off);
 5715   %}
 5716 %}
 5717 
 5718 operand indOffI8(iRegP reg, immIOffset8 off)
 5719 %{
 5720   constraint(ALLOC_IN_RC(ptr_reg));
 5721   match(AddP reg off);
 5722   op_cost(0);
 5723   format %{ "[$reg, $off]" %}
 5724   interface(MEMORY_INTER) %{
 5725     base($reg);
 5726     index(0xffffffff);
 5727     scale(0x0);
 5728     disp($off);
 5729   %}
 5730 %}
 5731 
 5732 operand indOffI16(iRegP reg, immIOffset16 off)
 5733 %{
 5734   constraint(ALLOC_IN_RC(ptr_reg));
 5735   match(AddP reg off);
 5736   op_cost(0);
 5737   format %{ "[$reg, $off]" %}
 5738   interface(MEMORY_INTER) %{
 5739     base($reg);
 5740     index(0xffffffff);
 5741     scale(0x0);
 5742     disp($off);
 5743   %}
 5744 %}
 5745 
 5746 operand indOffL(iRegP reg, immLoffset off)
 5747 %{
 5748   constraint(ALLOC_IN_RC(ptr_reg));
 5749   match(AddP reg off);
 5750   op_cost(0);
 5751   format %{ "[$reg, $off]" %}
 5752   interface(MEMORY_INTER) %{
 5753     base($reg);
 5754     index(0xffffffff);
 5755     scale(0x0);
 5756     disp($off);
 5757   %}
 5758 %}
 5759 
 5760 operand indOffL1(iRegP reg, immLoffset1 off)
 5761 %{
 5762   constraint(ALLOC_IN_RC(ptr_reg));
 5763   match(AddP reg off);
 5764   op_cost(0);
 5765   format %{ "[$reg, $off]" %}
 5766   interface(MEMORY_INTER) %{
 5767     base($reg);
 5768     index(0xffffffff);
 5769     scale(0x0);
 5770     disp($off);
 5771   %}
 5772 %}
 5773 
 5774 operand indOffL2(iRegP reg, immLoffset2 off)
 5775 %{
 5776   constraint(ALLOC_IN_RC(ptr_reg));
 5777   match(AddP reg off);
 5778   op_cost(0);
 5779   format %{ "[$reg, $off]" %}
 5780   interface(MEMORY_INTER) %{
 5781     base($reg);
 5782     index(0xffffffff);
 5783     scale(0x0);
 5784     disp($off);
 5785   %}
 5786 %}
 5787 
 5788 operand indOffL4(iRegP reg, immLoffset4 off)
 5789 %{
 5790   constraint(ALLOC_IN_RC(ptr_reg));
 5791   match(AddP reg off);
 5792   op_cost(0);
 5793   format %{ "[$reg, $off]" %}
 5794   interface(MEMORY_INTER) %{
 5795     base($reg);
 5796     index(0xffffffff);
 5797     scale(0x0);
 5798     disp($off);
 5799   %}
 5800 %}
 5801 
 5802 operand indOffL8(iRegP reg, immLoffset8 off)
 5803 %{
 5804   constraint(ALLOC_IN_RC(ptr_reg));
 5805   match(AddP reg off);
 5806   op_cost(0);
 5807   format %{ "[$reg, $off]" %}
 5808   interface(MEMORY_INTER) %{
 5809     base($reg);
 5810     index(0xffffffff);
 5811     scale(0x0);
 5812     disp($off);
 5813   %}
 5814 %}
 5815 
 5816 operand indOffL16(iRegP reg, immLoffset16 off)
 5817 %{
 5818   constraint(ALLOC_IN_RC(ptr_reg));
 5819   match(AddP reg off);
 5820   op_cost(0);
 5821   format %{ "[$reg, $off]" %}
 5822   interface(MEMORY_INTER) %{
 5823     base($reg);
 5824     index(0xffffffff);
 5825     scale(0x0);
 5826     disp($off);
 5827   %}
 5828 %}
 5829 
 5830 operand indirectN(iRegN reg)
 5831 %{
 5832   predicate(CompressedOops::shift() == 0);
 5833   constraint(ALLOC_IN_RC(ptr_reg));
 5834   match(DecodeN reg);
 5835   op_cost(0);
 5836   format %{ "[$reg]\t# narrow" %}
 5837   interface(MEMORY_INTER) %{
 5838     base($reg);
 5839     index(0xffffffff);
 5840     scale(0x0);
 5841     disp(0x0);
 5842   %}
 5843 %}
 5844 
 5845 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5846 %{
 5847   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5848   constraint(ALLOC_IN_RC(ptr_reg));
 5849   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5850   op_cost(0);
 5851   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5852   interface(MEMORY_INTER) %{
 5853     base($reg);
 5854     index($ireg);
 5855     scale($scale);
 5856     disp(0x0);
 5857   %}
 5858 %}
 5859 
 5860 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5861 %{
 5862   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5863   constraint(ALLOC_IN_RC(ptr_reg));
 5864   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5865   op_cost(0);
 5866   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5867   interface(MEMORY_INTER) %{
 5868     base($reg);
 5869     index($lreg);
 5870     scale($scale);
 5871     disp(0x0);
 5872   %}
 5873 %}
 5874 
 5875 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5876 %{
 5877   predicate(CompressedOops::shift() == 0);
 5878   constraint(ALLOC_IN_RC(ptr_reg));
 5879   match(AddP (DecodeN reg) (ConvI2L ireg));
 5880   op_cost(0);
 5881   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5882   interface(MEMORY_INTER) %{
 5883     base($reg);
 5884     index($ireg);
 5885     scale(0x0);
 5886     disp(0x0);
 5887   %}
 5888 %}
 5889 
 5890 operand indIndexN(iRegN reg, iRegL lreg)
 5891 %{
 5892   predicate(CompressedOops::shift() == 0);
 5893   constraint(ALLOC_IN_RC(ptr_reg));
 5894   match(AddP (DecodeN reg) lreg);
 5895   op_cost(0);
 5896   format %{ "$reg, $lreg\t# narrow" %}
 5897   interface(MEMORY_INTER) %{
 5898     base($reg);
 5899     index($lreg);
 5900     scale(0x0);
 5901     disp(0x0);
 5902   %}
 5903 %}
 5904 
 5905 operand indOffIN(iRegN reg, immIOffset off)
 5906 %{
 5907   predicate(CompressedOops::shift() == 0);
 5908   constraint(ALLOC_IN_RC(ptr_reg));
 5909   match(AddP (DecodeN reg) off);
 5910   op_cost(0);
 5911   format %{ "[$reg, $off]\t# narrow" %}
 5912   interface(MEMORY_INTER) %{
 5913     base($reg);
 5914     index(0xffffffff);
 5915     scale(0x0);
 5916     disp($off);
 5917   %}
 5918 %}
 5919 
 5920 operand indOffLN(iRegN reg, immLoffset off)
 5921 %{
 5922   predicate(CompressedOops::shift() == 0);
 5923   constraint(ALLOC_IN_RC(ptr_reg));
 5924   match(AddP (DecodeN reg) off);
 5925   op_cost(0);
 5926   format %{ "[$reg, $off]\t# narrow" %}
 5927   interface(MEMORY_INTER) %{
 5928     base($reg);
 5929     index(0xffffffff);
 5930     scale(0x0);
 5931     disp($off);
 5932   %}
 5933 %}
 5934 
 5935 
 5936 
 5937 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 5938 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 5939 %{
 5940   constraint(ALLOC_IN_RC(ptr_reg));
 5941   match(AddP reg off);
 5942   op_cost(0);
 5943   format %{ "[$reg, $off]" %}
 5944   interface(MEMORY_INTER) %{
 5945     base($reg);
 5946     index(0xffffffff);
 5947     scale(0x0);
 5948     disp($off);
 5949   %}
 5950 %}
 5951 
 5952 //----------Special Memory Operands--------------------------------------------
 5953 // Stack Slot Operand - This operand is used for loading and storing temporary
 5954 //                      values on the stack where a match requires a value to
 5955 //                      flow through memory.
 5956 operand stackSlotP(sRegP reg)
 5957 %{
 5958   constraint(ALLOC_IN_RC(stack_slots));
 5959   op_cost(100);
 5960   // No match rule because this operand is only generated in matching
 5961   // match(RegP);
 5962   format %{ "[$reg]" %}
 5963   interface(MEMORY_INTER) %{
 5964     base(0x1e);  // RSP
 5965     index(0x0);  // No Index
 5966     scale(0x0);  // No Scale
 5967     disp($reg);  // Stack Offset
 5968   %}
 5969 %}
 5970 
 5971 operand stackSlotI(sRegI reg)
 5972 %{
 5973   constraint(ALLOC_IN_RC(stack_slots));
 5974   // No match rule because this operand is only generated in matching
 5975   // match(RegI);
 5976   format %{ "[$reg]" %}
 5977   interface(MEMORY_INTER) %{
 5978     base(0x1e);  // RSP
 5979     index(0x0);  // No Index
 5980     scale(0x0);  // No Scale
 5981     disp($reg);  // Stack Offset
 5982   %}
 5983 %}
 5984 
 5985 operand stackSlotF(sRegF reg)
 5986 %{
 5987   constraint(ALLOC_IN_RC(stack_slots));
 5988   // No match rule because this operand is only generated in matching
 5989   // match(RegF);
 5990   format %{ "[$reg]" %}
 5991   interface(MEMORY_INTER) %{
 5992     base(0x1e);  // RSP
 5993     index(0x0);  // No Index
 5994     scale(0x0);  // No Scale
 5995     disp($reg);  // Stack Offset
 5996   %}
 5997 %}
 5998 
 5999 operand stackSlotD(sRegD reg)
 6000 %{
 6001   constraint(ALLOC_IN_RC(stack_slots));
 6002   // No match rule because this operand is only generated in matching
 6003   // match(RegD);
 6004   format %{ "[$reg]" %}
 6005   interface(MEMORY_INTER) %{
 6006     base(0x1e);  // RSP
 6007     index(0x0);  // No Index
 6008     scale(0x0);  // No Scale
 6009     disp($reg);  // Stack Offset
 6010   %}
 6011 %}
 6012 
 6013 operand stackSlotL(sRegL reg)
 6014 %{
 6015   constraint(ALLOC_IN_RC(stack_slots));
 6016   // No match rule because this operand is only generated in matching
 6017   // match(RegL);
 6018   format %{ "[$reg]" %}
 6019   interface(MEMORY_INTER) %{
 6020     base(0x1e);  // RSP
 6021     index(0x0);  // No Index
 6022     scale(0x0);  // No Scale
 6023     disp($reg);  // Stack Offset
 6024   %}
 6025 %}
 6026 
 6027 // Operands for expressing Control Flow
 6028 // NOTE: Label is a predefined operand which should not be redefined in
 6029 //       the AD file. It is generically handled within the ADLC.
 6030 
 6031 //----------Conditional Branch Operands----------------------------------------
 6032 // Comparison Op  - This is the operation of the comparison, and is limited to
 6033 //                  the following set of codes:
 6034 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6035 //
 6036 // Other attributes of the comparison, such as unsignedness, are specified
 6037 // by the comparison instruction that sets a condition code flags register.
 6038 // That result is represented by a flags operand whose subtype is appropriate
 6039 // to the unsignedness (etc.) of the comparison.
 6040 //
 6041 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6042 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6043 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6044 
 6045 // used for signed integral comparisons and fp comparisons
 6046 
 6047 operand cmpOp()
 6048 %{
 6049   match(Bool);
 6050 
 6051   format %{ "" %}
 6052   interface(COND_INTER) %{
 6053     equal(0x0, "eq");
 6054     not_equal(0x1, "ne");
 6055     less(0xb, "lt");
 6056     greater_equal(0xa, "ge");
 6057     less_equal(0xd, "le");
 6058     greater(0xc, "gt");
 6059     overflow(0x6, "vs");
 6060     no_overflow(0x7, "vc");
 6061   %}
 6062 %}
 6063 
 6064 // used for unsigned integral comparisons
 6065 
 6066 operand cmpOpU()
 6067 %{
 6068   match(Bool);
 6069 
 6070   format %{ "" %}
 6071   interface(COND_INTER) %{
 6072     equal(0x0, "eq");
 6073     not_equal(0x1, "ne");
 6074     less(0x3, "lo");
 6075     greater_equal(0x2, "hs");
 6076     less_equal(0x9, "ls");
 6077     greater(0x8, "hi");
 6078     overflow(0x6, "vs");
 6079     no_overflow(0x7, "vc");
 6080   %}
 6081 %}
 6082 
 6083 // used for certain integral comparisons which can be
 6084 // converted to cbxx or tbxx instructions
 6085 
 6086 operand cmpOpEqNe()
 6087 %{
 6088   match(Bool);
 6089   op_cost(0);
 6090   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6091             || n->as_Bool()->_test._test == BoolTest::eq);
 6092 
 6093   format %{ "" %}
 6094   interface(COND_INTER) %{
 6095     equal(0x0, "eq");
 6096     not_equal(0x1, "ne");
 6097     less(0xb, "lt");
 6098     greater_equal(0xa, "ge");
 6099     less_equal(0xd, "le");
 6100     greater(0xc, "gt");
 6101     overflow(0x6, "vs");
 6102     no_overflow(0x7, "vc");
 6103   %}
 6104 %}
 6105 
 6106 // used for certain integral comparisons which can be
 6107 // converted to cbxx or tbxx instructions
 6108 
 6109 operand cmpOpLtGe()
 6110 %{
 6111   match(Bool);
 6112   op_cost(0);
 6113 
 6114   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6115             || n->as_Bool()->_test._test == BoolTest::ge);
 6116 
 6117   format %{ "" %}
 6118   interface(COND_INTER) %{
 6119     equal(0x0, "eq");
 6120     not_equal(0x1, "ne");
 6121     less(0xb, "lt");
 6122     greater_equal(0xa, "ge");
 6123     less_equal(0xd, "le");
 6124     greater(0xc, "gt");
 6125     overflow(0x6, "vs");
 6126     no_overflow(0x7, "vc");
 6127   %}
 6128 %}
 6129 
 6130 // used for certain unsigned integral comparisons which can be
 6131 // converted to cbxx or tbxx instructions
 6132 
 6133 operand cmpOpUEqNeLtGe()
 6134 %{
 6135   match(Bool);
 6136   op_cost(0);
 6137 
 6138   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6139             || n->as_Bool()->_test._test == BoolTest::ne
 6140             || n->as_Bool()->_test._test == BoolTest::lt
 6141             || n->as_Bool()->_test._test == BoolTest::ge);
 6142 
 6143   format %{ "" %}
 6144   interface(COND_INTER) %{
 6145     equal(0x0, "eq");
 6146     not_equal(0x1, "ne");
 6147     less(0xb, "lt");
 6148     greater_equal(0xa, "ge");
 6149     less_equal(0xd, "le");
 6150     greater(0xc, "gt");
 6151     overflow(0x6, "vs");
 6152     no_overflow(0x7, "vc");
 6153   %}
 6154 %}
 6155 
 6156 // Special operand allowing long args to int ops to be truncated for free
 6157 
 6158 operand iRegL2I(iRegL reg) %{
 6159 
 6160   op_cost(0);
 6161 
 6162   match(ConvL2I reg);
 6163 
 6164   format %{ "l2i($reg)" %}
 6165 
 6166   interface(REG_INTER)
 6167 %}
 6168 
 6169 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6170 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6171 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6172 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6173 
 6174 //----------OPERAND CLASSES----------------------------------------------------
 6175 // Operand Classes are groups of operands that are used as to simplify
 6176 // instruction definitions by not requiring the AD writer to specify
 6177 // separate instructions for every form of operand when the
 6178 // instruction accepts multiple operand types with the same basic
 6179 // encoding and format. The classic case of this is memory operands.
 6180 
 6181 // memory is used to define read/write location for load/store
 6182 // instruction defs. we can turn a memory op into an Address
 6183 
 6184 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6185                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6186 
 6187 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6188                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6189 
 6190 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6191                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6192 
 6193 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6194                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6195 
 6196 // All of the memory operands. For the pipeline description.
 6197 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6198                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6199                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6200 
 6201 
 6202 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6203 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6204 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6205 // can be elided because the 32-bit instruction will just employ the
 6206 // lower 32 bits anyway.
 6207 //
 6208 // n.b. this does not elide all L2I conversions. if the truncated
 6209 // value is consumed by more than one operation then the ConvL2I
 6210 // cannot be bundled into the consuming nodes so an l2i gets planted
 6211 // (actually a movw $dst $src) and the downstream instructions consume
 6212 // the result of the l2i as an iRegI input. That's a shame since the
 6213 // movw is actually redundant but its not too costly.
 6214 
 6215 opclass iRegIorL2I(iRegI, iRegL2I);
 6216 
 6217 //----------PIPELINE-----------------------------------------------------------
 6218 // Rules which define the behavior of the target architectures pipeline.
 6219 
 6220 // For specific pipelines, eg A53, define the stages of that pipeline
 6221 //pipe_desc(ISS, EX1, EX2, WR);
 6222 #define ISS S0
 6223 #define EX1 S1
 6224 #define EX2 S2
 6225 #define WR  S3
 6226 
 6227 // Integer ALU reg operation
 6228 pipeline %{
 6229 
 6230 attributes %{
 6231   // ARM instructions are of fixed length
 6232   fixed_size_instructions;        // Fixed size instructions TODO does
 6233   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6234   // ARM instructions come in 32-bit word units
 6235   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6236   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6237   instruction_fetch_units = 1;       // of 64 bytes
 6238 
 6239   // List of nop instructions
 6240   nops( MachNop );
 6241 %}
 6242 
 6243 // We don't use an actual pipeline model so don't care about resources
 6244 // or description. we do use pipeline classes to introduce fixed
 6245 // latencies
 6246 
 6247 //----------RESOURCES----------------------------------------------------------
 6248 // Resources are the functional units available to the machine
 6249 
 6250 resources( INS0, INS1, INS01 = INS0 | INS1,
 6251            ALU0, ALU1, ALU = ALU0 | ALU1,
 6252            MAC,
 6253            DIV,
 6254            BRANCH,
 6255            LDST,
 6256            NEON_FP);
 6257 
 6258 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6259 // Pipeline Description specifies the stages in the machine's pipeline
 6260 
 6261 // Define the pipeline as a generic 6 stage pipeline
 6262 pipe_desc(S0, S1, S2, S3, S4, S5);
 6263 
 6264 //----------PIPELINE CLASSES---------------------------------------------------
 6265 // Pipeline Classes describe the stages in which input and output are
 6266 // referenced by the hardware pipeline.
 6267 
 6268 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6269 %{
 6270   single_instruction;
 6271   src1   : S1(read);
 6272   src2   : S2(read);
 6273   dst    : S5(write);
 6274   INS01  : ISS;
 6275   NEON_FP : S5;
 6276 %}
 6277 
 6278 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6279 %{
 6280   single_instruction;
 6281   src1   : S1(read);
 6282   src2   : S2(read);
 6283   dst    : S5(write);
 6284   INS01  : ISS;
 6285   NEON_FP : S5;
 6286 %}
 6287 
 6288 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6289 %{
 6290   single_instruction;
 6291   src    : S1(read);
 6292   dst    : S5(write);
 6293   INS01  : ISS;
 6294   NEON_FP : S5;
 6295 %}
 6296 
 6297 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6298 %{
 6299   single_instruction;
 6300   src    : S1(read);
 6301   dst    : S5(write);
 6302   INS01  : ISS;
 6303   NEON_FP : S5;
 6304 %}
 6305 
 6306 pipe_class fp_d2f(vRegF dst, vRegD src)
 6307 %{
 6308   single_instruction;
 6309   src    : S1(read);
 6310   dst    : S5(write);
 6311   INS01  : ISS;
 6312   NEON_FP : S5;
 6313 %}
 6314 
 6315 pipe_class fp_f2d(vRegD dst, vRegF src)
 6316 %{
 6317   single_instruction;
 6318   src    : S1(read);
 6319   dst    : S5(write);
 6320   INS01  : ISS;
 6321   NEON_FP : S5;
 6322 %}
 6323 
 6324 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6325 %{
 6326   single_instruction;
 6327   src    : S1(read);
 6328   dst    : S5(write);
 6329   INS01  : ISS;
 6330   NEON_FP : S5;
 6331 %}
 6332 
 6333 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6334 %{
 6335   single_instruction;
 6336   src    : S1(read);
 6337   dst    : S5(write);
 6338   INS01  : ISS;
 6339   NEON_FP : S5;
 6340 %}
 6341 
 6342 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6343 %{
 6344   single_instruction;
 6345   src    : S1(read);
 6346   dst    : S5(write);
 6347   INS01  : ISS;
 6348   NEON_FP : S5;
 6349 %}
 6350 
 6351 pipe_class fp_l2f(vRegF dst, iRegL src)
 6352 %{
 6353   single_instruction;
 6354   src    : S1(read);
 6355   dst    : S5(write);
 6356   INS01  : ISS;
 6357   NEON_FP : S5;
 6358 %}
 6359 
 6360 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6361 %{
 6362   single_instruction;
 6363   src    : S1(read);
 6364   dst    : S5(write);
 6365   INS01  : ISS;
 6366   NEON_FP : S5;
 6367 %}
 6368 
 6369 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6370 %{
 6371   single_instruction;
 6372   src    : S1(read);
 6373   dst    : S5(write);
 6374   INS01  : ISS;
 6375   NEON_FP : S5;
 6376 %}
 6377 
 6378 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6379 %{
 6380   single_instruction;
 6381   src    : S1(read);
 6382   dst    : S5(write);
 6383   INS01  : ISS;
 6384   NEON_FP : S5;
 6385 %}
 6386 
 6387 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6388 %{
 6389   single_instruction;
 6390   src    : S1(read);
 6391   dst    : S5(write);
 6392   INS01  : ISS;
 6393   NEON_FP : S5;
 6394 %}
 6395 
 6396 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6397 %{
 6398   single_instruction;
 6399   src1   : S1(read);
 6400   src2   : S2(read);
 6401   dst    : S5(write);
 6402   INS0   : ISS;
 6403   NEON_FP : S5;
 6404 %}
 6405 
 6406 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6407 %{
 6408   single_instruction;
 6409   src1   : S1(read);
 6410   src2   : S2(read);
 6411   dst    : S5(write);
 6412   INS0   : ISS;
 6413   NEON_FP : S5;
 6414 %}
 6415 
 6416 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6417 %{
 6418   single_instruction;
 6419   cr     : S1(read);
 6420   src1   : S1(read);
 6421   src2   : S1(read);
 6422   dst    : S3(write);
 6423   INS01  : ISS;
 6424   NEON_FP : S3;
 6425 %}
 6426 
 6427 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6428 %{
 6429   single_instruction;
 6430   cr     : S1(read);
 6431   src1   : S1(read);
 6432   src2   : S1(read);
 6433   dst    : S3(write);
 6434   INS01  : ISS;
 6435   NEON_FP : S3;
 6436 %}
 6437 
 6438 pipe_class fp_imm_s(vRegF dst)
 6439 %{
 6440   single_instruction;
 6441   dst    : S3(write);
 6442   INS01  : ISS;
 6443   NEON_FP : S3;
 6444 %}
 6445 
 6446 pipe_class fp_imm_d(vRegD dst)
 6447 %{
 6448   single_instruction;
 6449   dst    : S3(write);
 6450   INS01  : ISS;
 6451   NEON_FP : S3;
 6452 %}
 6453 
 6454 pipe_class fp_load_constant_s(vRegF dst)
 6455 %{
 6456   single_instruction;
 6457   dst    : S4(write);
 6458   INS01  : ISS;
 6459   NEON_FP : S4;
 6460 %}
 6461 
 6462 pipe_class fp_load_constant_d(vRegD dst)
 6463 %{
 6464   single_instruction;
 6465   dst    : S4(write);
 6466   INS01  : ISS;
 6467   NEON_FP : S4;
 6468 %}
 6469 
 6470 //------- Integer ALU operations --------------------------
 6471 
 6472 // Integer ALU reg-reg operation
 6473 // Operands needed in EX1, result generated in EX2
 6474 // Eg.  ADD     x0, x1, x2
 6475 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6476 %{
 6477   single_instruction;
 6478   dst    : EX2(write);
 6479   src1   : EX1(read);
 6480   src2   : EX1(read);
 6481   INS01  : ISS; // Dual issue as instruction 0 or 1
 6482   ALU    : EX2;
 6483 %}
 6484 
 6485 // Integer ALU reg-reg operation with constant shift
 6486 // Shifted register must be available in LATE_ISS instead of EX1
 6487 // Eg.  ADD     x0, x1, x2, LSL #2
 6488 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6489 %{
 6490   single_instruction;
 6491   dst    : EX2(write);
 6492   src1   : EX1(read);
 6493   src2   : ISS(read);
 6494   INS01  : ISS;
 6495   ALU    : EX2;
 6496 %}
 6497 
 6498 // Integer ALU reg operation with constant shift
 6499 // Eg.  LSL     x0, x1, #shift
 6500 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6501 %{
 6502   single_instruction;
 6503   dst    : EX2(write);
 6504   src1   : ISS(read);
 6505   INS01  : ISS;
 6506   ALU    : EX2;
 6507 %}
 6508 
 6509 // Integer ALU reg-reg operation with variable shift
 6510 // Both operands must be available in LATE_ISS instead of EX1
 6511 // Result is available in EX1 instead of EX2
 6512 // Eg.  LSLV    x0, x1, x2
 6513 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6514 %{
 6515   single_instruction;
 6516   dst    : EX1(write);
 6517   src1   : ISS(read);
 6518   src2   : ISS(read);
 6519   INS01  : ISS;
 6520   ALU    : EX1;
 6521 %}
 6522 
 6523 // Integer ALU reg-reg operation with extract
 6524 // As for _vshift above, but result generated in EX2
 6525 // Eg.  EXTR    x0, x1, x2, #N
 6526 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6527 %{
 6528   single_instruction;
 6529   dst    : EX2(write);
 6530   src1   : ISS(read);
 6531   src2   : ISS(read);
 6532   INS1   : ISS; // Can only dual issue as Instruction 1
 6533   ALU    : EX1;
 6534 %}
 6535 
 6536 // Integer ALU reg operation
 6537 // Eg.  NEG     x0, x1
 6538 pipe_class ialu_reg(iRegI dst, iRegI src)
 6539 %{
 6540   single_instruction;
 6541   dst    : EX2(write);
 6542   src    : EX1(read);
 6543   INS01  : ISS;
 6544   ALU    : EX2;
 6545 %}
 6546 
 6547 // Integer ALU reg mmediate operation
 6548 // Eg.  ADD     x0, x1, #N
 6549 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6550 %{
 6551   single_instruction;
 6552   dst    : EX2(write);
 6553   src1   : EX1(read);
 6554   INS01  : ISS;
 6555   ALU    : EX2;
 6556 %}
 6557 
 6558 // Integer ALU immediate operation (no source operands)
 6559 // Eg.  MOV     x0, #N
 6560 pipe_class ialu_imm(iRegI dst)
 6561 %{
 6562   single_instruction;
 6563   dst    : EX1(write);
 6564   INS01  : ISS;
 6565   ALU    : EX1;
 6566 %}
 6567 
 6568 //------- Compare operation -------------------------------
 6569 
 6570 // Compare reg-reg
 6571 // Eg.  CMP     x0, x1
 6572 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6573 %{
 6574   single_instruction;
 6575 //  fixed_latency(16);
 6576   cr     : EX2(write);
 6577   op1    : EX1(read);
 6578   op2    : EX1(read);
 6579   INS01  : ISS;
 6580   ALU    : EX2;
 6581 %}
 6582 
 6583 // Compare reg-reg
 6584 // Eg.  CMP     x0, #N
 6585 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6586 %{
 6587   single_instruction;
 6588 //  fixed_latency(16);
 6589   cr     : EX2(write);
 6590   op1    : EX1(read);
 6591   INS01  : ISS;
 6592   ALU    : EX2;
 6593 %}
 6594 
 6595 //------- Conditional instructions ------------------------
 6596 
 6597 // Conditional no operands
 6598 // Eg.  CSINC   x0, zr, zr, <cond>
 6599 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6600 %{
 6601   single_instruction;
 6602   cr     : EX1(read);
 6603   dst    : EX2(write);
 6604   INS01  : ISS;
 6605   ALU    : EX2;
 6606 %}
 6607 
 6608 // Conditional 2 operand
 6609 // EG.  CSEL    X0, X1, X2, <cond>
 6610 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6611 %{
 6612   single_instruction;
 6613   cr     : EX1(read);
 6614   src1   : EX1(read);
 6615   src2   : EX1(read);
 6616   dst    : EX2(write);
 6617   INS01  : ISS;
 6618   ALU    : EX2;
 6619 %}
 6620 
 6621 // Conditional 2 operand
 6622 // EG.  CSEL    X0, X1, X2, <cond>
 6623 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6624 %{
 6625   single_instruction;
 6626   cr     : EX1(read);
 6627   src    : EX1(read);
 6628   dst    : EX2(write);
 6629   INS01  : ISS;
 6630   ALU    : EX2;
 6631 %}
 6632 
 6633 //------- Multiply pipeline operations --------------------
 6634 
 6635 // Multiply reg-reg
 6636 // Eg.  MUL     w0, w1, w2
 6637 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6638 %{
 6639   single_instruction;
 6640   dst    : WR(write);
 6641   src1   : ISS(read);
 6642   src2   : ISS(read);
 6643   INS01  : ISS;
 6644   MAC    : WR;
 6645 %}
 6646 
 6647 // Multiply accumulate
 6648 // Eg.  MADD    w0, w1, w2, w3
 6649 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6650 %{
 6651   single_instruction;
 6652   dst    : WR(write);
 6653   src1   : ISS(read);
 6654   src2   : ISS(read);
 6655   src3   : ISS(read);
 6656   INS01  : ISS;
 6657   MAC    : WR;
 6658 %}
 6659 
 6660 // Eg.  MUL     w0, w1, w2
 6661 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6662 %{
 6663   single_instruction;
 6664   fixed_latency(3); // Maximum latency for 64 bit mul
 6665   dst    : WR(write);
 6666   src1   : ISS(read);
 6667   src2   : ISS(read);
 6668   INS01  : ISS;
 6669   MAC    : WR;
 6670 %}
 6671 
 6672 // Multiply accumulate
 6673 // Eg.  MADD    w0, w1, w2, w3
 6674 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6675 %{
 6676   single_instruction;
 6677   fixed_latency(3); // Maximum latency for 64 bit mul
 6678   dst    : WR(write);
 6679   src1   : ISS(read);
 6680   src2   : ISS(read);
 6681   src3   : ISS(read);
 6682   INS01  : ISS;
 6683   MAC    : WR;
 6684 %}
 6685 
 6686 //------- Divide pipeline operations --------------------
 6687 
 6688 // Eg.  SDIV    w0, w1, w2
 6689 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6690 %{
 6691   single_instruction;
 6692   fixed_latency(8); // Maximum latency for 32 bit divide
 6693   dst    : WR(write);
 6694   src1   : ISS(read);
 6695   src2   : ISS(read);
 6696   INS0   : ISS; // Can only dual issue as instruction 0
 6697   DIV    : WR;
 6698 %}
 6699 
 6700 // Eg.  SDIV    x0, x1, x2
 6701 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6702 %{
 6703   single_instruction;
 6704   fixed_latency(16); // Maximum latency for 64 bit divide
 6705   dst    : WR(write);
 6706   src1   : ISS(read);
 6707   src2   : ISS(read);
 6708   INS0   : ISS; // Can only dual issue as instruction 0
 6709   DIV    : WR;
 6710 %}
 6711 
 6712 //------- Load pipeline operations ------------------------
 6713 
 6714 // Load - prefetch
 6715 // Eg.  PFRM    <mem>
 6716 pipe_class iload_prefetch(memory mem)
 6717 %{
 6718   single_instruction;
 6719   mem    : ISS(read);
 6720   INS01  : ISS;
 6721   LDST   : WR;
 6722 %}
 6723 
 6724 // Load - reg, mem
 6725 // Eg.  LDR     x0, <mem>
 6726 pipe_class iload_reg_mem(iRegI dst, memory mem)
 6727 %{
 6728   single_instruction;
 6729   dst    : WR(write);
 6730   mem    : ISS(read);
 6731   INS01  : ISS;
 6732   LDST   : WR;
 6733 %}
 6734 
 6735 // Load - reg, reg
 6736 // Eg.  LDR     x0, [sp, x1]
 6737 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 6738 %{
 6739   single_instruction;
 6740   dst    : WR(write);
 6741   src    : ISS(read);
 6742   INS01  : ISS;
 6743   LDST   : WR;
 6744 %}
 6745 
 6746 //------- Store pipeline operations -----------------------
 6747 
 6748 // Store - zr, mem
 6749 // Eg.  STR     zr, <mem>
 6750 pipe_class istore_mem(memory mem)
 6751 %{
 6752   single_instruction;
 6753   mem    : ISS(read);
 6754   INS01  : ISS;
 6755   LDST   : WR;
 6756 %}
 6757 
 6758 // Store - reg, mem
 6759 // Eg.  STR     x0, <mem>
 6760 pipe_class istore_reg_mem(iRegI src, memory mem)
 6761 %{
 6762   single_instruction;
 6763   mem    : ISS(read);
 6764   src    : EX2(read);
 6765   INS01  : ISS;
 6766   LDST   : WR;
 6767 %}
 6768 
 6769 // Store - reg, reg
 6770 // Eg. STR      x0, [sp, x1]
 6771 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 6772 %{
 6773   single_instruction;
 6774   dst    : ISS(read);
 6775   src    : EX2(read);
 6776   INS01  : ISS;
 6777   LDST   : WR;
 6778 %}
 6779 
 6780 //------- Store pipeline operations -----------------------
 6781 
 6782 // Branch
 6783 pipe_class pipe_branch()
 6784 %{
 6785   single_instruction;
 6786   INS01  : ISS;
 6787   BRANCH : EX1;
 6788 %}
 6789 
 6790 // Conditional branch
 6791 pipe_class pipe_branch_cond(rFlagsReg cr)
 6792 %{
 6793   single_instruction;
 6794   cr     : EX1(read);
 6795   INS01  : ISS;
 6796   BRANCH : EX1;
 6797 %}
 6798 
 6799 // Compare & Branch
 6800 // EG.  CBZ/CBNZ
 6801 pipe_class pipe_cmp_branch(iRegI op1)
 6802 %{
 6803   single_instruction;
 6804   op1    : EX1(read);
 6805   INS01  : ISS;
 6806   BRANCH : EX1;
 6807 %}
 6808 
 6809 //------- Synchronisation operations ----------------------
 6810 
 6811 // Any operation requiring serialization.
 6812 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 6813 pipe_class pipe_serial()
 6814 %{
 6815   single_instruction;
 6816   force_serialization;
 6817   fixed_latency(16);
 6818   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6819   LDST   : WR;
 6820 %}
 6821 
 6822 // Generic big/slow expanded idiom - also serialized
 6823 pipe_class pipe_slow()
 6824 %{
 6825   instruction_count(10);
 6826   multiple_bundles;
 6827   force_serialization;
 6828   fixed_latency(16);
 6829   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6830   LDST   : WR;
 6831 %}
 6832 
 6833 // Empty pipeline class
 6834 pipe_class pipe_class_empty()
 6835 %{
 6836   single_instruction;
 6837   fixed_latency(0);
 6838 %}
 6839 
 6840 // Default pipeline class.
 6841 pipe_class pipe_class_default()
 6842 %{
 6843   single_instruction;
 6844   fixed_latency(2);
 6845 %}
 6846 
 6847 // Pipeline class for compares.
 6848 pipe_class pipe_class_compare()
 6849 %{
 6850   single_instruction;
 6851   fixed_latency(16);
 6852 %}
 6853 
 6854 // Pipeline class for memory operations.
 6855 pipe_class pipe_class_memory()
 6856 %{
 6857   single_instruction;
 6858   fixed_latency(16);
 6859 %}
 6860 
 6861 // Pipeline class for call.
 6862 pipe_class pipe_class_call()
 6863 %{
 6864   single_instruction;
 6865   fixed_latency(100);
 6866 %}
 6867 
 6868 // Define the class for the Nop node.
 6869 define %{
 6870    MachNop = pipe_class_empty;
 6871 %}
 6872 
 6873 %}
 6874 //----------INSTRUCTIONS-------------------------------------------------------
 6875 //
 6876 // match      -- States which machine-independent subtree may be replaced
 6877 //               by this instruction.
 6878 // ins_cost   -- The estimated cost of this instruction is used by instruction
 6879 //               selection to identify a minimum cost tree of machine
 6880 //               instructions that matches a tree of machine-independent
 6881 //               instructions.
 6882 // format     -- A string providing the disassembly for this instruction.
 6883 //               The value of an instruction's operand may be inserted
 6884 //               by referring to it with a '$' prefix.
 6885 // opcode     -- Three instruction opcodes may be provided.  These are referred
 6886 //               to within an encode class as $primary, $secondary, and $tertiary
 6887 //               rrspectively.  The primary opcode is commonly used to
 6888 //               indicate the type of machine instruction, while secondary
 6889 //               and tertiary are often used for prefix options or addressing
 6890 //               modes.
 6891 // ins_encode -- A list of encode classes with parameters. The encode class
 6892 //               name must have been defined in an 'enc_class' specification
 6893 //               in the encode section of the architecture description.
 6894 
 6895 // ============================================================================
 6896 // Memory (Load/Store) Instructions
 6897 
 6898 // Load Instructions
 6899 
 6900 // Load Byte (8 bit signed)
 6901 instruct loadB(iRegINoSp dst, memory1 mem)
 6902 %{
 6903   match(Set dst (LoadB mem));
 6904   predicate(!needs_acquiring_load(n));
 6905 
 6906   ins_cost(4 * INSN_COST);
 6907   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 6908 
 6909   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 6910 
 6911   ins_pipe(iload_reg_mem);
 6912 %}
 6913 
 6914 // Load Byte (8 bit signed) into long
 6915 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 6916 %{
 6917   match(Set dst (ConvI2L (LoadB mem)));
 6918   predicate(!needs_acquiring_load(n->in(1)));
 6919 
 6920   ins_cost(4 * INSN_COST);
 6921   format %{ "ldrsb  $dst, $mem\t# byte" %}
 6922 
 6923   ins_encode(aarch64_enc_ldrsb(dst, mem));
 6924 
 6925   ins_pipe(iload_reg_mem);
 6926 %}
 6927 
 6928 // Load Byte (8 bit unsigned)
 6929 instruct loadUB(iRegINoSp dst, memory1 mem)
 6930 %{
 6931   match(Set dst (LoadUB mem));
 6932   predicate(!needs_acquiring_load(n));
 6933 
 6934   ins_cost(4 * INSN_COST);
 6935   format %{ "ldrbw  $dst, $mem\t# byte" %}
 6936 
 6937   ins_encode(aarch64_enc_ldrb(dst, mem));
 6938 
 6939   ins_pipe(iload_reg_mem);
 6940 %}
 6941 
 6942 // Load Byte (8 bit unsigned) into long
 6943 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 6944 %{
 6945   match(Set dst (ConvI2L (LoadUB mem)));
 6946   predicate(!needs_acquiring_load(n->in(1)));
 6947 
 6948   ins_cost(4 * INSN_COST);
 6949   format %{ "ldrb  $dst, $mem\t# byte" %}
 6950 
 6951   ins_encode(aarch64_enc_ldrb(dst, mem));
 6952 
 6953   ins_pipe(iload_reg_mem);
 6954 %}
 6955 
 6956 // Load Short (16 bit signed)
 6957 instruct loadS(iRegINoSp dst, memory2 mem)
 6958 %{
 6959   match(Set dst (LoadS mem));
 6960   predicate(!needs_acquiring_load(n));
 6961 
 6962   ins_cost(4 * INSN_COST);
 6963   format %{ "ldrshw  $dst, $mem\t# short" %}
 6964 
 6965   ins_encode(aarch64_enc_ldrshw(dst, mem));
 6966 
 6967   ins_pipe(iload_reg_mem);
 6968 %}
 6969 
 6970 // Load Short (16 bit signed) into long
 6971 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 6972 %{
 6973   match(Set dst (ConvI2L (LoadS mem)));
 6974   predicate(!needs_acquiring_load(n->in(1)));
 6975 
 6976   ins_cost(4 * INSN_COST);
 6977   format %{ "ldrsh  $dst, $mem\t# short" %}
 6978 
 6979   ins_encode(aarch64_enc_ldrsh(dst, mem));
 6980 
 6981   ins_pipe(iload_reg_mem);
 6982 %}
 6983 
 6984 // Load Char (16 bit unsigned)
 6985 instruct loadUS(iRegINoSp dst, memory2 mem)
 6986 %{
 6987   match(Set dst (LoadUS mem));
 6988   predicate(!needs_acquiring_load(n));
 6989 
 6990   ins_cost(4 * INSN_COST);
 6991   format %{ "ldrh  $dst, $mem\t# short" %}
 6992 
 6993   ins_encode(aarch64_enc_ldrh(dst, mem));
 6994 
 6995   ins_pipe(iload_reg_mem);
 6996 %}
 6997 
 6998 // Load Short/Char (16 bit unsigned) into long
 6999 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7000 %{
 7001   match(Set dst (ConvI2L (LoadUS mem)));
 7002   predicate(!needs_acquiring_load(n->in(1)));
 7003 
 7004   ins_cost(4 * INSN_COST);
 7005   format %{ "ldrh  $dst, $mem\t# short" %}
 7006 
 7007   ins_encode(aarch64_enc_ldrh(dst, mem));
 7008 
 7009   ins_pipe(iload_reg_mem);
 7010 %}
 7011 
 7012 // Load Integer (32 bit signed)
 7013 instruct loadI(iRegINoSp dst, memory4 mem)
 7014 %{
 7015   match(Set dst (LoadI mem));
 7016   predicate(!needs_acquiring_load(n));
 7017 
 7018   ins_cost(4 * INSN_COST);
 7019   format %{ "ldrw  $dst, $mem\t# int" %}
 7020 
 7021   ins_encode(aarch64_enc_ldrw(dst, mem));
 7022 
 7023   ins_pipe(iload_reg_mem);
 7024 %}
 7025 
 7026 // Load Integer (32 bit signed) into long
 7027 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7028 %{
 7029   match(Set dst (ConvI2L (LoadI mem)));
 7030   predicate(!needs_acquiring_load(n->in(1)));
 7031 
 7032   ins_cost(4 * INSN_COST);
 7033   format %{ "ldrsw  $dst, $mem\t# int" %}
 7034 
 7035   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7036 
 7037   ins_pipe(iload_reg_mem);
 7038 %}
 7039 
 7040 // Load Integer (32 bit unsigned) into long
 7041 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7042 %{
 7043   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7044   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7045 
 7046   ins_cost(4 * INSN_COST);
 7047   format %{ "ldrw  $dst, $mem\t# int" %}
 7048 
 7049   ins_encode(aarch64_enc_ldrw(dst, mem));
 7050 
 7051   ins_pipe(iload_reg_mem);
 7052 %}
 7053 
 7054 // Load Long (64 bit signed)
 7055 instruct loadL(iRegLNoSp dst, memory8 mem)
 7056 %{
 7057   match(Set dst (LoadL mem));
 7058   predicate(!needs_acquiring_load(n));
 7059 
 7060   ins_cost(4 * INSN_COST);
 7061   format %{ "ldr  $dst, $mem\t# int" %}
 7062 
 7063   ins_encode(aarch64_enc_ldr(dst, mem));
 7064 
 7065   ins_pipe(iload_reg_mem);
 7066 %}
 7067 
 7068 // Load Range
 7069 instruct loadRange(iRegINoSp dst, memory4 mem)
 7070 %{
 7071   match(Set dst (LoadRange mem));
 7072 
 7073   ins_cost(4 * INSN_COST);
 7074   format %{ "ldrw  $dst, $mem\t# range" %}
 7075 
 7076   ins_encode(aarch64_enc_ldrw(dst, mem));
 7077 
 7078   ins_pipe(iload_reg_mem);
 7079 %}
 7080 
 7081 // Load Pointer
 7082 instruct loadP(iRegPNoSp dst, memory8 mem)
 7083 %{
 7084   match(Set dst (LoadP mem));
 7085   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7086 
 7087   ins_cost(4 * INSN_COST);
 7088   format %{ "ldr  $dst, $mem\t# ptr" %}
 7089 
 7090   ins_encode(aarch64_enc_ldr(dst, mem));
 7091 
 7092   ins_pipe(iload_reg_mem);
 7093 %}
 7094 
 7095 // Load Compressed Pointer
 7096 instruct loadN(iRegNNoSp dst, memory4 mem)
 7097 %{
 7098   match(Set dst (LoadN mem));
 7099   predicate(!needs_acquiring_load(n));
 7100 
 7101   ins_cost(4 * INSN_COST);
 7102   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7103 
 7104   ins_encode(aarch64_enc_ldrw(dst, mem));
 7105 
 7106   ins_pipe(iload_reg_mem);
 7107 %}
 7108 
 7109 // Load Klass Pointer
 7110 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7111 %{
 7112   match(Set dst (LoadKlass mem));
 7113   predicate(!needs_acquiring_load(n));
 7114 
 7115   ins_cost(4 * INSN_COST);
 7116   format %{ "ldr  $dst, $mem\t# class" %}
 7117 
 7118   ins_encode(aarch64_enc_ldr(dst, mem));
 7119 
 7120   ins_pipe(iload_reg_mem);
 7121 %}
 7122 
 7123 // Load Narrow Klass Pointer
 7124 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7125 %{
 7126   match(Set dst (LoadNKlass mem));
 7127   predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
 7128 
 7129   ins_cost(4 * INSN_COST);
 7130   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7131 
 7132   ins_encode(aarch64_enc_ldrw(dst, mem));
 7133 
 7134   ins_pipe(iload_reg_mem);
 7135 %}
 7136 
 7137 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem, rFlagsReg cr)
 7138 %{
 7139   match(Set dst (LoadNKlass mem));
 7140   effect(KILL cr);
 7141   predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
 7142 
 7143   ins_cost(4 * INSN_COST);
 7144   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7145   ins_encode %{
 7146     __ load_nklass_compact($dst$$Register, $mem$$base$$Register, $mem$$index$$Register, $mem$$scale, $mem$$disp);
 7147   %}
 7148   ins_pipe(pipe_slow);
 7149 %}
 7150 
 7151 // Load Float
 7152 instruct loadF(vRegF dst, memory4 mem)
 7153 %{
 7154   match(Set dst (LoadF mem));
 7155   predicate(!needs_acquiring_load(n));
 7156 
 7157   ins_cost(4 * INSN_COST);
 7158   format %{ "ldrs  $dst, $mem\t# float" %}
 7159 
 7160   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7161 
 7162   ins_pipe(pipe_class_memory);
 7163 %}
 7164 
 7165 // Load Double
 7166 instruct loadD(vRegD dst, memory8 mem)
 7167 %{
 7168   match(Set dst (LoadD mem));
 7169   predicate(!needs_acquiring_load(n));
 7170 
 7171   ins_cost(4 * INSN_COST);
 7172   format %{ "ldrd  $dst, $mem\t# double" %}
 7173 
 7174   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7175 
 7176   ins_pipe(pipe_class_memory);
 7177 %}
 7178 
 7179 
 7180 // Load Int Constant
 7181 instruct loadConI(iRegINoSp dst, immI src)
 7182 %{
 7183   match(Set dst src);
 7184 
 7185   ins_cost(INSN_COST);
 7186   format %{ "mov $dst, $src\t# int" %}
 7187 
 7188   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7189 
 7190   ins_pipe(ialu_imm);
 7191 %}
 7192 
 7193 // Load Long Constant
 7194 instruct loadConL(iRegLNoSp dst, immL src)
 7195 %{
 7196   match(Set dst src);
 7197 
 7198   ins_cost(INSN_COST);
 7199   format %{ "mov $dst, $src\t# long" %}
 7200 
 7201   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7202 
 7203   ins_pipe(ialu_imm);
 7204 %}
 7205 
 7206 // Load Pointer Constant
 7207 
 7208 instruct loadConP(iRegPNoSp dst, immP con)
 7209 %{
 7210   match(Set dst con);
 7211 
 7212   ins_cost(INSN_COST * 4);
 7213   format %{
 7214     "mov  $dst, $con\t# ptr\n\t"
 7215   %}
 7216 
 7217   ins_encode(aarch64_enc_mov_p(dst, con));
 7218 
 7219   ins_pipe(ialu_imm);
 7220 %}
 7221 
 7222 // Load Null Pointer Constant
 7223 
 7224 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7225 %{
 7226   match(Set dst con);
 7227 
 7228   ins_cost(INSN_COST);
 7229   format %{ "mov  $dst, $con\t# null pointer" %}
 7230 
 7231   ins_encode(aarch64_enc_mov_p0(dst, con));
 7232 
 7233   ins_pipe(ialu_imm);
 7234 %}
 7235 
 7236 // Load Pointer Constant One
 7237 
 7238 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7239 %{
 7240   match(Set dst con);
 7241 
 7242   ins_cost(INSN_COST);
 7243   format %{ "mov  $dst, $con\t# null pointer" %}
 7244 
 7245   ins_encode(aarch64_enc_mov_p1(dst, con));
 7246 
 7247   ins_pipe(ialu_imm);
 7248 %}
 7249 
 7250 // Load Byte Map Base Constant
 7251 
 7252 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7253 %{
 7254   match(Set dst con);
 7255 
 7256   ins_cost(INSN_COST);
 7257   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7258 
 7259   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7260 
 7261   ins_pipe(ialu_imm);
 7262 %}
 7263 
 7264 // Load Narrow Pointer Constant
 7265 
 7266 instruct loadConN(iRegNNoSp dst, immN con)
 7267 %{
 7268   match(Set dst con);
 7269 
 7270   ins_cost(INSN_COST * 4);
 7271   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7272 
 7273   ins_encode(aarch64_enc_mov_n(dst, con));
 7274 
 7275   ins_pipe(ialu_imm);
 7276 %}
 7277 
 7278 // Load Narrow Null Pointer Constant
 7279 
 7280 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7281 %{
 7282   match(Set dst con);
 7283 
 7284   ins_cost(INSN_COST);
 7285   format %{ "mov  $dst, $con\t# compressed null pointer" %}
 7286 
 7287   ins_encode(aarch64_enc_mov_n0(dst, con));
 7288 
 7289   ins_pipe(ialu_imm);
 7290 %}
 7291 
 7292 // Load Narrow Klass Constant
 7293 
 7294 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7295 %{
 7296   match(Set dst con);
 7297 
 7298   ins_cost(INSN_COST);
 7299   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7300 
 7301   ins_encode(aarch64_enc_mov_nk(dst, con));
 7302 
 7303   ins_pipe(ialu_imm);
 7304 %}
 7305 
 7306 // Load Packed Float Constant
 7307 
 7308 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7309   match(Set dst con);
 7310   ins_cost(INSN_COST * 4);
 7311   format %{ "fmovs  $dst, $con"%}
 7312   ins_encode %{
 7313     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7314   %}
 7315 
 7316   ins_pipe(fp_imm_s);
 7317 %}
 7318 
 7319 // Load Float Constant
 7320 
 7321 instruct loadConF(vRegF dst, immF con) %{
 7322   match(Set dst con);
 7323 
 7324   ins_cost(INSN_COST * 4);
 7325 
 7326   format %{
 7327     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7328   %}
 7329 
 7330   ins_encode %{
 7331     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7332   %}
 7333 
 7334   ins_pipe(fp_load_constant_s);
 7335 %}
 7336 
 7337 // Load Packed Double Constant
 7338 
 7339 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7340   match(Set dst con);
 7341   ins_cost(INSN_COST);
 7342   format %{ "fmovd  $dst, $con"%}
 7343   ins_encode %{
 7344     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7345   %}
 7346 
 7347   ins_pipe(fp_imm_d);
 7348 %}
 7349 
 7350 // Load Double Constant
 7351 
 7352 instruct loadConD(vRegD dst, immD con) %{
 7353   match(Set dst con);
 7354 
 7355   ins_cost(INSN_COST * 5);
 7356   format %{
 7357     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7358   %}
 7359 
 7360   ins_encode %{
 7361     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7362   %}
 7363 
 7364   ins_pipe(fp_load_constant_d);
 7365 %}
 7366 
 7367 // Store Instructions
 7368 
 7369 // Store CMS card-mark Immediate
 7370 instruct storeimmCM0(immI0 zero, memory1 mem)
 7371 %{
 7372   match(Set mem (StoreCM mem zero));
 7373 
 7374   ins_cost(INSN_COST);
 7375   format %{ "storestore (elided)\n\t"
 7376             "strb zr, $mem\t# byte" %}
 7377 
 7378   ins_encode(aarch64_enc_strb0(mem));
 7379 
 7380   ins_pipe(istore_mem);
 7381 %}
 7382 
 7383 // Store CMS card-mark Immediate with intervening StoreStore
 7384 // needed when using CMS with no conditional card marking
 7385 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7386 %{
 7387   match(Set mem (StoreCM mem zero));
 7388 
 7389   ins_cost(INSN_COST * 2);
 7390   format %{ "storestore\n\t"
 7391             "dmb ishst"
 7392             "\n\tstrb zr, $mem\t# byte" %}
 7393 
 7394   ins_encode(aarch64_enc_strb0_ordered(mem));
 7395 
 7396   ins_pipe(istore_mem);
 7397 %}
 7398 
 7399 // Store Byte
 7400 instruct storeB(iRegIorL2I src, memory1 mem)
 7401 %{
 7402   match(Set mem (StoreB mem src));
 7403   predicate(!needs_releasing_store(n));
 7404 
 7405   ins_cost(INSN_COST);
 7406   format %{ "strb  $src, $mem\t# byte" %}
 7407 
 7408   ins_encode(aarch64_enc_strb(src, mem));
 7409 
 7410   ins_pipe(istore_reg_mem);
 7411 %}
 7412 
 7413 
 7414 instruct storeimmB0(immI0 zero, memory1 mem)
 7415 %{
 7416   match(Set mem (StoreB mem zero));
 7417   predicate(!needs_releasing_store(n));
 7418 
 7419   ins_cost(INSN_COST);
 7420   format %{ "strb rscractch2, $mem\t# byte" %}
 7421 
 7422   ins_encode(aarch64_enc_strb0(mem));
 7423 
 7424   ins_pipe(istore_mem);
 7425 %}
 7426 
 7427 // Store Char/Short
 7428 instruct storeC(iRegIorL2I src, memory2 mem)
 7429 %{
 7430   match(Set mem (StoreC mem src));
 7431   predicate(!needs_releasing_store(n));
 7432 
 7433   ins_cost(INSN_COST);
 7434   format %{ "strh  $src, $mem\t# short" %}
 7435 
 7436   ins_encode(aarch64_enc_strh(src, mem));
 7437 
 7438   ins_pipe(istore_reg_mem);
 7439 %}
 7440 
 7441 instruct storeimmC0(immI0 zero, memory2 mem)
 7442 %{
 7443   match(Set mem (StoreC mem zero));
 7444   predicate(!needs_releasing_store(n));
 7445 
 7446   ins_cost(INSN_COST);
 7447   format %{ "strh  zr, $mem\t# short" %}
 7448 
 7449   ins_encode(aarch64_enc_strh0(mem));
 7450 
 7451   ins_pipe(istore_mem);
 7452 %}
 7453 
 7454 // Store Integer
 7455 
 7456 instruct storeI(iRegIorL2I src, memory4 mem)
 7457 %{
 7458   match(Set mem(StoreI mem src));
 7459   predicate(!needs_releasing_store(n));
 7460 
 7461   ins_cost(INSN_COST);
 7462   format %{ "strw  $src, $mem\t# int" %}
 7463 
 7464   ins_encode(aarch64_enc_strw(src, mem));
 7465 
 7466   ins_pipe(istore_reg_mem);
 7467 %}
 7468 
 7469 instruct storeimmI0(immI0 zero, memory4 mem)
 7470 %{
 7471   match(Set mem(StoreI mem zero));
 7472   predicate(!needs_releasing_store(n));
 7473 
 7474   ins_cost(INSN_COST);
 7475   format %{ "strw  zr, $mem\t# int" %}
 7476 
 7477   ins_encode(aarch64_enc_strw0(mem));
 7478 
 7479   ins_pipe(istore_mem);
 7480 %}
 7481 
 7482 // Store Long (64 bit signed)
 7483 instruct storeL(iRegL src, memory8 mem)
 7484 %{
 7485   match(Set mem (StoreL mem src));
 7486   predicate(!needs_releasing_store(n));
 7487 
 7488   ins_cost(INSN_COST);
 7489   format %{ "str  $src, $mem\t# int" %}
 7490 
 7491   ins_encode(aarch64_enc_str(src, mem));
 7492 
 7493   ins_pipe(istore_reg_mem);
 7494 %}
 7495 
 7496 // Store Long (64 bit signed)
 7497 instruct storeimmL0(immL0 zero, memory8 mem)
 7498 %{
 7499   match(Set mem (StoreL mem zero));
 7500   predicate(!needs_releasing_store(n));
 7501 
 7502   ins_cost(INSN_COST);
 7503   format %{ "str  zr, $mem\t# int" %}
 7504 
 7505   ins_encode(aarch64_enc_str0(mem));
 7506 
 7507   ins_pipe(istore_mem);
 7508 %}
 7509 
 7510 // Store Pointer
 7511 instruct storeP(iRegP src, memory8 mem)
 7512 %{
 7513   match(Set mem (StoreP mem src));
 7514   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7515 
 7516   ins_cost(INSN_COST);
 7517   format %{ "str  $src, $mem\t# ptr" %}
 7518 
 7519   ins_encode(aarch64_enc_str(src, mem));
 7520 
 7521   ins_pipe(istore_reg_mem);
 7522 %}
 7523 
 7524 // Store Pointer
 7525 instruct storeimmP0(immP0 zero, memory8 mem)
 7526 %{
 7527   match(Set mem (StoreP mem zero));
 7528   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7529 
 7530   ins_cost(INSN_COST);
 7531   format %{ "str zr, $mem\t# ptr" %}
 7532 
 7533   ins_encode(aarch64_enc_str0(mem));
 7534 
 7535   ins_pipe(istore_mem);
 7536 %}
 7537 
 7538 // Store Compressed Pointer
 7539 instruct storeN(iRegN src, memory4 mem)
 7540 %{
 7541   match(Set mem (StoreN mem src));
 7542   predicate(!needs_releasing_store(n));
 7543 
 7544   ins_cost(INSN_COST);
 7545   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7546 
 7547   ins_encode(aarch64_enc_strw(src, mem));
 7548 
 7549   ins_pipe(istore_reg_mem);
 7550 %}
 7551 
 7552 instruct storeImmN0(immN0 zero, memory4 mem)
 7553 %{
 7554   match(Set mem (StoreN mem zero));
 7555   predicate(!needs_releasing_store(n));
 7556 
 7557   ins_cost(INSN_COST);
 7558   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7559 
 7560   ins_encode(aarch64_enc_strw0(mem));
 7561 
 7562   ins_pipe(istore_mem);
 7563 %}
 7564 
 7565 // Store Float
 7566 instruct storeF(vRegF src, memory4 mem)
 7567 %{
 7568   match(Set mem (StoreF mem src));
 7569   predicate(!needs_releasing_store(n));
 7570 
 7571   ins_cost(INSN_COST);
 7572   format %{ "strs  $src, $mem\t# float" %}
 7573 
 7574   ins_encode( aarch64_enc_strs(src, mem) );
 7575 
 7576   ins_pipe(pipe_class_memory);
 7577 %}
 7578 
 7579 // TODO
 7580 // implement storeImmF0 and storeFImmPacked
 7581 
 7582 // Store Double
 7583 instruct storeD(vRegD src, memory8 mem)
 7584 %{
 7585   match(Set mem (StoreD mem src));
 7586   predicate(!needs_releasing_store(n));
 7587 
 7588   ins_cost(INSN_COST);
 7589   format %{ "strd  $src, $mem\t# double" %}
 7590 
 7591   ins_encode( aarch64_enc_strd(src, mem) );
 7592 
 7593   ins_pipe(pipe_class_memory);
 7594 %}
 7595 
 7596 // Store Compressed Klass Pointer
 7597 instruct storeNKlass(iRegN src, memory4 mem)
 7598 %{
 7599   predicate(!needs_releasing_store(n));
 7600   match(Set mem (StoreNKlass mem src));
 7601 
 7602   ins_cost(INSN_COST);
 7603   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7604 
 7605   ins_encode(aarch64_enc_strw(src, mem));
 7606 
 7607   ins_pipe(istore_reg_mem);
 7608 %}
 7609 
 7610 // TODO
 7611 // implement storeImmD0 and storeDImmPacked
 7612 
 7613 // prefetch instructions
 7614 // Must be safe to execute with invalid address (cannot fault).
 7615 
 7616 instruct prefetchalloc( memory8 mem ) %{
 7617   match(PrefetchAllocation mem);
 7618 
 7619   ins_cost(INSN_COST);
 7620   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7621 
 7622   ins_encode( aarch64_enc_prefetchw(mem) );
 7623 
 7624   ins_pipe(iload_prefetch);
 7625 %}
 7626 
 7627 //  ---------------- volatile loads and stores ----------------
 7628 
 7629 // Load Byte (8 bit signed)
 7630 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7631 %{
 7632   match(Set dst (LoadB mem));
 7633 
 7634   ins_cost(VOLATILE_REF_COST);
 7635   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7636 
 7637   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7638 
 7639   ins_pipe(pipe_serial);
 7640 %}
 7641 
 7642 // Load Byte (8 bit signed) into long
 7643 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7644 %{
 7645   match(Set dst (ConvI2L (LoadB mem)));
 7646 
 7647   ins_cost(VOLATILE_REF_COST);
 7648   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7649 
 7650   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7651 
 7652   ins_pipe(pipe_serial);
 7653 %}
 7654 
 7655 // Load Byte (8 bit unsigned)
 7656 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7657 %{
 7658   match(Set dst (LoadUB mem));
 7659 
 7660   ins_cost(VOLATILE_REF_COST);
 7661   format %{ "ldarb  $dst, $mem\t# byte" %}
 7662 
 7663   ins_encode(aarch64_enc_ldarb(dst, mem));
 7664 
 7665   ins_pipe(pipe_serial);
 7666 %}
 7667 
 7668 // Load Byte (8 bit unsigned) into long
 7669 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7670 %{
 7671   match(Set dst (ConvI2L (LoadUB mem)));
 7672 
 7673   ins_cost(VOLATILE_REF_COST);
 7674   format %{ "ldarb  $dst, $mem\t# byte" %}
 7675 
 7676   ins_encode(aarch64_enc_ldarb(dst, mem));
 7677 
 7678   ins_pipe(pipe_serial);
 7679 %}
 7680 
 7681 // Load Short (16 bit signed)
 7682 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7683 %{
 7684   match(Set dst (LoadS mem));
 7685 
 7686   ins_cost(VOLATILE_REF_COST);
 7687   format %{ "ldarshw  $dst, $mem\t# short" %}
 7688 
 7689   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7690 
 7691   ins_pipe(pipe_serial);
 7692 %}
 7693 
 7694 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7695 %{
 7696   match(Set dst (LoadUS mem));
 7697 
 7698   ins_cost(VOLATILE_REF_COST);
 7699   format %{ "ldarhw  $dst, $mem\t# short" %}
 7700 
 7701   ins_encode(aarch64_enc_ldarhw(dst, mem));
 7702 
 7703   ins_pipe(pipe_serial);
 7704 %}
 7705 
 7706 // Load Short/Char (16 bit unsigned) into long
 7707 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7708 %{
 7709   match(Set dst (ConvI2L (LoadUS mem)));
 7710 
 7711   ins_cost(VOLATILE_REF_COST);
 7712   format %{ "ldarh  $dst, $mem\t# short" %}
 7713 
 7714   ins_encode(aarch64_enc_ldarh(dst, mem));
 7715 
 7716   ins_pipe(pipe_serial);
 7717 %}
 7718 
 7719 // Load Short/Char (16 bit signed) into long
 7720 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7721 %{
 7722   match(Set dst (ConvI2L (LoadS mem)));
 7723 
 7724   ins_cost(VOLATILE_REF_COST);
 7725   format %{ "ldarh  $dst, $mem\t# short" %}
 7726 
 7727   ins_encode(aarch64_enc_ldarsh(dst, mem));
 7728 
 7729   ins_pipe(pipe_serial);
 7730 %}
 7731 
 7732 // Load Integer (32 bit signed)
 7733 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7734 %{
 7735   match(Set dst (LoadI mem));
 7736 
 7737   ins_cost(VOLATILE_REF_COST);
 7738   format %{ "ldarw  $dst, $mem\t# int" %}
 7739 
 7740   ins_encode(aarch64_enc_ldarw(dst, mem));
 7741 
 7742   ins_pipe(pipe_serial);
 7743 %}
 7744 
 7745 // Load Integer (32 bit unsigned) into long
 7746 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 7747 %{
 7748   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7749 
 7750   ins_cost(VOLATILE_REF_COST);
 7751   format %{ "ldarw  $dst, $mem\t# int" %}
 7752 
 7753   ins_encode(aarch64_enc_ldarw(dst, mem));
 7754 
 7755   ins_pipe(pipe_serial);
 7756 %}
 7757 
 7758 // Load Long (64 bit signed)
 7759 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7760 %{
 7761   match(Set dst (LoadL mem));
 7762 
 7763   ins_cost(VOLATILE_REF_COST);
 7764   format %{ "ldar  $dst, $mem\t# int" %}
 7765 
 7766   ins_encode(aarch64_enc_ldar(dst, mem));
 7767 
 7768   ins_pipe(pipe_serial);
 7769 %}
 7770 
 7771 // Load Pointer
 7772 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 7773 %{
 7774   match(Set dst (LoadP mem));
 7775   predicate(n->as_Load()->barrier_data() == 0);
 7776 
 7777   ins_cost(VOLATILE_REF_COST);
 7778   format %{ "ldar  $dst, $mem\t# ptr" %}
 7779 
 7780   ins_encode(aarch64_enc_ldar(dst, mem));
 7781 
 7782   ins_pipe(pipe_serial);
 7783 %}
 7784 
 7785 // Load Compressed Pointer
 7786 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 7787 %{
 7788   match(Set dst (LoadN mem));
 7789 
 7790   ins_cost(VOLATILE_REF_COST);
 7791   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 7792 
 7793   ins_encode(aarch64_enc_ldarw(dst, mem));
 7794 
 7795   ins_pipe(pipe_serial);
 7796 %}
 7797 
 7798 // Load Float
 7799 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 7800 %{
 7801   match(Set dst (LoadF mem));
 7802 
 7803   ins_cost(VOLATILE_REF_COST);
 7804   format %{ "ldars  $dst, $mem\t# float" %}
 7805 
 7806   ins_encode( aarch64_enc_fldars(dst, mem) );
 7807 
 7808   ins_pipe(pipe_serial);
 7809 %}
 7810 
 7811 // Load Double
 7812 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 7813 %{
 7814   match(Set dst (LoadD mem));
 7815 
 7816   ins_cost(VOLATILE_REF_COST);
 7817   format %{ "ldard  $dst, $mem\t# double" %}
 7818 
 7819   ins_encode( aarch64_enc_fldard(dst, mem) );
 7820 
 7821   ins_pipe(pipe_serial);
 7822 %}
 7823 
 7824 // Store Byte
 7825 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7826 %{
 7827   match(Set mem (StoreB mem src));
 7828 
 7829   ins_cost(VOLATILE_REF_COST);
 7830   format %{ "stlrb  $src, $mem\t# byte" %}
 7831 
 7832   ins_encode(aarch64_enc_stlrb(src, mem));
 7833 
 7834   ins_pipe(pipe_class_memory);
 7835 %}
 7836 
 7837 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7838 %{
 7839   match(Set mem (StoreB mem zero));
 7840 
 7841   ins_cost(VOLATILE_REF_COST);
 7842   format %{ "stlrb  zr, $mem\t# byte" %}
 7843 
 7844   ins_encode(aarch64_enc_stlrb0(mem));
 7845 
 7846   ins_pipe(pipe_class_memory);
 7847 %}
 7848 
 7849 // Store Char/Short
 7850 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7851 %{
 7852   match(Set mem (StoreC mem src));
 7853 
 7854   ins_cost(VOLATILE_REF_COST);
 7855   format %{ "stlrh  $src, $mem\t# short" %}
 7856 
 7857   ins_encode(aarch64_enc_stlrh(src, mem));
 7858 
 7859   ins_pipe(pipe_class_memory);
 7860 %}
 7861 
 7862 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7863 %{
 7864   match(Set mem (StoreC mem zero));
 7865 
 7866   ins_cost(VOLATILE_REF_COST);
 7867   format %{ "stlrh  zr, $mem\t# short" %}
 7868 
 7869   ins_encode(aarch64_enc_stlrh0(mem));
 7870 
 7871   ins_pipe(pipe_class_memory);
 7872 %}
 7873 
 7874 // Store Integer
 7875 
 7876 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7877 %{
 7878   match(Set mem(StoreI mem src));
 7879 
 7880   ins_cost(VOLATILE_REF_COST);
 7881   format %{ "stlrw  $src, $mem\t# int" %}
 7882 
 7883   ins_encode(aarch64_enc_stlrw(src, mem));
 7884 
 7885   ins_pipe(pipe_class_memory);
 7886 %}
 7887 
 7888 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7889 %{
 7890   match(Set mem(StoreI mem zero));
 7891 
 7892   ins_cost(VOLATILE_REF_COST);
 7893   format %{ "stlrw  zr, $mem\t# int" %}
 7894 
 7895   ins_encode(aarch64_enc_stlrw0(mem));
 7896 
 7897   ins_pipe(pipe_class_memory);
 7898 %}
 7899 
 7900 // Store Long (64 bit signed)
 7901 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 7902 %{
 7903   match(Set mem (StoreL mem src));
 7904 
 7905   ins_cost(VOLATILE_REF_COST);
 7906   format %{ "stlr  $src, $mem\t# int" %}
 7907 
 7908   ins_encode(aarch64_enc_stlr(src, mem));
 7909 
 7910   ins_pipe(pipe_class_memory);
 7911 %}
 7912 
 7913 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 7914 %{
 7915   match(Set mem (StoreL mem zero));
 7916 
 7917   ins_cost(VOLATILE_REF_COST);
 7918   format %{ "stlr  zr, $mem\t# int" %}
 7919 
 7920   ins_encode(aarch64_enc_stlr0(mem));
 7921 
 7922   ins_pipe(pipe_class_memory);
 7923 %}
 7924 
 7925 // Store Pointer
 7926 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 7927 %{
 7928   match(Set mem (StoreP mem src));
 7929   predicate(n->as_Store()->barrier_data() == 0);
 7930 
 7931   ins_cost(VOLATILE_REF_COST);
 7932   format %{ "stlr  $src, $mem\t# ptr" %}
 7933 
 7934   ins_encode(aarch64_enc_stlr(src, mem));
 7935 
 7936   ins_pipe(pipe_class_memory);
 7937 %}
 7938 
 7939 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 7940 %{
 7941   match(Set mem (StoreP mem zero));
 7942   predicate(n->as_Store()->barrier_data() == 0);
 7943 
 7944   ins_cost(VOLATILE_REF_COST);
 7945   format %{ "stlr  zr, $mem\t# ptr" %}
 7946 
 7947   ins_encode(aarch64_enc_stlr0(mem));
 7948 
 7949   ins_pipe(pipe_class_memory);
 7950 %}
 7951 
 7952 // Store Compressed Pointer
 7953 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 7954 %{
 7955   match(Set mem (StoreN mem src));
 7956 
 7957   ins_cost(VOLATILE_REF_COST);
 7958   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 7959 
 7960   ins_encode(aarch64_enc_stlrw(src, mem));
 7961 
 7962   ins_pipe(pipe_class_memory);
 7963 %}
 7964 
 7965 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 7966 %{
 7967   match(Set mem (StoreN mem zero));
 7968 
 7969   ins_cost(VOLATILE_REF_COST);
 7970   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 7971 
 7972   ins_encode(aarch64_enc_stlrw0(mem));
 7973 
 7974   ins_pipe(pipe_class_memory);
 7975 %}
 7976 
 7977 // Store Float
 7978 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 7979 %{
 7980   match(Set mem (StoreF mem src));
 7981 
 7982   ins_cost(VOLATILE_REF_COST);
 7983   format %{ "stlrs  $src, $mem\t# float" %}
 7984 
 7985   ins_encode( aarch64_enc_fstlrs(src, mem) );
 7986 
 7987   ins_pipe(pipe_class_memory);
 7988 %}
 7989 
 7990 // TODO
 7991 // implement storeImmF0 and storeFImmPacked
 7992 
 7993 // Store Double
 7994 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 7995 %{
 7996   match(Set mem (StoreD mem src));
 7997 
 7998   ins_cost(VOLATILE_REF_COST);
 7999   format %{ "stlrd  $src, $mem\t# double" %}
 8000 
 8001   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8002 
 8003   ins_pipe(pipe_class_memory);
 8004 %}
 8005 
 8006 //  ---------------- end of volatile loads and stores ----------------
 8007 
 8008 instruct cacheWB(indirect addr)
 8009 %{
 8010   predicate(VM_Version::supports_data_cache_line_flush());
 8011   match(CacheWB addr);
 8012 
 8013   ins_cost(100);
 8014   format %{"cache wb $addr" %}
 8015   ins_encode %{
 8016     assert($addr->index_position() < 0, "should be");
 8017     assert($addr$$disp == 0, "should be");
 8018     __ cache_wb(Address($addr$$base$$Register, 0));
 8019   %}
 8020   ins_pipe(pipe_slow); // XXX
 8021 %}
 8022 
 8023 instruct cacheWBPreSync()
 8024 %{
 8025   predicate(VM_Version::supports_data_cache_line_flush());
 8026   match(CacheWBPreSync);
 8027 
 8028   ins_cost(100);
 8029   format %{"cache wb presync" %}
 8030   ins_encode %{
 8031     __ cache_wbsync(true);
 8032   %}
 8033   ins_pipe(pipe_slow); // XXX
 8034 %}
 8035 
 8036 instruct cacheWBPostSync()
 8037 %{
 8038   predicate(VM_Version::supports_data_cache_line_flush());
 8039   match(CacheWBPostSync);
 8040 
 8041   ins_cost(100);
 8042   format %{"cache wb postsync" %}
 8043   ins_encode %{
 8044     __ cache_wbsync(false);
 8045   %}
 8046   ins_pipe(pipe_slow); // XXX
 8047 %}
 8048 
 8049 // ============================================================================
 8050 // BSWAP Instructions
 8051 
 8052 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8053   match(Set dst (ReverseBytesI src));
 8054 
 8055   ins_cost(INSN_COST);
 8056   format %{ "revw  $dst, $src" %}
 8057 
 8058   ins_encode %{
 8059     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8060   %}
 8061 
 8062   ins_pipe(ialu_reg);
 8063 %}
 8064 
 8065 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8066   match(Set dst (ReverseBytesL src));
 8067 
 8068   ins_cost(INSN_COST);
 8069   format %{ "rev  $dst, $src" %}
 8070 
 8071   ins_encode %{
 8072     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8073   %}
 8074 
 8075   ins_pipe(ialu_reg);
 8076 %}
 8077 
 8078 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8079   match(Set dst (ReverseBytesUS src));
 8080 
 8081   ins_cost(INSN_COST);
 8082   format %{ "rev16w  $dst, $src" %}
 8083 
 8084   ins_encode %{
 8085     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8086   %}
 8087 
 8088   ins_pipe(ialu_reg);
 8089 %}
 8090 
 8091 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8092   match(Set dst (ReverseBytesS src));
 8093 
 8094   ins_cost(INSN_COST);
 8095   format %{ "rev16w  $dst, $src\n\t"
 8096             "sbfmw $dst, $dst, #0, #15" %}
 8097 
 8098   ins_encode %{
 8099     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8100     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8101   %}
 8102 
 8103   ins_pipe(ialu_reg);
 8104 %}
 8105 
 8106 // ============================================================================
 8107 // Zero Count Instructions
 8108 
 8109 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8110   match(Set dst (CountLeadingZerosI src));
 8111 
 8112   ins_cost(INSN_COST);
 8113   format %{ "clzw  $dst, $src" %}
 8114   ins_encode %{
 8115     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8116   %}
 8117 
 8118   ins_pipe(ialu_reg);
 8119 %}
 8120 
 8121 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8122   match(Set dst (CountLeadingZerosL src));
 8123 
 8124   ins_cost(INSN_COST);
 8125   format %{ "clz   $dst, $src" %}
 8126   ins_encode %{
 8127     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8128   %}
 8129 
 8130   ins_pipe(ialu_reg);
 8131 %}
 8132 
 8133 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8134   match(Set dst (CountTrailingZerosI src));
 8135 
 8136   ins_cost(INSN_COST * 2);
 8137   format %{ "rbitw  $dst, $src\n\t"
 8138             "clzw   $dst, $dst" %}
 8139   ins_encode %{
 8140     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8141     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8142   %}
 8143 
 8144   ins_pipe(ialu_reg);
 8145 %}
 8146 
 8147 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8148   match(Set dst (CountTrailingZerosL src));
 8149 
 8150   ins_cost(INSN_COST * 2);
 8151   format %{ "rbit   $dst, $src\n\t"
 8152             "clz    $dst, $dst" %}
 8153   ins_encode %{
 8154     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8155     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8156   %}
 8157 
 8158   ins_pipe(ialu_reg);
 8159 %}
 8160 
 8161 //---------- Population Count Instructions -------------------------------------
 8162 //
 8163 
 8164 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8165   match(Set dst (PopCountI src));
 8166   effect(TEMP tmp);
 8167   ins_cost(INSN_COST * 13);
 8168 
 8169   format %{ "movw   $src, $src\n\t"
 8170             "mov    $tmp, $src\t# vector (1D)\n\t"
 8171             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8172             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8173             "mov    $dst, $tmp\t# vector (1D)" %}
 8174   ins_encode %{
 8175     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8176     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8177     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8178     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8179     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8180   %}
 8181 
 8182   ins_pipe(pipe_class_default);
 8183 %}
 8184 
 8185 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8186   match(Set dst (PopCountI (LoadI mem)));
 8187   effect(TEMP tmp);
 8188   ins_cost(INSN_COST * 13);
 8189 
 8190   format %{ "ldrs   $tmp, $mem\n\t"
 8191             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8192             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8193             "mov    $dst, $tmp\t# vector (1D)" %}
 8194   ins_encode %{
 8195     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8196     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8197               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8198     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8199     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8200     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8201   %}
 8202 
 8203   ins_pipe(pipe_class_default);
 8204 %}
 8205 
 8206 // Note: Long.bitCount(long) returns an int.
 8207 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8208   match(Set dst (PopCountL src));
 8209   effect(TEMP tmp);
 8210   ins_cost(INSN_COST * 13);
 8211 
 8212   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8213             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8214             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8215             "mov    $dst, $tmp\t# vector (1D)" %}
 8216   ins_encode %{
 8217     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8218     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8219     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8220     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8221   %}
 8222 
 8223   ins_pipe(pipe_class_default);
 8224 %}
 8225 
 8226 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8227   match(Set dst (PopCountL (LoadL mem)));
 8228   effect(TEMP tmp);
 8229   ins_cost(INSN_COST * 13);
 8230 
 8231   format %{ "ldrd   $tmp, $mem\n\t"
 8232             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8233             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8234             "mov    $dst, $tmp\t# vector (1D)" %}
 8235   ins_encode %{
 8236     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8237     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8238               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8239     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8240     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8241     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8242   %}
 8243 
 8244   ins_pipe(pipe_class_default);
 8245 %}
 8246 
 8247 // ============================================================================
 8248 // VerifyVectorAlignment Instruction
 8249 
 8250 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
 8251   match(Set addr (VerifyVectorAlignment addr mask));
 8252   effect(KILL cr);
 8253   format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
 8254   ins_encode %{
 8255     Label Lskip;
 8256     // check if masked bits of addr are zero
 8257     __ tst($addr$$Register, $mask$$constant);
 8258     __ br(Assembler::EQ, Lskip);
 8259     __ stop("verify_vector_alignment found a misaligned vector memory access");
 8260     __ bind(Lskip);
 8261   %}
 8262   ins_pipe(pipe_slow);
 8263 %}
 8264 
 8265 // ============================================================================
 8266 // MemBar Instruction
 8267 
 8268 instruct load_fence() %{
 8269   match(LoadFence);
 8270   ins_cost(VOLATILE_REF_COST);
 8271 
 8272   format %{ "load_fence" %}
 8273 
 8274   ins_encode %{
 8275     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8276   %}
 8277   ins_pipe(pipe_serial);
 8278 %}
 8279 
 8280 instruct unnecessary_membar_acquire() %{
 8281   predicate(unnecessary_acquire(n));
 8282   match(MemBarAcquire);
 8283   ins_cost(0);
 8284 
 8285   format %{ "membar_acquire (elided)" %}
 8286 
 8287   ins_encode %{
 8288     __ block_comment("membar_acquire (elided)");
 8289   %}
 8290 
 8291   ins_pipe(pipe_class_empty);
 8292 %}
 8293 
 8294 instruct membar_acquire() %{
 8295   match(MemBarAcquire);
 8296   ins_cost(VOLATILE_REF_COST);
 8297 
 8298   format %{ "membar_acquire\n\t"
 8299             "dmb ish" %}
 8300 
 8301   ins_encode %{
 8302     __ block_comment("membar_acquire");
 8303     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8304   %}
 8305 
 8306   ins_pipe(pipe_serial);
 8307 %}
 8308 
 8309 
 8310 instruct membar_acquire_lock() %{
 8311   match(MemBarAcquireLock);
 8312   ins_cost(VOLATILE_REF_COST);
 8313 
 8314   format %{ "membar_acquire_lock (elided)" %}
 8315 
 8316   ins_encode %{
 8317     __ block_comment("membar_acquire_lock (elided)");
 8318   %}
 8319 
 8320   ins_pipe(pipe_serial);
 8321 %}
 8322 
 8323 instruct store_fence() %{
 8324   match(StoreFence);
 8325   ins_cost(VOLATILE_REF_COST);
 8326 
 8327   format %{ "store_fence" %}
 8328 
 8329   ins_encode %{
 8330     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8331   %}
 8332   ins_pipe(pipe_serial);
 8333 %}
 8334 
 8335 instruct unnecessary_membar_release() %{
 8336   predicate(unnecessary_release(n));
 8337   match(MemBarRelease);
 8338   ins_cost(0);
 8339 
 8340   format %{ "membar_release (elided)" %}
 8341 
 8342   ins_encode %{
 8343     __ block_comment("membar_release (elided)");
 8344   %}
 8345   ins_pipe(pipe_serial);
 8346 %}
 8347 
 8348 instruct membar_release() %{
 8349   match(MemBarRelease);
 8350   ins_cost(VOLATILE_REF_COST);
 8351 
 8352   format %{ "membar_release\n\t"
 8353             "dmb ish" %}
 8354 
 8355   ins_encode %{
 8356     __ block_comment("membar_release");
 8357     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8358   %}
 8359   ins_pipe(pipe_serial);
 8360 %}
 8361 
 8362 instruct membar_storestore() %{
 8363   match(MemBarStoreStore);
 8364   match(StoreStoreFence);
 8365   ins_cost(VOLATILE_REF_COST);
 8366 
 8367   format %{ "MEMBAR-store-store" %}
 8368 
 8369   ins_encode %{
 8370     __ membar(Assembler::StoreStore);
 8371   %}
 8372   ins_pipe(pipe_serial);
 8373 %}
 8374 
 8375 instruct membar_release_lock() %{
 8376   match(MemBarReleaseLock);
 8377   ins_cost(VOLATILE_REF_COST);
 8378 
 8379   format %{ "membar_release_lock (elided)" %}
 8380 
 8381   ins_encode %{
 8382     __ block_comment("membar_release_lock (elided)");
 8383   %}
 8384 
 8385   ins_pipe(pipe_serial);
 8386 %}
 8387 
 8388 instruct unnecessary_membar_volatile() %{
 8389   predicate(unnecessary_volatile(n));
 8390   match(MemBarVolatile);
 8391   ins_cost(0);
 8392 
 8393   format %{ "membar_volatile (elided)" %}
 8394 
 8395   ins_encode %{
 8396     __ block_comment("membar_volatile (elided)");
 8397   %}
 8398 
 8399   ins_pipe(pipe_serial);
 8400 %}
 8401 
 8402 instruct membar_volatile() %{
 8403   match(MemBarVolatile);
 8404   ins_cost(VOLATILE_REF_COST*100);
 8405 
 8406   format %{ "membar_volatile\n\t"
 8407              "dmb ish"%}
 8408 
 8409   ins_encode %{
 8410     __ block_comment("membar_volatile");
 8411     __ membar(Assembler::StoreLoad);
 8412   %}
 8413 
 8414   ins_pipe(pipe_serial);
 8415 %}
 8416 
 8417 // ============================================================================
 8418 // Cast/Convert Instructions
 8419 
 8420 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8421   match(Set dst (CastX2P src));
 8422 
 8423   ins_cost(INSN_COST);
 8424   format %{ "mov $dst, $src\t# long -> ptr" %}
 8425 
 8426   ins_encode %{
 8427     if ($dst$$reg != $src$$reg) {
 8428       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8429     }
 8430   %}
 8431 
 8432   ins_pipe(ialu_reg);
 8433 %}
 8434 
 8435 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8436   match(Set dst (CastP2X src));
 8437 
 8438   ins_cost(INSN_COST);
 8439   format %{ "mov $dst, $src\t# ptr -> long" %}
 8440 
 8441   ins_encode %{
 8442     if ($dst$$reg != $src$$reg) {
 8443       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8444     }
 8445   %}
 8446 
 8447   ins_pipe(ialu_reg);
 8448 %}
 8449 
 8450 // Convert oop into int for vectors alignment masking
 8451 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8452   match(Set dst (ConvL2I (CastP2X src)));
 8453 
 8454   ins_cost(INSN_COST);
 8455   format %{ "movw $dst, $src\t# ptr -> int" %}
 8456   ins_encode %{
 8457     __ movw($dst$$Register, $src$$Register);
 8458   %}
 8459 
 8460   ins_pipe(ialu_reg);
 8461 %}
 8462 
 8463 // Convert compressed oop into int for vectors alignment masking
 8464 // in case of 32bit oops (heap < 4Gb).
 8465 instruct convN2I(iRegINoSp dst, iRegN src)
 8466 %{
 8467   predicate(CompressedOops::shift() == 0);
 8468   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8469 
 8470   ins_cost(INSN_COST);
 8471   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8472   ins_encode %{
 8473     __ movw($dst$$Register, $src$$Register);
 8474   %}
 8475 
 8476   ins_pipe(ialu_reg);
 8477 %}
 8478 
 8479 
 8480 // Convert oop pointer into compressed form
 8481 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8482   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8483   match(Set dst (EncodeP src));
 8484   effect(KILL cr);
 8485   ins_cost(INSN_COST * 3);
 8486   format %{ "encode_heap_oop $dst, $src" %}
 8487   ins_encode %{
 8488     Register s = $src$$Register;
 8489     Register d = $dst$$Register;
 8490     __ encode_heap_oop(d, s);
 8491   %}
 8492   ins_pipe(ialu_reg);
 8493 %}
 8494 
 8495 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8496   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8497   match(Set dst (EncodeP src));
 8498   ins_cost(INSN_COST * 3);
 8499   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8500   ins_encode %{
 8501     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8502   %}
 8503   ins_pipe(ialu_reg);
 8504 %}
 8505 
 8506 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8507   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8508             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8509   match(Set dst (DecodeN src));
 8510   ins_cost(INSN_COST * 3);
 8511   format %{ "decode_heap_oop $dst, $src" %}
 8512   ins_encode %{
 8513     Register s = $src$$Register;
 8514     Register d = $dst$$Register;
 8515     __ decode_heap_oop(d, s);
 8516   %}
 8517   ins_pipe(ialu_reg);
 8518 %}
 8519 
 8520 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8521   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8522             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8523   match(Set dst (DecodeN src));
 8524   ins_cost(INSN_COST * 3);
 8525   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8526   ins_encode %{
 8527     Register s = $src$$Register;
 8528     Register d = $dst$$Register;
 8529     __ decode_heap_oop_not_null(d, s);
 8530   %}
 8531   ins_pipe(ialu_reg);
 8532 %}
 8533 
 8534 // n.b. AArch64 implementations of encode_klass_not_null and
 8535 // decode_klass_not_null do not modify the flags register so, unlike
 8536 // Intel, we don't kill CR as a side effect here
 8537 
 8538 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8539   match(Set dst (EncodePKlass src));
 8540 
 8541   ins_cost(INSN_COST * 3);
 8542   format %{ "encode_klass_not_null $dst,$src" %}
 8543 
 8544   ins_encode %{
 8545     Register src_reg = as_Register($src$$reg);
 8546     Register dst_reg = as_Register($dst$$reg);
 8547     __ encode_klass_not_null(dst_reg, src_reg);
 8548   %}
 8549 
 8550    ins_pipe(ialu_reg);
 8551 %}
 8552 
 8553 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8554   match(Set dst (DecodeNKlass src));
 8555 
 8556   ins_cost(INSN_COST * 3);
 8557   format %{ "decode_klass_not_null $dst,$src" %}
 8558 
 8559   ins_encode %{
 8560     Register src_reg = as_Register($src$$reg);
 8561     Register dst_reg = as_Register($dst$$reg);
 8562     if (dst_reg != src_reg) {
 8563       __ decode_klass_not_null(dst_reg, src_reg);
 8564     } else {
 8565       __ decode_klass_not_null(dst_reg);
 8566     }
 8567   %}
 8568 
 8569    ins_pipe(ialu_reg);
 8570 %}
 8571 
 8572 instruct checkCastPP(iRegPNoSp dst)
 8573 %{
 8574   match(Set dst (CheckCastPP dst));
 8575 
 8576   size(0);
 8577   format %{ "# checkcastPP of $dst" %}
 8578   ins_encode(/* empty encoding */);
 8579   ins_pipe(pipe_class_empty);
 8580 %}
 8581 
 8582 instruct castPP(iRegPNoSp dst)
 8583 %{
 8584   match(Set dst (CastPP dst));
 8585 
 8586   size(0);
 8587   format %{ "# castPP of $dst" %}
 8588   ins_encode(/* empty encoding */);
 8589   ins_pipe(pipe_class_empty);
 8590 %}
 8591 
 8592 instruct castII(iRegI dst)
 8593 %{
 8594   match(Set dst (CastII dst));
 8595 
 8596   size(0);
 8597   format %{ "# castII of $dst" %}
 8598   ins_encode(/* empty encoding */);
 8599   ins_cost(0);
 8600   ins_pipe(pipe_class_empty);
 8601 %}
 8602 
 8603 instruct castLL(iRegL dst)
 8604 %{
 8605   match(Set dst (CastLL dst));
 8606 
 8607   size(0);
 8608   format %{ "# castLL of $dst" %}
 8609   ins_encode(/* empty encoding */);
 8610   ins_cost(0);
 8611   ins_pipe(pipe_class_empty);
 8612 %}
 8613 
 8614 instruct castFF(vRegF dst)
 8615 %{
 8616   match(Set dst (CastFF dst));
 8617 
 8618   size(0);
 8619   format %{ "# castFF of $dst" %}
 8620   ins_encode(/* empty encoding */);
 8621   ins_cost(0);
 8622   ins_pipe(pipe_class_empty);
 8623 %}
 8624 
 8625 instruct castDD(vRegD dst)
 8626 %{
 8627   match(Set dst (CastDD dst));
 8628 
 8629   size(0);
 8630   format %{ "# castDD of $dst" %}
 8631   ins_encode(/* empty encoding */);
 8632   ins_cost(0);
 8633   ins_pipe(pipe_class_empty);
 8634 %}
 8635 
 8636 instruct castVV(vReg dst)
 8637 %{
 8638   match(Set dst (CastVV dst));
 8639 
 8640   size(0);
 8641   format %{ "# castVV of $dst" %}
 8642   ins_encode(/* empty encoding */);
 8643   ins_cost(0);
 8644   ins_pipe(pipe_class_empty);
 8645 %}
 8646 
 8647 instruct castVVMask(pRegGov dst)
 8648 %{
 8649   match(Set dst (CastVV dst));
 8650 
 8651   size(0);
 8652   format %{ "# castVV of $dst" %}
 8653   ins_encode(/* empty encoding */);
 8654   ins_cost(0);
 8655   ins_pipe(pipe_class_empty);
 8656 %}
 8657 
 8658 // ============================================================================
 8659 // Atomic operation instructions
 8660 //
 8661 
 8662 // standard CompareAndSwapX when we are using barriers
 8663 // these have higher priority than the rules selected by a predicate
 8664 
 8665 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 8666 // can't match them
 8667 
 8668 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8669 
 8670   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8671   ins_cost(2 * VOLATILE_REF_COST);
 8672 
 8673   effect(KILL cr);
 8674 
 8675   format %{
 8676     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8677     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8678   %}
 8679 
 8680   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 8681             aarch64_enc_cset_eq(res));
 8682 
 8683   ins_pipe(pipe_slow);
 8684 %}
 8685 
 8686 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8687 
 8688   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8689   ins_cost(2 * VOLATILE_REF_COST);
 8690 
 8691   effect(KILL cr);
 8692 
 8693   format %{
 8694     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8695     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8696   %}
 8697 
 8698   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 8699             aarch64_enc_cset_eq(res));
 8700 
 8701   ins_pipe(pipe_slow);
 8702 %}
 8703 
 8704 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8705 
 8706   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8707   ins_cost(2 * VOLATILE_REF_COST);
 8708 
 8709   effect(KILL cr);
 8710 
 8711  format %{
 8712     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8713     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8714  %}
 8715 
 8716  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8717             aarch64_enc_cset_eq(res));
 8718 
 8719   ins_pipe(pipe_slow);
 8720 %}
 8721 
 8722 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8723 
 8724   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8725   ins_cost(2 * VOLATILE_REF_COST);
 8726 
 8727   effect(KILL cr);
 8728 
 8729  format %{
 8730     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8731     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8732  %}
 8733 
 8734  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8735             aarch64_enc_cset_eq(res));
 8736 
 8737   ins_pipe(pipe_slow);
 8738 %}
 8739 
 8740 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8741 
 8742   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8743   predicate(n->as_LoadStore()->barrier_data() == 0);
 8744   ins_cost(2 * VOLATILE_REF_COST);
 8745 
 8746   effect(KILL cr);
 8747 
 8748  format %{
 8749     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8750     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8751  %}
 8752 
 8753  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8754             aarch64_enc_cset_eq(res));
 8755 
 8756   ins_pipe(pipe_slow);
 8757 %}
 8758 
 8759 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8760 
 8761   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8762   ins_cost(2 * VOLATILE_REF_COST);
 8763 
 8764   effect(KILL cr);
 8765 
 8766  format %{
 8767     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8768     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8769  %}
 8770 
 8771  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8772             aarch64_enc_cset_eq(res));
 8773 
 8774   ins_pipe(pipe_slow);
 8775 %}
 8776 
 8777 // alternative CompareAndSwapX when we are eliding barriers
 8778 
 8779 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8780 
 8781   predicate(needs_acquiring_load_exclusive(n));
 8782   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8783   ins_cost(VOLATILE_REF_COST);
 8784 
 8785   effect(KILL cr);
 8786 
 8787   format %{
 8788     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8789     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8790   %}
 8791 
 8792   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 8793             aarch64_enc_cset_eq(res));
 8794 
 8795   ins_pipe(pipe_slow);
 8796 %}
 8797 
 8798 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8799 
 8800   predicate(needs_acquiring_load_exclusive(n));
 8801   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8802   ins_cost(VOLATILE_REF_COST);
 8803 
 8804   effect(KILL cr);
 8805 
 8806   format %{
 8807     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8808     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8809   %}
 8810 
 8811   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 8812             aarch64_enc_cset_eq(res));
 8813 
 8814   ins_pipe(pipe_slow);
 8815 %}
 8816 
 8817 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8818 
 8819   predicate(needs_acquiring_load_exclusive(n));
 8820   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8821   ins_cost(VOLATILE_REF_COST);
 8822 
 8823   effect(KILL cr);
 8824 
 8825  format %{
 8826     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8827     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8828  %}
 8829 
 8830  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8831             aarch64_enc_cset_eq(res));
 8832 
 8833   ins_pipe(pipe_slow);
 8834 %}
 8835 
 8836 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8837 
 8838   predicate(needs_acquiring_load_exclusive(n));
 8839   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8840   ins_cost(VOLATILE_REF_COST);
 8841 
 8842   effect(KILL cr);
 8843 
 8844  format %{
 8845     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8846     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8847  %}
 8848 
 8849  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8850             aarch64_enc_cset_eq(res));
 8851 
 8852   ins_pipe(pipe_slow);
 8853 %}
 8854 
 8855 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8856 
 8857   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8858   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8859   ins_cost(VOLATILE_REF_COST);
 8860 
 8861   effect(KILL cr);
 8862 
 8863  format %{
 8864     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8865     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8866  %}
 8867 
 8868  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8869             aarch64_enc_cset_eq(res));
 8870 
 8871   ins_pipe(pipe_slow);
 8872 %}
 8873 
 8874 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8875 
 8876   predicate(needs_acquiring_load_exclusive(n));
 8877   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8878   ins_cost(VOLATILE_REF_COST);
 8879 
 8880   effect(KILL cr);
 8881 
 8882  format %{
 8883     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8884     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8885  %}
 8886 
 8887  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8888             aarch64_enc_cset_eq(res));
 8889 
 8890   ins_pipe(pipe_slow);
 8891 %}
 8892 
 8893 
 8894 // ---------------------------------------------------------------------
 8895 
 8896 // BEGIN This section of the file is automatically generated. Do not edit --------------
 8897 
 8898 // Sundry CAS operations.  Note that release is always true,
 8899 // regardless of the memory ordering of the CAS.  This is because we
 8900 // need the volatile case to be sequentially consistent but there is
 8901 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 8902 // can't check the type of memory ordering here, so we always emit a
 8903 // STLXR.
 8904 
 8905 // This section is generated from cas.m4
 8906 
 8907 
 8908 // This pattern is generated automatically from cas.m4.
 8909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8910 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8911   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8912   ins_cost(2 * VOLATILE_REF_COST);
 8913   effect(TEMP_DEF res, KILL cr);
 8914   format %{
 8915     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8916   %}
 8917   ins_encode %{
 8918     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8919                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8920                /*weak*/ false, $res$$Register);
 8921     __ sxtbw($res$$Register, $res$$Register);
 8922   %}
 8923   ins_pipe(pipe_slow);
 8924 %}
 8925 
 8926 // This pattern is generated automatically from cas.m4.
 8927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8928 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8929   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 8930   ins_cost(2 * VOLATILE_REF_COST);
 8931   effect(TEMP_DEF res, KILL cr);
 8932   format %{
 8933     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8934   %}
 8935   ins_encode %{
 8936     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8937                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 8938                /*weak*/ false, $res$$Register);
 8939     __ sxthw($res$$Register, $res$$Register);
 8940   %}
 8941   ins_pipe(pipe_slow);
 8942 %}
 8943 
 8944 // This pattern is generated automatically from cas.m4.
 8945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8946 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8947   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 8948   ins_cost(2 * VOLATILE_REF_COST);
 8949   effect(TEMP_DEF res, KILL cr);
 8950   format %{
 8951     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8952   %}
 8953   ins_encode %{
 8954     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8955                Assembler::word, /*acquire*/ false, /*release*/ true,
 8956                /*weak*/ false, $res$$Register);
 8957   %}
 8958   ins_pipe(pipe_slow);
 8959 %}
 8960 
 8961 // This pattern is generated automatically from cas.m4.
 8962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8963 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8964   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 8965   ins_cost(2 * VOLATILE_REF_COST);
 8966   effect(TEMP_DEF res, KILL cr);
 8967   format %{
 8968     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8969   %}
 8970   ins_encode %{
 8971     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8972                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8973                /*weak*/ false, $res$$Register);
 8974   %}
 8975   ins_pipe(pipe_slow);
 8976 %}
 8977 
 8978 // This pattern is generated automatically from cas.m4.
 8979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8980 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8981   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 8982   ins_cost(2 * VOLATILE_REF_COST);
 8983   effect(TEMP_DEF res, KILL cr);
 8984   format %{
 8985     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8986   %}
 8987   ins_encode %{
 8988     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8989                Assembler::word, /*acquire*/ false, /*release*/ true,
 8990                /*weak*/ false, $res$$Register);
 8991   %}
 8992   ins_pipe(pipe_slow);
 8993 %}
 8994 
 8995 // This pattern is generated automatically from cas.m4.
 8996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8997 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8998   predicate(n->as_LoadStore()->barrier_data() == 0);
 8999   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9000   ins_cost(2 * VOLATILE_REF_COST);
 9001   effect(TEMP_DEF res, KILL cr);
 9002   format %{
 9003     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9004   %}
 9005   ins_encode %{
 9006     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9007                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9008                /*weak*/ false, $res$$Register);
 9009   %}
 9010   ins_pipe(pipe_slow);
 9011 %}
 9012 
 9013 // This pattern is generated automatically from cas.m4.
 9014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9015 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9016   predicate(needs_acquiring_load_exclusive(n));
 9017   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9018   ins_cost(VOLATILE_REF_COST);
 9019   effect(TEMP_DEF res, KILL cr);
 9020   format %{
 9021     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9022   %}
 9023   ins_encode %{
 9024     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9025                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9026                /*weak*/ false, $res$$Register);
 9027     __ sxtbw($res$$Register, $res$$Register);
 9028   %}
 9029   ins_pipe(pipe_slow);
 9030 %}
 9031 
 9032 // This pattern is generated automatically from cas.m4.
 9033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9034 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9035   predicate(needs_acquiring_load_exclusive(n));
 9036   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9037   ins_cost(VOLATILE_REF_COST);
 9038   effect(TEMP_DEF res, KILL cr);
 9039   format %{
 9040     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9041   %}
 9042   ins_encode %{
 9043     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9044                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9045                /*weak*/ false, $res$$Register);
 9046     __ sxthw($res$$Register, $res$$Register);
 9047   %}
 9048   ins_pipe(pipe_slow);
 9049 %}
 9050 
 9051 // This pattern is generated automatically from cas.m4.
 9052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9053 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9054   predicate(needs_acquiring_load_exclusive(n));
 9055   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9056   ins_cost(VOLATILE_REF_COST);
 9057   effect(TEMP_DEF res, KILL cr);
 9058   format %{
 9059     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9060   %}
 9061   ins_encode %{
 9062     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9063                Assembler::word, /*acquire*/ true, /*release*/ true,
 9064                /*weak*/ false, $res$$Register);
 9065   %}
 9066   ins_pipe(pipe_slow);
 9067 %}
 9068 
 9069 // This pattern is generated automatically from cas.m4.
 9070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9071 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9072   predicate(needs_acquiring_load_exclusive(n));
 9073   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9074   ins_cost(VOLATILE_REF_COST);
 9075   effect(TEMP_DEF res, KILL cr);
 9076   format %{
 9077     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9078   %}
 9079   ins_encode %{
 9080     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9081                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9082                /*weak*/ false, $res$$Register);
 9083   %}
 9084   ins_pipe(pipe_slow);
 9085 %}
 9086 
 9087 // This pattern is generated automatically from cas.m4.
 9088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9089 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9090   predicate(needs_acquiring_load_exclusive(n));
 9091   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9092   ins_cost(VOLATILE_REF_COST);
 9093   effect(TEMP_DEF res, KILL cr);
 9094   format %{
 9095     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9096   %}
 9097   ins_encode %{
 9098     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9099                Assembler::word, /*acquire*/ true, /*release*/ true,
 9100                /*weak*/ false, $res$$Register);
 9101   %}
 9102   ins_pipe(pipe_slow);
 9103 %}
 9104 
 9105 // This pattern is generated automatically from cas.m4.
 9106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9107 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9108   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9109   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9110   ins_cost(VOLATILE_REF_COST);
 9111   effect(TEMP_DEF res, KILL cr);
 9112   format %{
 9113     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9114   %}
 9115   ins_encode %{
 9116     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9117                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9118                /*weak*/ false, $res$$Register);
 9119   %}
 9120   ins_pipe(pipe_slow);
 9121 %}
 9122 
 9123 // This pattern is generated automatically from cas.m4.
 9124 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9125 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9126   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9127   ins_cost(2 * VOLATILE_REF_COST);
 9128   effect(KILL cr);
 9129   format %{
 9130     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9131     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9132   %}
 9133   ins_encode %{
 9134     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9135                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9136                /*weak*/ true, noreg);
 9137     __ csetw($res$$Register, Assembler::EQ);
 9138   %}
 9139   ins_pipe(pipe_slow);
 9140 %}
 9141 
 9142 // This pattern is generated automatically from cas.m4.
 9143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9144 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9145   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9146   ins_cost(2 * VOLATILE_REF_COST);
 9147   effect(KILL cr);
 9148   format %{
 9149     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9150     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9151   %}
 9152   ins_encode %{
 9153     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9154                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9155                /*weak*/ true, noreg);
 9156     __ csetw($res$$Register, Assembler::EQ);
 9157   %}
 9158   ins_pipe(pipe_slow);
 9159 %}
 9160 
 9161 // This pattern is generated automatically from cas.m4.
 9162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9163 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9164   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9165   ins_cost(2 * VOLATILE_REF_COST);
 9166   effect(KILL cr);
 9167   format %{
 9168     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9169     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9170   %}
 9171   ins_encode %{
 9172     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9173                Assembler::word, /*acquire*/ false, /*release*/ true,
 9174                /*weak*/ true, noreg);
 9175     __ csetw($res$$Register, Assembler::EQ);
 9176   %}
 9177   ins_pipe(pipe_slow);
 9178 %}
 9179 
 9180 // This pattern is generated automatically from cas.m4.
 9181 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9182 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9183   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9184   ins_cost(2 * VOLATILE_REF_COST);
 9185   effect(KILL cr);
 9186   format %{
 9187     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9188     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9189   %}
 9190   ins_encode %{
 9191     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9192                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9193                /*weak*/ true, noreg);
 9194     __ csetw($res$$Register, Assembler::EQ);
 9195   %}
 9196   ins_pipe(pipe_slow);
 9197 %}
 9198 
 9199 // This pattern is generated automatically from cas.m4.
 9200 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9201 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9202   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9203   ins_cost(2 * VOLATILE_REF_COST);
 9204   effect(KILL cr);
 9205   format %{
 9206     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9207     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9208   %}
 9209   ins_encode %{
 9210     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9211                Assembler::word, /*acquire*/ false, /*release*/ true,
 9212                /*weak*/ true, noreg);
 9213     __ csetw($res$$Register, Assembler::EQ);
 9214   %}
 9215   ins_pipe(pipe_slow);
 9216 %}
 9217 
 9218 // This pattern is generated automatically from cas.m4.
 9219 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9220 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9221   predicate(n->as_LoadStore()->barrier_data() == 0);
 9222   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9223   ins_cost(2 * VOLATILE_REF_COST);
 9224   effect(KILL cr);
 9225   format %{
 9226     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9227     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9228   %}
 9229   ins_encode %{
 9230     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9231                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9232                /*weak*/ true, noreg);
 9233     __ csetw($res$$Register, Assembler::EQ);
 9234   %}
 9235   ins_pipe(pipe_slow);
 9236 %}
 9237 
 9238 // This pattern is generated automatically from cas.m4.
 9239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9240 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9241   predicate(needs_acquiring_load_exclusive(n));
 9242   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9243   ins_cost(VOLATILE_REF_COST);
 9244   effect(KILL cr);
 9245   format %{
 9246     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9247     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9248   %}
 9249   ins_encode %{
 9250     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9251                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9252                /*weak*/ true, noreg);
 9253     __ csetw($res$$Register, Assembler::EQ);
 9254   %}
 9255   ins_pipe(pipe_slow);
 9256 %}
 9257 
 9258 // This pattern is generated automatically from cas.m4.
 9259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9260 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9261   predicate(needs_acquiring_load_exclusive(n));
 9262   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9263   ins_cost(VOLATILE_REF_COST);
 9264   effect(KILL cr);
 9265   format %{
 9266     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9267     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9268   %}
 9269   ins_encode %{
 9270     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9271                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9272                /*weak*/ true, noreg);
 9273     __ csetw($res$$Register, Assembler::EQ);
 9274   %}
 9275   ins_pipe(pipe_slow);
 9276 %}
 9277 
 9278 // This pattern is generated automatically from cas.m4.
 9279 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9280 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9281   predicate(needs_acquiring_load_exclusive(n));
 9282   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9283   ins_cost(VOLATILE_REF_COST);
 9284   effect(KILL cr);
 9285   format %{
 9286     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9287     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9288   %}
 9289   ins_encode %{
 9290     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9291                Assembler::word, /*acquire*/ true, /*release*/ true,
 9292                /*weak*/ true, noreg);
 9293     __ csetw($res$$Register, Assembler::EQ);
 9294   %}
 9295   ins_pipe(pipe_slow);
 9296 %}
 9297 
 9298 // This pattern is generated automatically from cas.m4.
 9299 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9300 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9301   predicate(needs_acquiring_load_exclusive(n));
 9302   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9303   ins_cost(VOLATILE_REF_COST);
 9304   effect(KILL cr);
 9305   format %{
 9306     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9307     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9308   %}
 9309   ins_encode %{
 9310     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9311                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9312                /*weak*/ true, noreg);
 9313     __ csetw($res$$Register, Assembler::EQ);
 9314   %}
 9315   ins_pipe(pipe_slow);
 9316 %}
 9317 
 9318 // This pattern is generated automatically from cas.m4.
 9319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9320 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9321   predicate(needs_acquiring_load_exclusive(n));
 9322   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9323   ins_cost(VOLATILE_REF_COST);
 9324   effect(KILL cr);
 9325   format %{
 9326     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9327     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9328   %}
 9329   ins_encode %{
 9330     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9331                Assembler::word, /*acquire*/ true, /*release*/ true,
 9332                /*weak*/ true, noreg);
 9333     __ csetw($res$$Register, Assembler::EQ);
 9334   %}
 9335   ins_pipe(pipe_slow);
 9336 %}
 9337 
 9338 // This pattern is generated automatically from cas.m4.
 9339 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9340 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9341   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9342   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9343   ins_cost(VOLATILE_REF_COST);
 9344   effect(KILL cr);
 9345   format %{
 9346     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9347     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9348   %}
 9349   ins_encode %{
 9350     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9351                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9352                /*weak*/ true, noreg);
 9353     __ csetw($res$$Register, Assembler::EQ);
 9354   %}
 9355   ins_pipe(pipe_slow);
 9356 %}
 9357 
 9358 // END This section of the file is automatically generated. Do not edit --------------
 9359 // ---------------------------------------------------------------------
 9360 
 9361 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9362   match(Set prev (GetAndSetI mem newv));
 9363   ins_cost(2 * VOLATILE_REF_COST);
 9364   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9365   ins_encode %{
 9366     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9367   %}
 9368   ins_pipe(pipe_serial);
 9369 %}
 9370 
 9371 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9372   match(Set prev (GetAndSetL mem newv));
 9373   ins_cost(2 * VOLATILE_REF_COST);
 9374   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9375   ins_encode %{
 9376     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9377   %}
 9378   ins_pipe(pipe_serial);
 9379 %}
 9380 
 9381 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9382   match(Set prev (GetAndSetN mem newv));
 9383   ins_cost(2 * VOLATILE_REF_COST);
 9384   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9385   ins_encode %{
 9386     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9387   %}
 9388   ins_pipe(pipe_serial);
 9389 %}
 9390 
 9391 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9392   predicate(n->as_LoadStore()->barrier_data() == 0);
 9393   match(Set prev (GetAndSetP mem newv));
 9394   ins_cost(2 * VOLATILE_REF_COST);
 9395   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9396   ins_encode %{
 9397     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9398   %}
 9399   ins_pipe(pipe_serial);
 9400 %}
 9401 
 9402 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9403   predicate(needs_acquiring_load_exclusive(n));
 9404   match(Set prev (GetAndSetI mem newv));
 9405   ins_cost(VOLATILE_REF_COST);
 9406   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9407   ins_encode %{
 9408     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9409   %}
 9410   ins_pipe(pipe_serial);
 9411 %}
 9412 
 9413 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9414   predicate(needs_acquiring_load_exclusive(n));
 9415   match(Set prev (GetAndSetL mem newv));
 9416   ins_cost(VOLATILE_REF_COST);
 9417   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9418   ins_encode %{
 9419     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9420   %}
 9421   ins_pipe(pipe_serial);
 9422 %}
 9423 
 9424 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9425   predicate(needs_acquiring_load_exclusive(n));
 9426   match(Set prev (GetAndSetN mem newv));
 9427   ins_cost(VOLATILE_REF_COST);
 9428   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9429   ins_encode %{
 9430     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9431   %}
 9432   ins_pipe(pipe_serial);
 9433 %}
 9434 
 9435 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9436   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9437   match(Set prev (GetAndSetP mem newv));
 9438   ins_cost(VOLATILE_REF_COST);
 9439   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9440   ins_encode %{
 9441     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9442   %}
 9443   ins_pipe(pipe_serial);
 9444 %}
 9445 
 9446 
 9447 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9448   match(Set newval (GetAndAddL mem incr));
 9449   ins_cost(2 * VOLATILE_REF_COST + 1);
 9450   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9451   ins_encode %{
 9452     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9453   %}
 9454   ins_pipe(pipe_serial);
 9455 %}
 9456 
 9457 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9458   predicate(n->as_LoadStore()->result_not_used());
 9459   match(Set dummy (GetAndAddL mem incr));
 9460   ins_cost(2 * VOLATILE_REF_COST);
 9461   format %{ "get_and_addL [$mem], $incr" %}
 9462   ins_encode %{
 9463     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9464   %}
 9465   ins_pipe(pipe_serial);
 9466 %}
 9467 
 9468 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9469   match(Set newval (GetAndAddL mem incr));
 9470   ins_cost(2 * VOLATILE_REF_COST + 1);
 9471   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9472   ins_encode %{
 9473     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9474   %}
 9475   ins_pipe(pipe_serial);
 9476 %}
 9477 
 9478 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9479   predicate(n->as_LoadStore()->result_not_used());
 9480   match(Set dummy (GetAndAddL mem incr));
 9481   ins_cost(2 * VOLATILE_REF_COST);
 9482   format %{ "get_and_addL [$mem], $incr" %}
 9483   ins_encode %{
 9484     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9485   %}
 9486   ins_pipe(pipe_serial);
 9487 %}
 9488 
 9489 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9490   match(Set newval (GetAndAddI mem incr));
 9491   ins_cost(2 * VOLATILE_REF_COST + 1);
 9492   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9493   ins_encode %{
 9494     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9495   %}
 9496   ins_pipe(pipe_serial);
 9497 %}
 9498 
 9499 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9500   predicate(n->as_LoadStore()->result_not_used());
 9501   match(Set dummy (GetAndAddI mem incr));
 9502   ins_cost(2 * VOLATILE_REF_COST);
 9503   format %{ "get_and_addI [$mem], $incr" %}
 9504   ins_encode %{
 9505     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9506   %}
 9507   ins_pipe(pipe_serial);
 9508 %}
 9509 
 9510 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9511   match(Set newval (GetAndAddI mem incr));
 9512   ins_cost(2 * VOLATILE_REF_COST + 1);
 9513   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9514   ins_encode %{
 9515     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9516   %}
 9517   ins_pipe(pipe_serial);
 9518 %}
 9519 
 9520 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9521   predicate(n->as_LoadStore()->result_not_used());
 9522   match(Set dummy (GetAndAddI mem incr));
 9523   ins_cost(2 * VOLATILE_REF_COST);
 9524   format %{ "get_and_addI [$mem], $incr" %}
 9525   ins_encode %{
 9526     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9527   %}
 9528   ins_pipe(pipe_serial);
 9529 %}
 9530 
 9531 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9532   predicate(needs_acquiring_load_exclusive(n));
 9533   match(Set newval (GetAndAddL mem incr));
 9534   ins_cost(VOLATILE_REF_COST + 1);
 9535   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9536   ins_encode %{
 9537     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9538   %}
 9539   ins_pipe(pipe_serial);
 9540 %}
 9541 
 9542 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9543   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9544   match(Set dummy (GetAndAddL mem incr));
 9545   ins_cost(VOLATILE_REF_COST);
 9546   format %{ "get_and_addL_acq [$mem], $incr" %}
 9547   ins_encode %{
 9548     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9549   %}
 9550   ins_pipe(pipe_serial);
 9551 %}
 9552 
 9553 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9554   predicate(needs_acquiring_load_exclusive(n));
 9555   match(Set newval (GetAndAddL mem incr));
 9556   ins_cost(VOLATILE_REF_COST + 1);
 9557   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9558   ins_encode %{
 9559     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9560   %}
 9561   ins_pipe(pipe_serial);
 9562 %}
 9563 
 9564 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9565   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9566   match(Set dummy (GetAndAddL mem incr));
 9567   ins_cost(VOLATILE_REF_COST);
 9568   format %{ "get_and_addL_acq [$mem], $incr" %}
 9569   ins_encode %{
 9570     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9571   %}
 9572   ins_pipe(pipe_serial);
 9573 %}
 9574 
 9575 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9576   predicate(needs_acquiring_load_exclusive(n));
 9577   match(Set newval (GetAndAddI mem incr));
 9578   ins_cost(VOLATILE_REF_COST + 1);
 9579   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9580   ins_encode %{
 9581     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9582   %}
 9583   ins_pipe(pipe_serial);
 9584 %}
 9585 
 9586 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9587   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9588   match(Set dummy (GetAndAddI mem incr));
 9589   ins_cost(VOLATILE_REF_COST);
 9590   format %{ "get_and_addI_acq [$mem], $incr" %}
 9591   ins_encode %{
 9592     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9593   %}
 9594   ins_pipe(pipe_serial);
 9595 %}
 9596 
 9597 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9598   predicate(needs_acquiring_load_exclusive(n));
 9599   match(Set newval (GetAndAddI mem incr));
 9600   ins_cost(VOLATILE_REF_COST + 1);
 9601   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9602   ins_encode %{
 9603     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9604   %}
 9605   ins_pipe(pipe_serial);
 9606 %}
 9607 
 9608 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9609   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9610   match(Set dummy (GetAndAddI mem incr));
 9611   ins_cost(VOLATILE_REF_COST);
 9612   format %{ "get_and_addI_acq [$mem], $incr" %}
 9613   ins_encode %{
 9614     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9615   %}
 9616   ins_pipe(pipe_serial);
 9617 %}
 9618 
 9619 // Manifest a CmpU result in an integer register.
 9620 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9621 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
 9622 %{
 9623   match(Set dst (CmpU3 src1 src2));
 9624   effect(KILL flags);
 9625 
 9626   ins_cost(INSN_COST * 3);
 9627   format %{
 9628       "cmpw $src1, $src2\n\t"
 9629       "csetw $dst, ne\n\t"
 9630       "cnegw $dst, lo\t# CmpU3(reg)"
 9631   %}
 9632   ins_encode %{
 9633     __ cmpw($src1$$Register, $src2$$Register);
 9634     __ csetw($dst$$Register, Assembler::NE);
 9635     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9636   %}
 9637 
 9638   ins_pipe(pipe_class_default);
 9639 %}
 9640 
 9641 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
 9642 %{
 9643   match(Set dst (CmpU3 src1 src2));
 9644   effect(KILL flags);
 9645 
 9646   ins_cost(INSN_COST * 3);
 9647   format %{
 9648       "subsw zr, $src1, $src2\n\t"
 9649       "csetw $dst, ne\n\t"
 9650       "cnegw $dst, lo\t# CmpU3(imm)"
 9651   %}
 9652   ins_encode %{
 9653     __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
 9654     __ csetw($dst$$Register, Assembler::NE);
 9655     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9656   %}
 9657 
 9658   ins_pipe(pipe_class_default);
 9659 %}
 9660 
 9661 // Manifest a CmpUL result in an integer register.
 9662 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9663 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9664 %{
 9665   match(Set dst (CmpUL3 src1 src2));
 9666   effect(KILL flags);
 9667 
 9668   ins_cost(INSN_COST * 3);
 9669   format %{
 9670       "cmp $src1, $src2\n\t"
 9671       "csetw $dst, ne\n\t"
 9672       "cnegw $dst, lo\t# CmpUL3(reg)"
 9673   %}
 9674   ins_encode %{
 9675     __ cmp($src1$$Register, $src2$$Register);
 9676     __ csetw($dst$$Register, Assembler::NE);
 9677     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9678   %}
 9679 
 9680   ins_pipe(pipe_class_default);
 9681 %}
 9682 
 9683 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9684 %{
 9685   match(Set dst (CmpUL3 src1 src2));
 9686   effect(KILL flags);
 9687 
 9688   ins_cost(INSN_COST * 3);
 9689   format %{
 9690       "subs zr, $src1, $src2\n\t"
 9691       "csetw $dst, ne\n\t"
 9692       "cnegw $dst, lo\t# CmpUL3(imm)"
 9693   %}
 9694   ins_encode %{
 9695     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9696     __ csetw($dst$$Register, Assembler::NE);
 9697     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9698   %}
 9699 
 9700   ins_pipe(pipe_class_default);
 9701 %}
 9702 
 9703 // Manifest a CmpL result in an integer register.
 9704 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9705 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9706 %{
 9707   match(Set dst (CmpL3 src1 src2));
 9708   effect(KILL flags);
 9709 
 9710   ins_cost(INSN_COST * 3);
 9711   format %{
 9712       "cmp $src1, $src2\n\t"
 9713       "csetw $dst, ne\n\t"
 9714       "cnegw $dst, lt\t# CmpL3(reg)"
 9715   %}
 9716   ins_encode %{
 9717     __ cmp($src1$$Register, $src2$$Register);
 9718     __ csetw($dst$$Register, Assembler::NE);
 9719     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9720   %}
 9721 
 9722   ins_pipe(pipe_class_default);
 9723 %}
 9724 
 9725 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9726 %{
 9727   match(Set dst (CmpL3 src1 src2));
 9728   effect(KILL flags);
 9729 
 9730   ins_cost(INSN_COST * 3);
 9731   format %{
 9732       "subs zr, $src1, $src2\n\t"
 9733       "csetw $dst, ne\n\t"
 9734       "cnegw $dst, lt\t# CmpL3(imm)"
 9735   %}
 9736   ins_encode %{
 9737     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9738     __ csetw($dst$$Register, Assembler::NE);
 9739     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9740   %}
 9741 
 9742   ins_pipe(pipe_class_default);
 9743 %}
 9744 
 9745 // ============================================================================
 9746 // Conditional Move Instructions
 9747 
 9748 // n.b. we have identical rules for both a signed compare op (cmpOp)
 9749 // and an unsigned compare op (cmpOpU). it would be nice if we could
 9750 // define an op class which merged both inputs and use it to type the
 9751 // argument to a single rule. unfortunatelyt his fails because the
 9752 // opclass does not live up to the COND_INTER interface of its
 9753 // component operands. When the generic code tries to negate the
 9754 // operand it ends up running the generci Machoper::negate method
 9755 // which throws a ShouldNotHappen. So, we have to provide two flavours
 9756 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
 9757 
 9758 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9759   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9760 
 9761   ins_cost(INSN_COST * 2);
 9762   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
 9763 
 9764   ins_encode %{
 9765     __ cselw(as_Register($dst$$reg),
 9766              as_Register($src2$$reg),
 9767              as_Register($src1$$reg),
 9768              (Assembler::Condition)$cmp$$cmpcode);
 9769   %}
 9770 
 9771   ins_pipe(icond_reg_reg);
 9772 %}
 9773 
 9774 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9775   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9776 
 9777   ins_cost(INSN_COST * 2);
 9778   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
 9779 
 9780   ins_encode %{
 9781     __ cselw(as_Register($dst$$reg),
 9782              as_Register($src2$$reg),
 9783              as_Register($src1$$reg),
 9784              (Assembler::Condition)$cmp$$cmpcode);
 9785   %}
 9786 
 9787   ins_pipe(icond_reg_reg);
 9788 %}
 9789 
 9790 // special cases where one arg is zero
 9791 
 9792 // n.b. this is selected in preference to the rule above because it
 9793 // avoids loading constant 0 into a source register
 9794 
 9795 // TODO
 9796 // we ought only to be able to cull one of these variants as the ideal
 9797 // transforms ought always to order the zero consistently (to left/right?)
 9798 
 9799 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9800   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9801 
 9802   ins_cost(INSN_COST * 2);
 9803   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
 9804 
 9805   ins_encode %{
 9806     __ cselw(as_Register($dst$$reg),
 9807              as_Register($src$$reg),
 9808              zr,
 9809              (Assembler::Condition)$cmp$$cmpcode);
 9810   %}
 9811 
 9812   ins_pipe(icond_reg);
 9813 %}
 9814 
 9815 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9816   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9817 
 9818   ins_cost(INSN_COST * 2);
 9819   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
 9820 
 9821   ins_encode %{
 9822     __ cselw(as_Register($dst$$reg),
 9823              as_Register($src$$reg),
 9824              zr,
 9825              (Assembler::Condition)$cmp$$cmpcode);
 9826   %}
 9827 
 9828   ins_pipe(icond_reg);
 9829 %}
 9830 
 9831 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9832   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9833 
 9834   ins_cost(INSN_COST * 2);
 9835   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
 9836 
 9837   ins_encode %{
 9838     __ cselw(as_Register($dst$$reg),
 9839              zr,
 9840              as_Register($src$$reg),
 9841              (Assembler::Condition)$cmp$$cmpcode);
 9842   %}
 9843 
 9844   ins_pipe(icond_reg);
 9845 %}
 9846 
 9847 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9848   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9849 
 9850   ins_cost(INSN_COST * 2);
 9851   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
 9852 
 9853   ins_encode %{
 9854     __ cselw(as_Register($dst$$reg),
 9855              zr,
 9856              as_Register($src$$reg),
 9857              (Assembler::Condition)$cmp$$cmpcode);
 9858   %}
 9859 
 9860   ins_pipe(icond_reg);
 9861 %}
 9862 
 9863 // special case for creating a boolean 0 or 1
 9864 
 9865 // n.b. this is selected in preference to the rule above because it
 9866 // avoids loading constants 0 and 1 into a source register
 9867 
 9868 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9869   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9870 
 9871   ins_cost(INSN_COST * 2);
 9872   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
 9873 
 9874   ins_encode %{
 9875     // equivalently
 9876     // cset(as_Register($dst$$reg),
 9877     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9878     __ csincw(as_Register($dst$$reg),
 9879              zr,
 9880              zr,
 9881              (Assembler::Condition)$cmp$$cmpcode);
 9882   %}
 9883 
 9884   ins_pipe(icond_none);
 9885 %}
 9886 
 9887 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9888   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9889 
 9890   ins_cost(INSN_COST * 2);
 9891   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
 9892 
 9893   ins_encode %{
 9894     // equivalently
 9895     // cset(as_Register($dst$$reg),
 9896     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9897     __ csincw(as_Register($dst$$reg),
 9898              zr,
 9899              zr,
 9900              (Assembler::Condition)$cmp$$cmpcode);
 9901   %}
 9902 
 9903   ins_pipe(icond_none);
 9904 %}
 9905 
 9906 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9907   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9908 
 9909   ins_cost(INSN_COST * 2);
 9910   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
 9911 
 9912   ins_encode %{
 9913     __ csel(as_Register($dst$$reg),
 9914             as_Register($src2$$reg),
 9915             as_Register($src1$$reg),
 9916             (Assembler::Condition)$cmp$$cmpcode);
 9917   %}
 9918 
 9919   ins_pipe(icond_reg_reg);
 9920 %}
 9921 
 9922 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9923   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9924 
 9925   ins_cost(INSN_COST * 2);
 9926   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
 9927 
 9928   ins_encode %{
 9929     __ csel(as_Register($dst$$reg),
 9930             as_Register($src2$$reg),
 9931             as_Register($src1$$reg),
 9932             (Assembler::Condition)$cmp$$cmpcode);
 9933   %}
 9934 
 9935   ins_pipe(icond_reg_reg);
 9936 %}
 9937 
 9938 // special cases where one arg is zero
 9939 
 9940 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9941   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9942 
 9943   ins_cost(INSN_COST * 2);
 9944   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
 9945 
 9946   ins_encode %{
 9947     __ csel(as_Register($dst$$reg),
 9948             zr,
 9949             as_Register($src$$reg),
 9950             (Assembler::Condition)$cmp$$cmpcode);
 9951   %}
 9952 
 9953   ins_pipe(icond_reg);
 9954 %}
 9955 
 9956 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9957   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9958 
 9959   ins_cost(INSN_COST * 2);
 9960   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
 9961 
 9962   ins_encode %{
 9963     __ csel(as_Register($dst$$reg),
 9964             zr,
 9965             as_Register($src$$reg),
 9966             (Assembler::Condition)$cmp$$cmpcode);
 9967   %}
 9968 
 9969   ins_pipe(icond_reg);
 9970 %}
 9971 
 9972 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9973   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9974 
 9975   ins_cost(INSN_COST * 2);
 9976   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
 9977 
 9978   ins_encode %{
 9979     __ csel(as_Register($dst$$reg),
 9980             as_Register($src$$reg),
 9981             zr,
 9982             (Assembler::Condition)$cmp$$cmpcode);
 9983   %}
 9984 
 9985   ins_pipe(icond_reg);
 9986 %}
 9987 
 9988 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9989   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9990 
 9991   ins_cost(INSN_COST * 2);
 9992   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
 9993 
 9994   ins_encode %{
 9995     __ csel(as_Register($dst$$reg),
 9996             as_Register($src$$reg),
 9997             zr,
 9998             (Assembler::Condition)$cmp$$cmpcode);
 9999   %}
10000 
10001   ins_pipe(icond_reg);
10002 %}
10003 
10004 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10005   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10006 
10007   ins_cost(INSN_COST * 2);
10008   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10009 
10010   ins_encode %{
10011     __ csel(as_Register($dst$$reg),
10012             as_Register($src2$$reg),
10013             as_Register($src1$$reg),
10014             (Assembler::Condition)$cmp$$cmpcode);
10015   %}
10016 
10017   ins_pipe(icond_reg_reg);
10018 %}
10019 
10020 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10021   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10022 
10023   ins_cost(INSN_COST * 2);
10024   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10025 
10026   ins_encode %{
10027     __ csel(as_Register($dst$$reg),
10028             as_Register($src2$$reg),
10029             as_Register($src1$$reg),
10030             (Assembler::Condition)$cmp$$cmpcode);
10031   %}
10032 
10033   ins_pipe(icond_reg_reg);
10034 %}
10035 
10036 // special cases where one arg is zero
10037 
10038 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10039   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10040 
10041   ins_cost(INSN_COST * 2);
10042   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10043 
10044   ins_encode %{
10045     __ csel(as_Register($dst$$reg),
10046             zr,
10047             as_Register($src$$reg),
10048             (Assembler::Condition)$cmp$$cmpcode);
10049   %}
10050 
10051   ins_pipe(icond_reg);
10052 %}
10053 
10054 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10055   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10056 
10057   ins_cost(INSN_COST * 2);
10058   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10059 
10060   ins_encode %{
10061     __ csel(as_Register($dst$$reg),
10062             zr,
10063             as_Register($src$$reg),
10064             (Assembler::Condition)$cmp$$cmpcode);
10065   %}
10066 
10067   ins_pipe(icond_reg);
10068 %}
10069 
10070 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10071   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10072 
10073   ins_cost(INSN_COST * 2);
10074   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10075 
10076   ins_encode %{
10077     __ csel(as_Register($dst$$reg),
10078             as_Register($src$$reg),
10079             zr,
10080             (Assembler::Condition)$cmp$$cmpcode);
10081   %}
10082 
10083   ins_pipe(icond_reg);
10084 %}
10085 
10086 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10087   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10088 
10089   ins_cost(INSN_COST * 2);
10090   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10091 
10092   ins_encode %{
10093     __ csel(as_Register($dst$$reg),
10094             as_Register($src$$reg),
10095             zr,
10096             (Assembler::Condition)$cmp$$cmpcode);
10097   %}
10098 
10099   ins_pipe(icond_reg);
10100 %}
10101 
10102 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10103   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10104 
10105   ins_cost(INSN_COST * 2);
10106   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10107 
10108   ins_encode %{
10109     __ cselw(as_Register($dst$$reg),
10110              as_Register($src2$$reg),
10111              as_Register($src1$$reg),
10112              (Assembler::Condition)$cmp$$cmpcode);
10113   %}
10114 
10115   ins_pipe(icond_reg_reg);
10116 %}
10117 
10118 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10119   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10120 
10121   ins_cost(INSN_COST * 2);
10122   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10123 
10124   ins_encode %{
10125     __ cselw(as_Register($dst$$reg),
10126              as_Register($src2$$reg),
10127              as_Register($src1$$reg),
10128              (Assembler::Condition)$cmp$$cmpcode);
10129   %}
10130 
10131   ins_pipe(icond_reg_reg);
10132 %}
10133 
10134 // special cases where one arg is zero
10135 
10136 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10137   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10138 
10139   ins_cost(INSN_COST * 2);
10140   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10141 
10142   ins_encode %{
10143     __ cselw(as_Register($dst$$reg),
10144              zr,
10145              as_Register($src$$reg),
10146              (Assembler::Condition)$cmp$$cmpcode);
10147   %}
10148 
10149   ins_pipe(icond_reg);
10150 %}
10151 
10152 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10153   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10154 
10155   ins_cost(INSN_COST * 2);
10156   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10157 
10158   ins_encode %{
10159     __ cselw(as_Register($dst$$reg),
10160              zr,
10161              as_Register($src$$reg),
10162              (Assembler::Condition)$cmp$$cmpcode);
10163   %}
10164 
10165   ins_pipe(icond_reg);
10166 %}
10167 
10168 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10169   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10170 
10171   ins_cost(INSN_COST * 2);
10172   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10173 
10174   ins_encode %{
10175     __ cselw(as_Register($dst$$reg),
10176              as_Register($src$$reg),
10177              zr,
10178              (Assembler::Condition)$cmp$$cmpcode);
10179   %}
10180 
10181   ins_pipe(icond_reg);
10182 %}
10183 
10184 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10185   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10186 
10187   ins_cost(INSN_COST * 2);
10188   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10189 
10190   ins_encode %{
10191     __ cselw(as_Register($dst$$reg),
10192              as_Register($src$$reg),
10193              zr,
10194              (Assembler::Condition)$cmp$$cmpcode);
10195   %}
10196 
10197   ins_pipe(icond_reg);
10198 %}
10199 
10200 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10201 %{
10202   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10203 
10204   ins_cost(INSN_COST * 3);
10205 
10206   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10207   ins_encode %{
10208     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10209     __ fcsels(as_FloatRegister($dst$$reg),
10210               as_FloatRegister($src2$$reg),
10211               as_FloatRegister($src1$$reg),
10212               cond);
10213   %}
10214 
10215   ins_pipe(fp_cond_reg_reg_s);
10216 %}
10217 
10218 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10219 %{
10220   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10221 
10222   ins_cost(INSN_COST * 3);
10223 
10224   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10225   ins_encode %{
10226     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10227     __ fcsels(as_FloatRegister($dst$$reg),
10228               as_FloatRegister($src2$$reg),
10229               as_FloatRegister($src1$$reg),
10230               cond);
10231   %}
10232 
10233   ins_pipe(fp_cond_reg_reg_s);
10234 %}
10235 
10236 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10237 %{
10238   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10239 
10240   ins_cost(INSN_COST * 3);
10241 
10242   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10243   ins_encode %{
10244     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10245     __ fcseld(as_FloatRegister($dst$$reg),
10246               as_FloatRegister($src2$$reg),
10247               as_FloatRegister($src1$$reg),
10248               cond);
10249   %}
10250 
10251   ins_pipe(fp_cond_reg_reg_d);
10252 %}
10253 
10254 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10255 %{
10256   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10257 
10258   ins_cost(INSN_COST * 3);
10259 
10260   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10261   ins_encode %{
10262     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10263     __ fcseld(as_FloatRegister($dst$$reg),
10264               as_FloatRegister($src2$$reg),
10265               as_FloatRegister($src1$$reg),
10266               cond);
10267   %}
10268 
10269   ins_pipe(fp_cond_reg_reg_d);
10270 %}
10271 
10272 // ============================================================================
10273 // Arithmetic Instructions
10274 //
10275 
10276 // Integer Addition
10277 
10278 // TODO
10279 // these currently employ operations which do not set CR and hence are
10280 // not flagged as killing CR but we would like to isolate the cases
10281 // where we want to set flags from those where we don't. need to work
10282 // out how to do that.
10283 
10284 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10285   match(Set dst (AddI src1 src2));
10286 
10287   ins_cost(INSN_COST);
10288   format %{ "addw  $dst, $src1, $src2" %}
10289 
10290   ins_encode %{
10291     __ addw(as_Register($dst$$reg),
10292             as_Register($src1$$reg),
10293             as_Register($src2$$reg));
10294   %}
10295 
10296   ins_pipe(ialu_reg_reg);
10297 %}
10298 
10299 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10300   match(Set dst (AddI src1 src2));
10301 
10302   ins_cost(INSN_COST);
10303   format %{ "addw $dst, $src1, $src2" %}
10304 
10305   // use opcode to indicate that this is an add not a sub
10306   opcode(0x0);
10307 
10308   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10309 
10310   ins_pipe(ialu_reg_imm);
10311 %}
10312 
10313 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10314   match(Set dst (AddI (ConvL2I src1) src2));
10315 
10316   ins_cost(INSN_COST);
10317   format %{ "addw $dst, $src1, $src2" %}
10318 
10319   // use opcode to indicate that this is an add not a sub
10320   opcode(0x0);
10321 
10322   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10323 
10324   ins_pipe(ialu_reg_imm);
10325 %}
10326 
10327 // Pointer Addition
10328 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10329   match(Set dst (AddP src1 src2));
10330 
10331   ins_cost(INSN_COST);
10332   format %{ "add $dst, $src1, $src2\t# ptr" %}
10333 
10334   ins_encode %{
10335     __ add(as_Register($dst$$reg),
10336            as_Register($src1$$reg),
10337            as_Register($src2$$reg));
10338   %}
10339 
10340   ins_pipe(ialu_reg_reg);
10341 %}
10342 
10343 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10344   match(Set dst (AddP src1 (ConvI2L src2)));
10345 
10346   ins_cost(1.9 * INSN_COST);
10347   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10348 
10349   ins_encode %{
10350     __ add(as_Register($dst$$reg),
10351            as_Register($src1$$reg),
10352            as_Register($src2$$reg), ext::sxtw);
10353   %}
10354 
10355   ins_pipe(ialu_reg_reg);
10356 %}
10357 
10358 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10359   match(Set dst (AddP src1 (LShiftL src2 scale)));
10360 
10361   ins_cost(1.9 * INSN_COST);
10362   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10363 
10364   ins_encode %{
10365     __ lea(as_Register($dst$$reg),
10366            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10367                    Address::lsl($scale$$constant)));
10368   %}
10369 
10370   ins_pipe(ialu_reg_reg_shift);
10371 %}
10372 
10373 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10374   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10375 
10376   ins_cost(1.9 * INSN_COST);
10377   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10378 
10379   ins_encode %{
10380     __ lea(as_Register($dst$$reg),
10381            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10382                    Address::sxtw($scale$$constant)));
10383   %}
10384 
10385   ins_pipe(ialu_reg_reg_shift);
10386 %}
10387 
10388 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10389   match(Set dst (LShiftL (ConvI2L src) scale));
10390 
10391   ins_cost(INSN_COST);
10392   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10393 
10394   ins_encode %{
10395     __ sbfiz(as_Register($dst$$reg),
10396           as_Register($src$$reg),
10397           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10398   %}
10399 
10400   ins_pipe(ialu_reg_shift);
10401 %}
10402 
10403 // Pointer Immediate Addition
10404 // n.b. this needs to be more expensive than using an indirect memory
10405 // operand
10406 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10407   match(Set dst (AddP src1 src2));
10408 
10409   ins_cost(INSN_COST);
10410   format %{ "add $dst, $src1, $src2\t# ptr" %}
10411 
10412   // use opcode to indicate that this is an add not a sub
10413   opcode(0x0);
10414 
10415   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10416 
10417   ins_pipe(ialu_reg_imm);
10418 %}
10419 
10420 // Long Addition
10421 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10422 
10423   match(Set dst (AddL src1 src2));
10424 
10425   ins_cost(INSN_COST);
10426   format %{ "add  $dst, $src1, $src2" %}
10427 
10428   ins_encode %{
10429     __ add(as_Register($dst$$reg),
10430            as_Register($src1$$reg),
10431            as_Register($src2$$reg));
10432   %}
10433 
10434   ins_pipe(ialu_reg_reg);
10435 %}
10436 
10437 // No constant pool entries requiredLong Immediate Addition.
10438 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10439   match(Set dst (AddL src1 src2));
10440 
10441   ins_cost(INSN_COST);
10442   format %{ "add $dst, $src1, $src2" %}
10443 
10444   // use opcode to indicate that this is an add not a sub
10445   opcode(0x0);
10446 
10447   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10448 
10449   ins_pipe(ialu_reg_imm);
10450 %}
10451 
10452 // Integer Subtraction
10453 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10454   match(Set dst (SubI src1 src2));
10455 
10456   ins_cost(INSN_COST);
10457   format %{ "subw  $dst, $src1, $src2" %}
10458 
10459   ins_encode %{
10460     __ subw(as_Register($dst$$reg),
10461             as_Register($src1$$reg),
10462             as_Register($src2$$reg));
10463   %}
10464 
10465   ins_pipe(ialu_reg_reg);
10466 %}
10467 
10468 // Immediate Subtraction
10469 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10470   match(Set dst (SubI src1 src2));
10471 
10472   ins_cost(INSN_COST);
10473   format %{ "subw $dst, $src1, $src2" %}
10474 
10475   // use opcode to indicate that this is a sub not an add
10476   opcode(0x1);
10477 
10478   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10479 
10480   ins_pipe(ialu_reg_imm);
10481 %}
10482 
10483 // Long Subtraction
10484 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10485 
10486   match(Set dst (SubL src1 src2));
10487 
10488   ins_cost(INSN_COST);
10489   format %{ "sub  $dst, $src1, $src2" %}
10490 
10491   ins_encode %{
10492     __ sub(as_Register($dst$$reg),
10493            as_Register($src1$$reg),
10494            as_Register($src2$$reg));
10495   %}
10496 
10497   ins_pipe(ialu_reg_reg);
10498 %}
10499 
10500 // No constant pool entries requiredLong Immediate Subtraction.
10501 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10502   match(Set dst (SubL src1 src2));
10503 
10504   ins_cost(INSN_COST);
10505   format %{ "sub$dst, $src1, $src2" %}
10506 
10507   // use opcode to indicate that this is a sub not an add
10508   opcode(0x1);
10509 
10510   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10511 
10512   ins_pipe(ialu_reg_imm);
10513 %}
10514 
10515 // Integer Negation (special case for sub)
10516 
10517 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10518   match(Set dst (SubI zero src));
10519 
10520   ins_cost(INSN_COST);
10521   format %{ "negw $dst, $src\t# int" %}
10522 
10523   ins_encode %{
10524     __ negw(as_Register($dst$$reg),
10525             as_Register($src$$reg));
10526   %}
10527 
10528   ins_pipe(ialu_reg);
10529 %}
10530 
10531 // Long Negation
10532 
10533 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10534   match(Set dst (SubL zero src));
10535 
10536   ins_cost(INSN_COST);
10537   format %{ "neg $dst, $src\t# long" %}
10538 
10539   ins_encode %{
10540     __ neg(as_Register($dst$$reg),
10541            as_Register($src$$reg));
10542   %}
10543 
10544   ins_pipe(ialu_reg);
10545 %}
10546 
10547 // Integer Multiply
10548 
10549 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10550   match(Set dst (MulI src1 src2));
10551 
10552   ins_cost(INSN_COST * 3);
10553   format %{ "mulw  $dst, $src1, $src2" %}
10554 
10555   ins_encode %{
10556     __ mulw(as_Register($dst$$reg),
10557             as_Register($src1$$reg),
10558             as_Register($src2$$reg));
10559   %}
10560 
10561   ins_pipe(imul_reg_reg);
10562 %}
10563 
10564 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10565   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10566 
10567   ins_cost(INSN_COST * 3);
10568   format %{ "smull  $dst, $src1, $src2" %}
10569 
10570   ins_encode %{
10571     __ smull(as_Register($dst$$reg),
10572              as_Register($src1$$reg),
10573              as_Register($src2$$reg));
10574   %}
10575 
10576   ins_pipe(imul_reg_reg);
10577 %}
10578 
10579 // Long Multiply
10580 
10581 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10582   match(Set dst (MulL src1 src2));
10583 
10584   ins_cost(INSN_COST * 5);
10585   format %{ "mul  $dst, $src1, $src2" %}
10586 
10587   ins_encode %{
10588     __ mul(as_Register($dst$$reg),
10589            as_Register($src1$$reg),
10590            as_Register($src2$$reg));
10591   %}
10592 
10593   ins_pipe(lmul_reg_reg);
10594 %}
10595 
10596 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10597 %{
10598   match(Set dst (MulHiL src1 src2));
10599 
10600   ins_cost(INSN_COST * 7);
10601   format %{ "smulh   $dst, $src1, $src2\t# mulhi" %}
10602 
10603   ins_encode %{
10604     __ smulh(as_Register($dst$$reg),
10605              as_Register($src1$$reg),
10606              as_Register($src2$$reg));
10607   %}
10608 
10609   ins_pipe(lmul_reg_reg);
10610 %}
10611 
10612 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10613 %{
10614   match(Set dst (UMulHiL src1 src2));
10615 
10616   ins_cost(INSN_COST * 7);
10617   format %{ "umulh   $dst, $src1, $src2\t# umulhi" %}
10618 
10619   ins_encode %{
10620     __ umulh(as_Register($dst$$reg),
10621              as_Register($src1$$reg),
10622              as_Register($src2$$reg));
10623   %}
10624 
10625   ins_pipe(lmul_reg_reg);
10626 %}
10627 
10628 // Combined Integer Multiply & Add/Sub
10629 
10630 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10631   match(Set dst (AddI src3 (MulI src1 src2)));
10632 
10633   ins_cost(INSN_COST * 3);
10634   format %{ "madd  $dst, $src1, $src2, $src3" %}
10635 
10636   ins_encode %{
10637     __ maddw(as_Register($dst$$reg),
10638              as_Register($src1$$reg),
10639              as_Register($src2$$reg),
10640              as_Register($src3$$reg));
10641   %}
10642 
10643   ins_pipe(imac_reg_reg);
10644 %}
10645 
10646 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10647   match(Set dst (SubI src3 (MulI src1 src2)));
10648 
10649   ins_cost(INSN_COST * 3);
10650   format %{ "msub  $dst, $src1, $src2, $src3" %}
10651 
10652   ins_encode %{
10653     __ msubw(as_Register($dst$$reg),
10654              as_Register($src1$$reg),
10655              as_Register($src2$$reg),
10656              as_Register($src3$$reg));
10657   %}
10658 
10659   ins_pipe(imac_reg_reg);
10660 %}
10661 
10662 // Combined Integer Multiply & Neg
10663 
10664 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10665   match(Set dst (MulI (SubI zero src1) src2));
10666 
10667   ins_cost(INSN_COST * 3);
10668   format %{ "mneg  $dst, $src1, $src2" %}
10669 
10670   ins_encode %{
10671     __ mnegw(as_Register($dst$$reg),
10672              as_Register($src1$$reg),
10673              as_Register($src2$$reg));
10674   %}
10675 
10676   ins_pipe(imac_reg_reg);
10677 %}
10678 
10679 // Combined Long Multiply & Add/Sub
10680 
10681 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10682   match(Set dst (AddL src3 (MulL src1 src2)));
10683 
10684   ins_cost(INSN_COST * 5);
10685   format %{ "madd  $dst, $src1, $src2, $src3" %}
10686 
10687   ins_encode %{
10688     __ madd(as_Register($dst$$reg),
10689             as_Register($src1$$reg),
10690             as_Register($src2$$reg),
10691             as_Register($src3$$reg));
10692   %}
10693 
10694   ins_pipe(lmac_reg_reg);
10695 %}
10696 
10697 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10698   match(Set dst (SubL src3 (MulL src1 src2)));
10699 
10700   ins_cost(INSN_COST * 5);
10701   format %{ "msub  $dst, $src1, $src2, $src3" %}
10702 
10703   ins_encode %{
10704     __ msub(as_Register($dst$$reg),
10705             as_Register($src1$$reg),
10706             as_Register($src2$$reg),
10707             as_Register($src3$$reg));
10708   %}
10709 
10710   ins_pipe(lmac_reg_reg);
10711 %}
10712 
10713 // Combined Long Multiply & Neg
10714 
10715 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10716   match(Set dst (MulL (SubL zero src1) src2));
10717 
10718   ins_cost(INSN_COST * 5);
10719   format %{ "mneg  $dst, $src1, $src2" %}
10720 
10721   ins_encode %{
10722     __ mneg(as_Register($dst$$reg),
10723             as_Register($src1$$reg),
10724             as_Register($src2$$reg));
10725   %}
10726 
10727   ins_pipe(lmac_reg_reg);
10728 %}
10729 
10730 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10731 
10732 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10733   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10734 
10735   ins_cost(INSN_COST * 3);
10736   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10737 
10738   ins_encode %{
10739     __ smaddl(as_Register($dst$$reg),
10740               as_Register($src1$$reg),
10741               as_Register($src2$$reg),
10742               as_Register($src3$$reg));
10743   %}
10744 
10745   ins_pipe(imac_reg_reg);
10746 %}
10747 
10748 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10749   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10750 
10751   ins_cost(INSN_COST * 3);
10752   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10753 
10754   ins_encode %{
10755     __ smsubl(as_Register($dst$$reg),
10756               as_Register($src1$$reg),
10757               as_Register($src2$$reg),
10758               as_Register($src3$$reg));
10759   %}
10760 
10761   ins_pipe(imac_reg_reg);
10762 %}
10763 
10764 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10765   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10766 
10767   ins_cost(INSN_COST * 3);
10768   format %{ "smnegl  $dst, $src1, $src2" %}
10769 
10770   ins_encode %{
10771     __ smnegl(as_Register($dst$$reg),
10772               as_Register($src1$$reg),
10773               as_Register($src2$$reg));
10774   %}
10775 
10776   ins_pipe(imac_reg_reg);
10777 %}
10778 
10779 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10780 
10781 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10782   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10783 
10784   ins_cost(INSN_COST * 5);
10785   format %{ "mulw  rscratch1, $src1, $src2\n\t"
10786             "maddw $dst, $src3, $src4, rscratch1" %}
10787 
10788   ins_encode %{
10789     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10790     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10791 
10792   ins_pipe(imac_reg_reg);
10793 %}
10794 
10795 // Integer Divide
10796 
10797 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10798   match(Set dst (DivI src1 src2));
10799 
10800   ins_cost(INSN_COST * 19);
10801   format %{ "sdivw  $dst, $src1, $src2" %}
10802 
10803   ins_encode(aarch64_enc_divw(dst, src1, src2));
10804   ins_pipe(idiv_reg_reg);
10805 %}
10806 
10807 // Long Divide
10808 
10809 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10810   match(Set dst (DivL src1 src2));
10811 
10812   ins_cost(INSN_COST * 35);
10813   format %{ "sdiv   $dst, $src1, $src2" %}
10814 
10815   ins_encode(aarch64_enc_div(dst, src1, src2));
10816   ins_pipe(ldiv_reg_reg);
10817 %}
10818 
10819 // Integer Remainder
10820 
10821 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10822   match(Set dst (ModI src1 src2));
10823 
10824   ins_cost(INSN_COST * 22);
10825   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
10826             "msubw  $dst, rscratch1, $src2, $src1" %}
10827 
10828   ins_encode(aarch64_enc_modw(dst, src1, src2));
10829   ins_pipe(idiv_reg_reg);
10830 %}
10831 
10832 // Long Remainder
10833 
10834 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10835   match(Set dst (ModL src1 src2));
10836 
10837   ins_cost(INSN_COST * 38);
10838   format %{ "sdiv   rscratch1, $src1, $src2\n"
10839             "msub   $dst, rscratch1, $src2, $src1" %}
10840 
10841   ins_encode(aarch64_enc_mod(dst, src1, src2));
10842   ins_pipe(ldiv_reg_reg);
10843 %}
10844 
10845 // Unsigned Integer Divide
10846 
10847 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10848   match(Set dst (UDivI src1 src2));
10849 
10850   ins_cost(INSN_COST * 19);
10851   format %{ "udivw  $dst, $src1, $src2" %}
10852 
10853   ins_encode %{
10854     __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10855   %}
10856 
10857   ins_pipe(idiv_reg_reg);
10858 %}
10859 
10860 //  Unsigned Long Divide
10861 
10862 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10863   match(Set dst (UDivL src1 src2));
10864 
10865   ins_cost(INSN_COST * 35);
10866   format %{ "udiv   $dst, $src1, $src2" %}
10867 
10868   ins_encode %{
10869     __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10870   %}
10871 
10872   ins_pipe(ldiv_reg_reg);
10873 %}
10874 
10875 // Unsigned Integer Remainder
10876 
10877 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10878   match(Set dst (UModI src1 src2));
10879 
10880   ins_cost(INSN_COST * 22);
10881   format %{ "udivw  rscratch1, $src1, $src2\n\t"
10882             "msubw  $dst, rscratch1, $src2, $src1" %}
10883 
10884   ins_encode %{
10885     __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10886     __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10887   %}
10888 
10889   ins_pipe(idiv_reg_reg);
10890 %}
10891 
10892 // Unsigned Long Remainder
10893 
10894 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10895   match(Set dst (UModL src1 src2));
10896 
10897   ins_cost(INSN_COST * 38);
10898   format %{ "udiv   rscratch1, $src1, $src2\n"
10899             "msub   $dst, rscratch1, $src2, $src1" %}
10900 
10901   ins_encode %{
10902     __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10903     __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10904   %}
10905 
10906   ins_pipe(ldiv_reg_reg);
10907 %}
10908 
10909 // Integer Shifts
10910 
10911 // Shift Left Register
10912 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10913   match(Set dst (LShiftI src1 src2));
10914 
10915   ins_cost(INSN_COST * 2);
10916   format %{ "lslvw  $dst, $src1, $src2" %}
10917 
10918   ins_encode %{
10919     __ lslvw(as_Register($dst$$reg),
10920              as_Register($src1$$reg),
10921              as_Register($src2$$reg));
10922   %}
10923 
10924   ins_pipe(ialu_reg_reg_vshift);
10925 %}
10926 
10927 // Shift Left Immediate
10928 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10929   match(Set dst (LShiftI src1 src2));
10930 
10931   ins_cost(INSN_COST);
10932   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10933 
10934   ins_encode %{
10935     __ lslw(as_Register($dst$$reg),
10936             as_Register($src1$$reg),
10937             $src2$$constant & 0x1f);
10938   %}
10939 
10940   ins_pipe(ialu_reg_shift);
10941 %}
10942 
10943 // Shift Right Logical Register
10944 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10945   match(Set dst (URShiftI src1 src2));
10946 
10947   ins_cost(INSN_COST * 2);
10948   format %{ "lsrvw  $dst, $src1, $src2" %}
10949 
10950   ins_encode %{
10951     __ lsrvw(as_Register($dst$$reg),
10952              as_Register($src1$$reg),
10953              as_Register($src2$$reg));
10954   %}
10955 
10956   ins_pipe(ialu_reg_reg_vshift);
10957 %}
10958 
10959 // Shift Right Logical Immediate
10960 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10961   match(Set dst (URShiftI src1 src2));
10962 
10963   ins_cost(INSN_COST);
10964   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10965 
10966   ins_encode %{
10967     __ lsrw(as_Register($dst$$reg),
10968             as_Register($src1$$reg),
10969             $src2$$constant & 0x1f);
10970   %}
10971 
10972   ins_pipe(ialu_reg_shift);
10973 %}
10974 
10975 // Shift Right Arithmetic Register
10976 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10977   match(Set dst (RShiftI src1 src2));
10978 
10979   ins_cost(INSN_COST * 2);
10980   format %{ "asrvw  $dst, $src1, $src2" %}
10981 
10982   ins_encode %{
10983     __ asrvw(as_Register($dst$$reg),
10984              as_Register($src1$$reg),
10985              as_Register($src2$$reg));
10986   %}
10987 
10988   ins_pipe(ialu_reg_reg_vshift);
10989 %}
10990 
10991 // Shift Right Arithmetic Immediate
10992 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10993   match(Set dst (RShiftI src1 src2));
10994 
10995   ins_cost(INSN_COST);
10996   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10997 
10998   ins_encode %{
10999     __ asrw(as_Register($dst$$reg),
11000             as_Register($src1$$reg),
11001             $src2$$constant & 0x1f);
11002   %}
11003 
11004   ins_pipe(ialu_reg_shift);
11005 %}
11006 
11007 // Combined Int Mask and Right Shift (using UBFM)
11008 // TODO
11009 
11010 // Long Shifts
11011 
11012 // Shift Left Register
11013 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11014   match(Set dst (LShiftL src1 src2));
11015 
11016   ins_cost(INSN_COST * 2);
11017   format %{ "lslv  $dst, $src1, $src2" %}
11018 
11019   ins_encode %{
11020     __ lslv(as_Register($dst$$reg),
11021             as_Register($src1$$reg),
11022             as_Register($src2$$reg));
11023   %}
11024 
11025   ins_pipe(ialu_reg_reg_vshift);
11026 %}
11027 
11028 // Shift Left Immediate
11029 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11030   match(Set dst (LShiftL src1 src2));
11031 
11032   ins_cost(INSN_COST);
11033   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11034 
11035   ins_encode %{
11036     __ lsl(as_Register($dst$$reg),
11037             as_Register($src1$$reg),
11038             $src2$$constant & 0x3f);
11039   %}
11040 
11041   ins_pipe(ialu_reg_shift);
11042 %}
11043 
11044 // Shift Right Logical Register
11045 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11046   match(Set dst (URShiftL src1 src2));
11047 
11048   ins_cost(INSN_COST * 2);
11049   format %{ "lsrv  $dst, $src1, $src2" %}
11050 
11051   ins_encode %{
11052     __ lsrv(as_Register($dst$$reg),
11053             as_Register($src1$$reg),
11054             as_Register($src2$$reg));
11055   %}
11056 
11057   ins_pipe(ialu_reg_reg_vshift);
11058 %}
11059 
11060 // Shift Right Logical Immediate
11061 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11062   match(Set dst (URShiftL src1 src2));
11063 
11064   ins_cost(INSN_COST);
11065   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11066 
11067   ins_encode %{
11068     __ lsr(as_Register($dst$$reg),
11069            as_Register($src1$$reg),
11070            $src2$$constant & 0x3f);
11071   %}
11072 
11073   ins_pipe(ialu_reg_shift);
11074 %}
11075 
11076 // A special-case pattern for card table stores.
11077 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11078   match(Set dst (URShiftL (CastP2X src1) src2));
11079 
11080   ins_cost(INSN_COST);
11081   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11082 
11083   ins_encode %{
11084     __ lsr(as_Register($dst$$reg),
11085            as_Register($src1$$reg),
11086            $src2$$constant & 0x3f);
11087   %}
11088 
11089   ins_pipe(ialu_reg_shift);
11090 %}
11091 
11092 // Shift Right Arithmetic Register
11093 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11094   match(Set dst (RShiftL src1 src2));
11095 
11096   ins_cost(INSN_COST * 2);
11097   format %{ "asrv  $dst, $src1, $src2" %}
11098 
11099   ins_encode %{
11100     __ asrv(as_Register($dst$$reg),
11101             as_Register($src1$$reg),
11102             as_Register($src2$$reg));
11103   %}
11104 
11105   ins_pipe(ialu_reg_reg_vshift);
11106 %}
11107 
11108 // Shift Right Arithmetic Immediate
11109 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11110   match(Set dst (RShiftL src1 src2));
11111 
11112   ins_cost(INSN_COST);
11113   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11114 
11115   ins_encode %{
11116     __ asr(as_Register($dst$$reg),
11117            as_Register($src1$$reg),
11118            $src2$$constant & 0x3f);
11119   %}
11120 
11121   ins_pipe(ialu_reg_shift);
11122 %}
11123 
11124 // BEGIN This section of the file is automatically generated. Do not edit --------------
11125 // This section is generated from aarch64_ad.m4
11126 
11127 // This pattern is automatically generated from aarch64_ad.m4
11128 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11129 instruct regL_not_reg(iRegLNoSp dst,
11130                          iRegL src1, immL_M1 m1,
11131                          rFlagsReg cr) %{
11132   match(Set dst (XorL src1 m1));
11133   ins_cost(INSN_COST);
11134   format %{ "eon  $dst, $src1, zr" %}
11135 
11136   ins_encode %{
11137     __ eon(as_Register($dst$$reg),
11138               as_Register($src1$$reg),
11139               zr,
11140               Assembler::LSL, 0);
11141   %}
11142 
11143   ins_pipe(ialu_reg);
11144 %}
11145 
11146 // This pattern is automatically generated from aarch64_ad.m4
11147 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11148 instruct regI_not_reg(iRegINoSp dst,
11149                          iRegIorL2I src1, immI_M1 m1,
11150                          rFlagsReg cr) %{
11151   match(Set dst (XorI src1 m1));
11152   ins_cost(INSN_COST);
11153   format %{ "eonw  $dst, $src1, zr" %}
11154 
11155   ins_encode %{
11156     __ eonw(as_Register($dst$$reg),
11157               as_Register($src1$$reg),
11158               zr,
11159               Assembler::LSL, 0);
11160   %}
11161 
11162   ins_pipe(ialu_reg);
11163 %}
11164 
11165 // This pattern is automatically generated from aarch64_ad.m4
11166 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11167 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11168                               immI0 zero, iRegIorL2I src1, immI src2) %{
11169   match(Set dst (SubI zero (URShiftI src1 src2)));
11170 
11171   ins_cost(1.9 * INSN_COST);
11172   format %{ "negw  $dst, $src1, LSR $src2" %}
11173 
11174   ins_encode %{
11175     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11176             Assembler::LSR, $src2$$constant & 0x1f);
11177   %}
11178 
11179   ins_pipe(ialu_reg_shift);
11180 %}
11181 
11182 // This pattern is automatically generated from aarch64_ad.m4
11183 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11184 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11185                               immI0 zero, iRegIorL2I src1, immI src2) %{
11186   match(Set dst (SubI zero (RShiftI src1 src2)));
11187 
11188   ins_cost(1.9 * INSN_COST);
11189   format %{ "negw  $dst, $src1, ASR $src2" %}
11190 
11191   ins_encode %{
11192     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11193             Assembler::ASR, $src2$$constant & 0x1f);
11194   %}
11195 
11196   ins_pipe(ialu_reg_shift);
11197 %}
11198 
11199 // This pattern is automatically generated from aarch64_ad.m4
11200 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11201 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11202                               immI0 zero, iRegIorL2I src1, immI src2) %{
11203   match(Set dst (SubI zero (LShiftI src1 src2)));
11204 
11205   ins_cost(1.9 * INSN_COST);
11206   format %{ "negw  $dst, $src1, LSL $src2" %}
11207 
11208   ins_encode %{
11209     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11210             Assembler::LSL, $src2$$constant & 0x1f);
11211   %}
11212 
11213   ins_pipe(ialu_reg_shift);
11214 %}
11215 
11216 // This pattern is automatically generated from aarch64_ad.m4
11217 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11218 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11219                               immL0 zero, iRegL src1, immI src2) %{
11220   match(Set dst (SubL zero (URShiftL src1 src2)));
11221 
11222   ins_cost(1.9 * INSN_COST);
11223   format %{ "neg  $dst, $src1, LSR $src2" %}
11224 
11225   ins_encode %{
11226     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11227             Assembler::LSR, $src2$$constant & 0x3f);
11228   %}
11229 
11230   ins_pipe(ialu_reg_shift);
11231 %}
11232 
11233 // This pattern is automatically generated from aarch64_ad.m4
11234 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11235 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11236                               immL0 zero, iRegL src1, immI src2) %{
11237   match(Set dst (SubL zero (RShiftL src1 src2)));
11238 
11239   ins_cost(1.9 * INSN_COST);
11240   format %{ "neg  $dst, $src1, ASR $src2" %}
11241 
11242   ins_encode %{
11243     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11244             Assembler::ASR, $src2$$constant & 0x3f);
11245   %}
11246 
11247   ins_pipe(ialu_reg_shift);
11248 %}
11249 
11250 // This pattern is automatically generated from aarch64_ad.m4
11251 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11252 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11253                               immL0 zero, iRegL src1, immI src2) %{
11254   match(Set dst (SubL zero (LShiftL src1 src2)));
11255 
11256   ins_cost(1.9 * INSN_COST);
11257   format %{ "neg  $dst, $src1, LSL $src2" %}
11258 
11259   ins_encode %{
11260     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11261             Assembler::LSL, $src2$$constant & 0x3f);
11262   %}
11263 
11264   ins_pipe(ialu_reg_shift);
11265 %}
11266 
11267 // This pattern is automatically generated from aarch64_ad.m4
11268 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11269 instruct AndI_reg_not_reg(iRegINoSp dst,
11270                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11271   match(Set dst (AndI src1 (XorI src2 m1)));
11272   ins_cost(INSN_COST);
11273   format %{ "bicw  $dst, $src1, $src2" %}
11274 
11275   ins_encode %{
11276     __ bicw(as_Register($dst$$reg),
11277               as_Register($src1$$reg),
11278               as_Register($src2$$reg),
11279               Assembler::LSL, 0);
11280   %}
11281 
11282   ins_pipe(ialu_reg_reg);
11283 %}
11284 
11285 // This pattern is automatically generated from aarch64_ad.m4
11286 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11287 instruct AndL_reg_not_reg(iRegLNoSp dst,
11288                          iRegL src1, iRegL src2, immL_M1 m1) %{
11289   match(Set dst (AndL src1 (XorL src2 m1)));
11290   ins_cost(INSN_COST);
11291   format %{ "bic  $dst, $src1, $src2" %}
11292 
11293   ins_encode %{
11294     __ bic(as_Register($dst$$reg),
11295               as_Register($src1$$reg),
11296               as_Register($src2$$reg),
11297               Assembler::LSL, 0);
11298   %}
11299 
11300   ins_pipe(ialu_reg_reg);
11301 %}
11302 
11303 // This pattern is automatically generated from aarch64_ad.m4
11304 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11305 instruct OrI_reg_not_reg(iRegINoSp dst,
11306                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11307   match(Set dst (OrI src1 (XorI src2 m1)));
11308   ins_cost(INSN_COST);
11309   format %{ "ornw  $dst, $src1, $src2" %}
11310 
11311   ins_encode %{
11312     __ ornw(as_Register($dst$$reg),
11313               as_Register($src1$$reg),
11314               as_Register($src2$$reg),
11315               Assembler::LSL, 0);
11316   %}
11317 
11318   ins_pipe(ialu_reg_reg);
11319 %}
11320 
11321 // This pattern is automatically generated from aarch64_ad.m4
11322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11323 instruct OrL_reg_not_reg(iRegLNoSp dst,
11324                          iRegL src1, iRegL src2, immL_M1 m1) %{
11325   match(Set dst (OrL src1 (XorL src2 m1)));
11326   ins_cost(INSN_COST);
11327   format %{ "orn  $dst, $src1, $src2" %}
11328 
11329   ins_encode %{
11330     __ orn(as_Register($dst$$reg),
11331               as_Register($src1$$reg),
11332               as_Register($src2$$reg),
11333               Assembler::LSL, 0);
11334   %}
11335 
11336   ins_pipe(ialu_reg_reg);
11337 %}
11338 
11339 // This pattern is automatically generated from aarch64_ad.m4
11340 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11341 instruct XorI_reg_not_reg(iRegINoSp dst,
11342                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11343   match(Set dst (XorI m1 (XorI src2 src1)));
11344   ins_cost(INSN_COST);
11345   format %{ "eonw  $dst, $src1, $src2" %}
11346 
11347   ins_encode %{
11348     __ eonw(as_Register($dst$$reg),
11349               as_Register($src1$$reg),
11350               as_Register($src2$$reg),
11351               Assembler::LSL, 0);
11352   %}
11353 
11354   ins_pipe(ialu_reg_reg);
11355 %}
11356 
11357 // This pattern is automatically generated from aarch64_ad.m4
11358 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11359 instruct XorL_reg_not_reg(iRegLNoSp dst,
11360                          iRegL src1, iRegL src2, immL_M1 m1) %{
11361   match(Set dst (XorL m1 (XorL src2 src1)));
11362   ins_cost(INSN_COST);
11363   format %{ "eon  $dst, $src1, $src2" %}
11364 
11365   ins_encode %{
11366     __ eon(as_Register($dst$$reg),
11367               as_Register($src1$$reg),
11368               as_Register($src2$$reg),
11369               Assembler::LSL, 0);
11370   %}
11371 
11372   ins_pipe(ialu_reg_reg);
11373 %}
11374 
11375 // This pattern is automatically generated from aarch64_ad.m4
11376 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11377 // val & (-1 ^ (val >>> shift)) ==> bicw
11378 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11379                          iRegIorL2I src1, iRegIorL2I src2,
11380                          immI src3, immI_M1 src4) %{
11381   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11382   ins_cost(1.9 * INSN_COST);
11383   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11384 
11385   ins_encode %{
11386     __ bicw(as_Register($dst$$reg),
11387               as_Register($src1$$reg),
11388               as_Register($src2$$reg),
11389               Assembler::LSR,
11390               $src3$$constant & 0x1f);
11391   %}
11392 
11393   ins_pipe(ialu_reg_reg_shift);
11394 %}
11395 
11396 // This pattern is automatically generated from aarch64_ad.m4
11397 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11398 // val & (-1 ^ (val >>> shift)) ==> bic
11399 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11400                          iRegL src1, iRegL src2,
11401                          immI src3, immL_M1 src4) %{
11402   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11403   ins_cost(1.9 * INSN_COST);
11404   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11405 
11406   ins_encode %{
11407     __ bic(as_Register($dst$$reg),
11408               as_Register($src1$$reg),
11409               as_Register($src2$$reg),
11410               Assembler::LSR,
11411               $src3$$constant & 0x3f);
11412   %}
11413 
11414   ins_pipe(ialu_reg_reg_shift);
11415 %}
11416 
11417 // This pattern is automatically generated from aarch64_ad.m4
11418 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11419 // val & (-1 ^ (val >> shift)) ==> bicw
11420 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11421                          iRegIorL2I src1, iRegIorL2I src2,
11422                          immI src3, immI_M1 src4) %{
11423   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11424   ins_cost(1.9 * INSN_COST);
11425   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11426 
11427   ins_encode %{
11428     __ bicw(as_Register($dst$$reg),
11429               as_Register($src1$$reg),
11430               as_Register($src2$$reg),
11431               Assembler::ASR,
11432               $src3$$constant & 0x1f);
11433   %}
11434 
11435   ins_pipe(ialu_reg_reg_shift);
11436 %}
11437 
11438 // This pattern is automatically generated from aarch64_ad.m4
11439 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11440 // val & (-1 ^ (val >> shift)) ==> bic
11441 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11442                          iRegL src1, iRegL src2,
11443                          immI src3, immL_M1 src4) %{
11444   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11445   ins_cost(1.9 * INSN_COST);
11446   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11447 
11448   ins_encode %{
11449     __ bic(as_Register($dst$$reg),
11450               as_Register($src1$$reg),
11451               as_Register($src2$$reg),
11452               Assembler::ASR,
11453               $src3$$constant & 0x3f);
11454   %}
11455 
11456   ins_pipe(ialu_reg_reg_shift);
11457 %}
11458 
11459 // This pattern is automatically generated from aarch64_ad.m4
11460 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11461 // val & (-1 ^ (val ror shift)) ==> bicw
11462 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11463                          iRegIorL2I src1, iRegIorL2I src2,
11464                          immI src3, immI_M1 src4) %{
11465   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11466   ins_cost(1.9 * INSN_COST);
11467   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11468 
11469   ins_encode %{
11470     __ bicw(as_Register($dst$$reg),
11471               as_Register($src1$$reg),
11472               as_Register($src2$$reg),
11473               Assembler::ROR,
11474               $src3$$constant & 0x1f);
11475   %}
11476 
11477   ins_pipe(ialu_reg_reg_shift);
11478 %}
11479 
11480 // This pattern is automatically generated from aarch64_ad.m4
11481 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11482 // val & (-1 ^ (val ror shift)) ==> bic
11483 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11484                          iRegL src1, iRegL src2,
11485                          immI src3, immL_M1 src4) %{
11486   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11487   ins_cost(1.9 * INSN_COST);
11488   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11489 
11490   ins_encode %{
11491     __ bic(as_Register($dst$$reg),
11492               as_Register($src1$$reg),
11493               as_Register($src2$$reg),
11494               Assembler::ROR,
11495               $src3$$constant & 0x3f);
11496   %}
11497 
11498   ins_pipe(ialu_reg_reg_shift);
11499 %}
11500 
11501 // This pattern is automatically generated from aarch64_ad.m4
11502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11503 // val & (-1 ^ (val << shift)) ==> bicw
11504 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11505                          iRegIorL2I src1, iRegIorL2I src2,
11506                          immI src3, immI_M1 src4) %{
11507   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11508   ins_cost(1.9 * INSN_COST);
11509   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11510 
11511   ins_encode %{
11512     __ bicw(as_Register($dst$$reg),
11513               as_Register($src1$$reg),
11514               as_Register($src2$$reg),
11515               Assembler::LSL,
11516               $src3$$constant & 0x1f);
11517   %}
11518 
11519   ins_pipe(ialu_reg_reg_shift);
11520 %}
11521 
11522 // This pattern is automatically generated from aarch64_ad.m4
11523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11524 // val & (-1 ^ (val << shift)) ==> bic
11525 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11526                          iRegL src1, iRegL src2,
11527                          immI src3, immL_M1 src4) %{
11528   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11529   ins_cost(1.9 * INSN_COST);
11530   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11531 
11532   ins_encode %{
11533     __ bic(as_Register($dst$$reg),
11534               as_Register($src1$$reg),
11535               as_Register($src2$$reg),
11536               Assembler::LSL,
11537               $src3$$constant & 0x3f);
11538   %}
11539 
11540   ins_pipe(ialu_reg_reg_shift);
11541 %}
11542 
11543 // This pattern is automatically generated from aarch64_ad.m4
11544 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11545 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11546 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11547                          iRegIorL2I src1, iRegIorL2I src2,
11548                          immI src3, immI_M1 src4) %{
11549   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11550   ins_cost(1.9 * INSN_COST);
11551   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11552 
11553   ins_encode %{
11554     __ eonw(as_Register($dst$$reg),
11555               as_Register($src1$$reg),
11556               as_Register($src2$$reg),
11557               Assembler::LSR,
11558               $src3$$constant & 0x1f);
11559   %}
11560 
11561   ins_pipe(ialu_reg_reg_shift);
11562 %}
11563 
11564 // This pattern is automatically generated from aarch64_ad.m4
11565 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11566 // val ^ (-1 ^ (val >>> shift)) ==> eon
11567 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11568                          iRegL src1, iRegL src2,
11569                          immI src3, immL_M1 src4) %{
11570   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11571   ins_cost(1.9 * INSN_COST);
11572   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11573 
11574   ins_encode %{
11575     __ eon(as_Register($dst$$reg),
11576               as_Register($src1$$reg),
11577               as_Register($src2$$reg),
11578               Assembler::LSR,
11579               $src3$$constant & 0x3f);
11580   %}
11581 
11582   ins_pipe(ialu_reg_reg_shift);
11583 %}
11584 
11585 // This pattern is automatically generated from aarch64_ad.m4
11586 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11587 // val ^ (-1 ^ (val >> shift)) ==> eonw
11588 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11589                          iRegIorL2I src1, iRegIorL2I src2,
11590                          immI src3, immI_M1 src4) %{
11591   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11592   ins_cost(1.9 * INSN_COST);
11593   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11594 
11595   ins_encode %{
11596     __ eonw(as_Register($dst$$reg),
11597               as_Register($src1$$reg),
11598               as_Register($src2$$reg),
11599               Assembler::ASR,
11600               $src3$$constant & 0x1f);
11601   %}
11602 
11603   ins_pipe(ialu_reg_reg_shift);
11604 %}
11605 
11606 // This pattern is automatically generated from aarch64_ad.m4
11607 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11608 // val ^ (-1 ^ (val >> shift)) ==> eon
11609 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11610                          iRegL src1, iRegL src2,
11611                          immI src3, immL_M1 src4) %{
11612   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11613   ins_cost(1.9 * INSN_COST);
11614   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11615 
11616   ins_encode %{
11617     __ eon(as_Register($dst$$reg),
11618               as_Register($src1$$reg),
11619               as_Register($src2$$reg),
11620               Assembler::ASR,
11621               $src3$$constant & 0x3f);
11622   %}
11623 
11624   ins_pipe(ialu_reg_reg_shift);
11625 %}
11626 
11627 // This pattern is automatically generated from aarch64_ad.m4
11628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11629 // val ^ (-1 ^ (val ror shift)) ==> eonw
11630 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11631                          iRegIorL2I src1, iRegIorL2I src2,
11632                          immI src3, immI_M1 src4) %{
11633   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11634   ins_cost(1.9 * INSN_COST);
11635   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11636 
11637   ins_encode %{
11638     __ eonw(as_Register($dst$$reg),
11639               as_Register($src1$$reg),
11640               as_Register($src2$$reg),
11641               Assembler::ROR,
11642               $src3$$constant & 0x1f);
11643   %}
11644 
11645   ins_pipe(ialu_reg_reg_shift);
11646 %}
11647 
11648 // This pattern is automatically generated from aarch64_ad.m4
11649 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11650 // val ^ (-1 ^ (val ror shift)) ==> eon
11651 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11652                          iRegL src1, iRegL src2,
11653                          immI src3, immL_M1 src4) %{
11654   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11655   ins_cost(1.9 * INSN_COST);
11656   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11657 
11658   ins_encode %{
11659     __ eon(as_Register($dst$$reg),
11660               as_Register($src1$$reg),
11661               as_Register($src2$$reg),
11662               Assembler::ROR,
11663               $src3$$constant & 0x3f);
11664   %}
11665 
11666   ins_pipe(ialu_reg_reg_shift);
11667 %}
11668 
11669 // This pattern is automatically generated from aarch64_ad.m4
11670 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11671 // val ^ (-1 ^ (val << shift)) ==> eonw
11672 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11673                          iRegIorL2I src1, iRegIorL2I src2,
11674                          immI src3, immI_M1 src4) %{
11675   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11676   ins_cost(1.9 * INSN_COST);
11677   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11678 
11679   ins_encode %{
11680     __ eonw(as_Register($dst$$reg),
11681               as_Register($src1$$reg),
11682               as_Register($src2$$reg),
11683               Assembler::LSL,
11684               $src3$$constant & 0x1f);
11685   %}
11686 
11687   ins_pipe(ialu_reg_reg_shift);
11688 %}
11689 
11690 // This pattern is automatically generated from aarch64_ad.m4
11691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11692 // val ^ (-1 ^ (val << shift)) ==> eon
11693 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11694                          iRegL src1, iRegL src2,
11695                          immI src3, immL_M1 src4) %{
11696   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11697   ins_cost(1.9 * INSN_COST);
11698   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11699 
11700   ins_encode %{
11701     __ eon(as_Register($dst$$reg),
11702               as_Register($src1$$reg),
11703               as_Register($src2$$reg),
11704               Assembler::LSL,
11705               $src3$$constant & 0x3f);
11706   %}
11707 
11708   ins_pipe(ialu_reg_reg_shift);
11709 %}
11710 
11711 // This pattern is automatically generated from aarch64_ad.m4
11712 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11713 // val | (-1 ^ (val >>> shift)) ==> ornw
11714 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11715                          iRegIorL2I src1, iRegIorL2I src2,
11716                          immI src3, immI_M1 src4) %{
11717   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11718   ins_cost(1.9 * INSN_COST);
11719   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11720 
11721   ins_encode %{
11722     __ ornw(as_Register($dst$$reg),
11723               as_Register($src1$$reg),
11724               as_Register($src2$$reg),
11725               Assembler::LSR,
11726               $src3$$constant & 0x1f);
11727   %}
11728 
11729   ins_pipe(ialu_reg_reg_shift);
11730 %}
11731 
11732 // This pattern is automatically generated from aarch64_ad.m4
11733 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11734 // val | (-1 ^ (val >>> shift)) ==> orn
11735 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11736                          iRegL src1, iRegL src2,
11737                          immI src3, immL_M1 src4) %{
11738   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11739   ins_cost(1.9 * INSN_COST);
11740   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11741 
11742   ins_encode %{
11743     __ orn(as_Register($dst$$reg),
11744               as_Register($src1$$reg),
11745               as_Register($src2$$reg),
11746               Assembler::LSR,
11747               $src3$$constant & 0x3f);
11748   %}
11749 
11750   ins_pipe(ialu_reg_reg_shift);
11751 %}
11752 
11753 // This pattern is automatically generated from aarch64_ad.m4
11754 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11755 // val | (-1 ^ (val >> shift)) ==> ornw
11756 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11757                          iRegIorL2I src1, iRegIorL2I src2,
11758                          immI src3, immI_M1 src4) %{
11759   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11760   ins_cost(1.9 * INSN_COST);
11761   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11762 
11763   ins_encode %{
11764     __ ornw(as_Register($dst$$reg),
11765               as_Register($src1$$reg),
11766               as_Register($src2$$reg),
11767               Assembler::ASR,
11768               $src3$$constant & 0x1f);
11769   %}
11770 
11771   ins_pipe(ialu_reg_reg_shift);
11772 %}
11773 
11774 // This pattern is automatically generated from aarch64_ad.m4
11775 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11776 // val | (-1 ^ (val >> shift)) ==> orn
11777 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11778                          iRegL src1, iRegL src2,
11779                          immI src3, immL_M1 src4) %{
11780   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11781   ins_cost(1.9 * INSN_COST);
11782   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11783 
11784   ins_encode %{
11785     __ orn(as_Register($dst$$reg),
11786               as_Register($src1$$reg),
11787               as_Register($src2$$reg),
11788               Assembler::ASR,
11789               $src3$$constant & 0x3f);
11790   %}
11791 
11792   ins_pipe(ialu_reg_reg_shift);
11793 %}
11794 
11795 // This pattern is automatically generated from aarch64_ad.m4
11796 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11797 // val | (-1 ^ (val ror shift)) ==> ornw
11798 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11799                          iRegIorL2I src1, iRegIorL2I src2,
11800                          immI src3, immI_M1 src4) %{
11801   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11802   ins_cost(1.9 * INSN_COST);
11803   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11804 
11805   ins_encode %{
11806     __ ornw(as_Register($dst$$reg),
11807               as_Register($src1$$reg),
11808               as_Register($src2$$reg),
11809               Assembler::ROR,
11810               $src3$$constant & 0x1f);
11811   %}
11812 
11813   ins_pipe(ialu_reg_reg_shift);
11814 %}
11815 
11816 // This pattern is automatically generated from aarch64_ad.m4
11817 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11818 // val | (-1 ^ (val ror shift)) ==> orn
11819 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11820                          iRegL src1, iRegL src2,
11821                          immI src3, immL_M1 src4) %{
11822   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11823   ins_cost(1.9 * INSN_COST);
11824   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11825 
11826   ins_encode %{
11827     __ orn(as_Register($dst$$reg),
11828               as_Register($src1$$reg),
11829               as_Register($src2$$reg),
11830               Assembler::ROR,
11831               $src3$$constant & 0x3f);
11832   %}
11833 
11834   ins_pipe(ialu_reg_reg_shift);
11835 %}
11836 
11837 // This pattern is automatically generated from aarch64_ad.m4
11838 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11839 // val | (-1 ^ (val << shift)) ==> ornw
11840 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11841                          iRegIorL2I src1, iRegIorL2I src2,
11842                          immI src3, immI_M1 src4) %{
11843   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11844   ins_cost(1.9 * INSN_COST);
11845   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11846 
11847   ins_encode %{
11848     __ ornw(as_Register($dst$$reg),
11849               as_Register($src1$$reg),
11850               as_Register($src2$$reg),
11851               Assembler::LSL,
11852               $src3$$constant & 0x1f);
11853   %}
11854 
11855   ins_pipe(ialu_reg_reg_shift);
11856 %}
11857 
11858 // This pattern is automatically generated from aarch64_ad.m4
11859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11860 // val | (-1 ^ (val << shift)) ==> orn
11861 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11862                          iRegL src1, iRegL src2,
11863                          immI src3, immL_M1 src4) %{
11864   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11865   ins_cost(1.9 * INSN_COST);
11866   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11867 
11868   ins_encode %{
11869     __ orn(as_Register($dst$$reg),
11870               as_Register($src1$$reg),
11871               as_Register($src2$$reg),
11872               Assembler::LSL,
11873               $src3$$constant & 0x3f);
11874   %}
11875 
11876   ins_pipe(ialu_reg_reg_shift);
11877 %}
11878 
11879 // This pattern is automatically generated from aarch64_ad.m4
11880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11881 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11882                          iRegIorL2I src1, iRegIorL2I src2,
11883                          immI src3) %{
11884   match(Set dst (AndI src1 (URShiftI src2 src3)));
11885 
11886   ins_cost(1.9 * INSN_COST);
11887   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
11888 
11889   ins_encode %{
11890     __ andw(as_Register($dst$$reg),
11891               as_Register($src1$$reg),
11892               as_Register($src2$$reg),
11893               Assembler::LSR,
11894               $src3$$constant & 0x1f);
11895   %}
11896 
11897   ins_pipe(ialu_reg_reg_shift);
11898 %}
11899 
11900 // This pattern is automatically generated from aarch64_ad.m4
11901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11902 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11903                          iRegL src1, iRegL src2,
11904                          immI src3) %{
11905   match(Set dst (AndL src1 (URShiftL src2 src3)));
11906 
11907   ins_cost(1.9 * INSN_COST);
11908   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
11909 
11910   ins_encode %{
11911     __ andr(as_Register($dst$$reg),
11912               as_Register($src1$$reg),
11913               as_Register($src2$$reg),
11914               Assembler::LSR,
11915               $src3$$constant & 0x3f);
11916   %}
11917 
11918   ins_pipe(ialu_reg_reg_shift);
11919 %}
11920 
11921 // This pattern is automatically generated from aarch64_ad.m4
11922 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11923 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11924                          iRegIorL2I src1, iRegIorL2I src2,
11925                          immI src3) %{
11926   match(Set dst (AndI src1 (RShiftI src2 src3)));
11927 
11928   ins_cost(1.9 * INSN_COST);
11929   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
11930 
11931   ins_encode %{
11932     __ andw(as_Register($dst$$reg),
11933               as_Register($src1$$reg),
11934               as_Register($src2$$reg),
11935               Assembler::ASR,
11936               $src3$$constant & 0x1f);
11937   %}
11938 
11939   ins_pipe(ialu_reg_reg_shift);
11940 %}
11941 
11942 // This pattern is automatically generated from aarch64_ad.m4
11943 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11944 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11945                          iRegL src1, iRegL src2,
11946                          immI src3) %{
11947   match(Set dst (AndL src1 (RShiftL src2 src3)));
11948 
11949   ins_cost(1.9 * INSN_COST);
11950   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
11951 
11952   ins_encode %{
11953     __ andr(as_Register($dst$$reg),
11954               as_Register($src1$$reg),
11955               as_Register($src2$$reg),
11956               Assembler::ASR,
11957               $src3$$constant & 0x3f);
11958   %}
11959 
11960   ins_pipe(ialu_reg_reg_shift);
11961 %}
11962 
11963 // This pattern is automatically generated from aarch64_ad.m4
11964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11965 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11966                          iRegIorL2I src1, iRegIorL2I src2,
11967                          immI src3) %{
11968   match(Set dst (AndI src1 (LShiftI src2 src3)));
11969 
11970   ins_cost(1.9 * INSN_COST);
11971   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
11972 
11973   ins_encode %{
11974     __ andw(as_Register($dst$$reg),
11975               as_Register($src1$$reg),
11976               as_Register($src2$$reg),
11977               Assembler::LSL,
11978               $src3$$constant & 0x1f);
11979   %}
11980 
11981   ins_pipe(ialu_reg_reg_shift);
11982 %}
11983 
11984 // This pattern is automatically generated from aarch64_ad.m4
11985 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11986 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11987                          iRegL src1, iRegL src2,
11988                          immI src3) %{
11989   match(Set dst (AndL src1 (LShiftL src2 src3)));
11990 
11991   ins_cost(1.9 * INSN_COST);
11992   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
11993 
11994   ins_encode %{
11995     __ andr(as_Register($dst$$reg),
11996               as_Register($src1$$reg),
11997               as_Register($src2$$reg),
11998               Assembler::LSL,
11999               $src3$$constant & 0x3f);
12000   %}
12001 
12002   ins_pipe(ialu_reg_reg_shift);
12003 %}
12004 
12005 // This pattern is automatically generated from aarch64_ad.m4
12006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12007 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12008                          iRegIorL2I src1, iRegIorL2I src2,
12009                          immI src3) %{
12010   match(Set dst (AndI src1 (RotateRight src2 src3)));
12011 
12012   ins_cost(1.9 * INSN_COST);
12013   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12014 
12015   ins_encode %{
12016     __ andw(as_Register($dst$$reg),
12017               as_Register($src1$$reg),
12018               as_Register($src2$$reg),
12019               Assembler::ROR,
12020               $src3$$constant & 0x1f);
12021   %}
12022 
12023   ins_pipe(ialu_reg_reg_shift);
12024 %}
12025 
12026 // This pattern is automatically generated from aarch64_ad.m4
12027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12028 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12029                          iRegL src1, iRegL src2,
12030                          immI src3) %{
12031   match(Set dst (AndL src1 (RotateRight src2 src3)));
12032 
12033   ins_cost(1.9 * INSN_COST);
12034   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12035 
12036   ins_encode %{
12037     __ andr(as_Register($dst$$reg),
12038               as_Register($src1$$reg),
12039               as_Register($src2$$reg),
12040               Assembler::ROR,
12041               $src3$$constant & 0x3f);
12042   %}
12043 
12044   ins_pipe(ialu_reg_reg_shift);
12045 %}
12046 
12047 // This pattern is automatically generated from aarch64_ad.m4
12048 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12049 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12050                          iRegIorL2I src1, iRegIorL2I src2,
12051                          immI src3) %{
12052   match(Set dst (XorI src1 (URShiftI src2 src3)));
12053 
12054   ins_cost(1.9 * INSN_COST);
12055   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12056 
12057   ins_encode %{
12058     __ eorw(as_Register($dst$$reg),
12059               as_Register($src1$$reg),
12060               as_Register($src2$$reg),
12061               Assembler::LSR,
12062               $src3$$constant & 0x1f);
12063   %}
12064 
12065   ins_pipe(ialu_reg_reg_shift);
12066 %}
12067 
12068 // This pattern is automatically generated from aarch64_ad.m4
12069 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12070 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12071                          iRegL src1, iRegL src2,
12072                          immI src3) %{
12073   match(Set dst (XorL src1 (URShiftL src2 src3)));
12074 
12075   ins_cost(1.9 * INSN_COST);
12076   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12077 
12078   ins_encode %{
12079     __ eor(as_Register($dst$$reg),
12080               as_Register($src1$$reg),
12081               as_Register($src2$$reg),
12082               Assembler::LSR,
12083               $src3$$constant & 0x3f);
12084   %}
12085 
12086   ins_pipe(ialu_reg_reg_shift);
12087 %}
12088 
12089 // This pattern is automatically generated from aarch64_ad.m4
12090 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12091 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12092                          iRegIorL2I src1, iRegIorL2I src2,
12093                          immI src3) %{
12094   match(Set dst (XorI src1 (RShiftI src2 src3)));
12095 
12096   ins_cost(1.9 * INSN_COST);
12097   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12098 
12099   ins_encode %{
12100     __ eorw(as_Register($dst$$reg),
12101               as_Register($src1$$reg),
12102               as_Register($src2$$reg),
12103               Assembler::ASR,
12104               $src3$$constant & 0x1f);
12105   %}
12106 
12107   ins_pipe(ialu_reg_reg_shift);
12108 %}
12109 
12110 // This pattern is automatically generated from aarch64_ad.m4
12111 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12112 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12113                          iRegL src1, iRegL src2,
12114                          immI src3) %{
12115   match(Set dst (XorL src1 (RShiftL src2 src3)));
12116 
12117   ins_cost(1.9 * INSN_COST);
12118   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12119 
12120   ins_encode %{
12121     __ eor(as_Register($dst$$reg),
12122               as_Register($src1$$reg),
12123               as_Register($src2$$reg),
12124               Assembler::ASR,
12125               $src3$$constant & 0x3f);
12126   %}
12127 
12128   ins_pipe(ialu_reg_reg_shift);
12129 %}
12130 
12131 // This pattern is automatically generated from aarch64_ad.m4
12132 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12133 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12134                          iRegIorL2I src1, iRegIorL2I src2,
12135                          immI src3) %{
12136   match(Set dst (XorI src1 (LShiftI src2 src3)));
12137 
12138   ins_cost(1.9 * INSN_COST);
12139   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12140 
12141   ins_encode %{
12142     __ eorw(as_Register($dst$$reg),
12143               as_Register($src1$$reg),
12144               as_Register($src2$$reg),
12145               Assembler::LSL,
12146               $src3$$constant & 0x1f);
12147   %}
12148 
12149   ins_pipe(ialu_reg_reg_shift);
12150 %}
12151 
12152 // This pattern is automatically generated from aarch64_ad.m4
12153 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12154 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12155                          iRegL src1, iRegL src2,
12156                          immI src3) %{
12157   match(Set dst (XorL src1 (LShiftL src2 src3)));
12158 
12159   ins_cost(1.9 * INSN_COST);
12160   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12161 
12162   ins_encode %{
12163     __ eor(as_Register($dst$$reg),
12164               as_Register($src1$$reg),
12165               as_Register($src2$$reg),
12166               Assembler::LSL,
12167               $src3$$constant & 0x3f);
12168   %}
12169 
12170   ins_pipe(ialu_reg_reg_shift);
12171 %}
12172 
12173 // This pattern is automatically generated from aarch64_ad.m4
12174 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12175 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12176                          iRegIorL2I src1, iRegIorL2I src2,
12177                          immI src3) %{
12178   match(Set dst (XorI src1 (RotateRight src2 src3)));
12179 
12180   ins_cost(1.9 * INSN_COST);
12181   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12182 
12183   ins_encode %{
12184     __ eorw(as_Register($dst$$reg),
12185               as_Register($src1$$reg),
12186               as_Register($src2$$reg),
12187               Assembler::ROR,
12188               $src3$$constant & 0x1f);
12189   %}
12190 
12191   ins_pipe(ialu_reg_reg_shift);
12192 %}
12193 
12194 // This pattern is automatically generated from aarch64_ad.m4
12195 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12196 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12197                          iRegL src1, iRegL src2,
12198                          immI src3) %{
12199   match(Set dst (XorL src1 (RotateRight src2 src3)));
12200 
12201   ins_cost(1.9 * INSN_COST);
12202   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12203 
12204   ins_encode %{
12205     __ eor(as_Register($dst$$reg),
12206               as_Register($src1$$reg),
12207               as_Register($src2$$reg),
12208               Assembler::ROR,
12209               $src3$$constant & 0x3f);
12210   %}
12211 
12212   ins_pipe(ialu_reg_reg_shift);
12213 %}
12214 
12215 // This pattern is automatically generated from aarch64_ad.m4
12216 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12217 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12218                          iRegIorL2I src1, iRegIorL2I src2,
12219                          immI src3) %{
12220   match(Set dst (OrI src1 (URShiftI src2 src3)));
12221 
12222   ins_cost(1.9 * INSN_COST);
12223   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12224 
12225   ins_encode %{
12226     __ orrw(as_Register($dst$$reg),
12227               as_Register($src1$$reg),
12228               as_Register($src2$$reg),
12229               Assembler::LSR,
12230               $src3$$constant & 0x1f);
12231   %}
12232 
12233   ins_pipe(ialu_reg_reg_shift);
12234 %}
12235 
12236 // This pattern is automatically generated from aarch64_ad.m4
12237 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12238 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12239                          iRegL src1, iRegL src2,
12240                          immI src3) %{
12241   match(Set dst (OrL src1 (URShiftL src2 src3)));
12242 
12243   ins_cost(1.9 * INSN_COST);
12244   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12245 
12246   ins_encode %{
12247     __ orr(as_Register($dst$$reg),
12248               as_Register($src1$$reg),
12249               as_Register($src2$$reg),
12250               Assembler::LSR,
12251               $src3$$constant & 0x3f);
12252   %}
12253 
12254   ins_pipe(ialu_reg_reg_shift);
12255 %}
12256 
12257 // This pattern is automatically generated from aarch64_ad.m4
12258 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12259 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12260                          iRegIorL2I src1, iRegIorL2I src2,
12261                          immI src3) %{
12262   match(Set dst (OrI src1 (RShiftI src2 src3)));
12263 
12264   ins_cost(1.9 * INSN_COST);
12265   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12266 
12267   ins_encode %{
12268     __ orrw(as_Register($dst$$reg),
12269               as_Register($src1$$reg),
12270               as_Register($src2$$reg),
12271               Assembler::ASR,
12272               $src3$$constant & 0x1f);
12273   %}
12274 
12275   ins_pipe(ialu_reg_reg_shift);
12276 %}
12277 
12278 // This pattern is automatically generated from aarch64_ad.m4
12279 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12280 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12281                          iRegL src1, iRegL src2,
12282                          immI src3) %{
12283   match(Set dst (OrL src1 (RShiftL src2 src3)));
12284 
12285   ins_cost(1.9 * INSN_COST);
12286   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12287 
12288   ins_encode %{
12289     __ orr(as_Register($dst$$reg),
12290               as_Register($src1$$reg),
12291               as_Register($src2$$reg),
12292               Assembler::ASR,
12293               $src3$$constant & 0x3f);
12294   %}
12295 
12296   ins_pipe(ialu_reg_reg_shift);
12297 %}
12298 
12299 // This pattern is automatically generated from aarch64_ad.m4
12300 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12301 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12302                          iRegIorL2I src1, iRegIorL2I src2,
12303                          immI src3) %{
12304   match(Set dst (OrI src1 (LShiftI src2 src3)));
12305 
12306   ins_cost(1.9 * INSN_COST);
12307   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12308 
12309   ins_encode %{
12310     __ orrw(as_Register($dst$$reg),
12311               as_Register($src1$$reg),
12312               as_Register($src2$$reg),
12313               Assembler::LSL,
12314               $src3$$constant & 0x1f);
12315   %}
12316 
12317   ins_pipe(ialu_reg_reg_shift);
12318 %}
12319 
12320 // This pattern is automatically generated from aarch64_ad.m4
12321 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12322 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12323                          iRegL src1, iRegL src2,
12324                          immI src3) %{
12325   match(Set dst (OrL src1 (LShiftL src2 src3)));
12326 
12327   ins_cost(1.9 * INSN_COST);
12328   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12329 
12330   ins_encode %{
12331     __ orr(as_Register($dst$$reg),
12332               as_Register($src1$$reg),
12333               as_Register($src2$$reg),
12334               Assembler::LSL,
12335               $src3$$constant & 0x3f);
12336   %}
12337 
12338   ins_pipe(ialu_reg_reg_shift);
12339 %}
12340 
12341 // This pattern is automatically generated from aarch64_ad.m4
12342 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12343 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12344                          iRegIorL2I src1, iRegIorL2I src2,
12345                          immI src3) %{
12346   match(Set dst (OrI src1 (RotateRight src2 src3)));
12347 
12348   ins_cost(1.9 * INSN_COST);
12349   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12350 
12351   ins_encode %{
12352     __ orrw(as_Register($dst$$reg),
12353               as_Register($src1$$reg),
12354               as_Register($src2$$reg),
12355               Assembler::ROR,
12356               $src3$$constant & 0x1f);
12357   %}
12358 
12359   ins_pipe(ialu_reg_reg_shift);
12360 %}
12361 
12362 // This pattern is automatically generated from aarch64_ad.m4
12363 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12364 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12365                          iRegL src1, iRegL src2,
12366                          immI src3) %{
12367   match(Set dst (OrL src1 (RotateRight src2 src3)));
12368 
12369   ins_cost(1.9 * INSN_COST);
12370   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12371 
12372   ins_encode %{
12373     __ orr(as_Register($dst$$reg),
12374               as_Register($src1$$reg),
12375               as_Register($src2$$reg),
12376               Assembler::ROR,
12377               $src3$$constant & 0x3f);
12378   %}
12379 
12380   ins_pipe(ialu_reg_reg_shift);
12381 %}
12382 
12383 // This pattern is automatically generated from aarch64_ad.m4
12384 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12385 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12386                          iRegIorL2I src1, iRegIorL2I src2,
12387                          immI src3) %{
12388   match(Set dst (AddI src1 (URShiftI src2 src3)));
12389 
12390   ins_cost(1.9 * INSN_COST);
12391   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12392 
12393   ins_encode %{
12394     __ addw(as_Register($dst$$reg),
12395               as_Register($src1$$reg),
12396               as_Register($src2$$reg),
12397               Assembler::LSR,
12398               $src3$$constant & 0x1f);
12399   %}
12400 
12401   ins_pipe(ialu_reg_reg_shift);
12402 %}
12403 
12404 // This pattern is automatically generated from aarch64_ad.m4
12405 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12406 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12407                          iRegL src1, iRegL src2,
12408                          immI src3) %{
12409   match(Set dst (AddL src1 (URShiftL src2 src3)));
12410 
12411   ins_cost(1.9 * INSN_COST);
12412   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12413 
12414   ins_encode %{
12415     __ add(as_Register($dst$$reg),
12416               as_Register($src1$$reg),
12417               as_Register($src2$$reg),
12418               Assembler::LSR,
12419               $src3$$constant & 0x3f);
12420   %}
12421 
12422   ins_pipe(ialu_reg_reg_shift);
12423 %}
12424 
12425 // This pattern is automatically generated from aarch64_ad.m4
12426 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12427 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12428                          iRegIorL2I src1, iRegIorL2I src2,
12429                          immI src3) %{
12430   match(Set dst (AddI src1 (RShiftI src2 src3)));
12431 
12432   ins_cost(1.9 * INSN_COST);
12433   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12434 
12435   ins_encode %{
12436     __ addw(as_Register($dst$$reg),
12437               as_Register($src1$$reg),
12438               as_Register($src2$$reg),
12439               Assembler::ASR,
12440               $src3$$constant & 0x1f);
12441   %}
12442 
12443   ins_pipe(ialu_reg_reg_shift);
12444 %}
12445 
12446 // This pattern is automatically generated from aarch64_ad.m4
12447 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12448 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12449                          iRegL src1, iRegL src2,
12450                          immI src3) %{
12451   match(Set dst (AddL src1 (RShiftL src2 src3)));
12452 
12453   ins_cost(1.9 * INSN_COST);
12454   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12455 
12456   ins_encode %{
12457     __ add(as_Register($dst$$reg),
12458               as_Register($src1$$reg),
12459               as_Register($src2$$reg),
12460               Assembler::ASR,
12461               $src3$$constant & 0x3f);
12462   %}
12463 
12464   ins_pipe(ialu_reg_reg_shift);
12465 %}
12466 
12467 // This pattern is automatically generated from aarch64_ad.m4
12468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12469 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12470                          iRegIorL2I src1, iRegIorL2I src2,
12471                          immI src3) %{
12472   match(Set dst (AddI src1 (LShiftI src2 src3)));
12473 
12474   ins_cost(1.9 * INSN_COST);
12475   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12476 
12477   ins_encode %{
12478     __ addw(as_Register($dst$$reg),
12479               as_Register($src1$$reg),
12480               as_Register($src2$$reg),
12481               Assembler::LSL,
12482               $src3$$constant & 0x1f);
12483   %}
12484 
12485   ins_pipe(ialu_reg_reg_shift);
12486 %}
12487 
12488 // This pattern is automatically generated from aarch64_ad.m4
12489 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12490 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12491                          iRegL src1, iRegL src2,
12492                          immI src3) %{
12493   match(Set dst (AddL src1 (LShiftL src2 src3)));
12494 
12495   ins_cost(1.9 * INSN_COST);
12496   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12497 
12498   ins_encode %{
12499     __ add(as_Register($dst$$reg),
12500               as_Register($src1$$reg),
12501               as_Register($src2$$reg),
12502               Assembler::LSL,
12503               $src3$$constant & 0x3f);
12504   %}
12505 
12506   ins_pipe(ialu_reg_reg_shift);
12507 %}
12508 
12509 // This pattern is automatically generated from aarch64_ad.m4
12510 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12511 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12512                          iRegIorL2I src1, iRegIorL2I src2,
12513                          immI src3) %{
12514   match(Set dst (SubI src1 (URShiftI src2 src3)));
12515 
12516   ins_cost(1.9 * INSN_COST);
12517   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12518 
12519   ins_encode %{
12520     __ subw(as_Register($dst$$reg),
12521               as_Register($src1$$reg),
12522               as_Register($src2$$reg),
12523               Assembler::LSR,
12524               $src3$$constant & 0x1f);
12525   %}
12526 
12527   ins_pipe(ialu_reg_reg_shift);
12528 %}
12529 
12530 // This pattern is automatically generated from aarch64_ad.m4
12531 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12532 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12533                          iRegL src1, iRegL src2,
12534                          immI src3) %{
12535   match(Set dst (SubL src1 (URShiftL src2 src3)));
12536 
12537   ins_cost(1.9 * INSN_COST);
12538   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12539 
12540   ins_encode %{
12541     __ sub(as_Register($dst$$reg),
12542               as_Register($src1$$reg),
12543               as_Register($src2$$reg),
12544               Assembler::LSR,
12545               $src3$$constant & 0x3f);
12546   %}
12547 
12548   ins_pipe(ialu_reg_reg_shift);
12549 %}
12550 
12551 // This pattern is automatically generated from aarch64_ad.m4
12552 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12553 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12554                          iRegIorL2I src1, iRegIorL2I src2,
12555                          immI src3) %{
12556   match(Set dst (SubI src1 (RShiftI src2 src3)));
12557 
12558   ins_cost(1.9 * INSN_COST);
12559   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12560 
12561   ins_encode %{
12562     __ subw(as_Register($dst$$reg),
12563               as_Register($src1$$reg),
12564               as_Register($src2$$reg),
12565               Assembler::ASR,
12566               $src3$$constant & 0x1f);
12567   %}
12568 
12569   ins_pipe(ialu_reg_reg_shift);
12570 %}
12571 
12572 // This pattern is automatically generated from aarch64_ad.m4
12573 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12574 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12575                          iRegL src1, iRegL src2,
12576                          immI src3) %{
12577   match(Set dst (SubL src1 (RShiftL src2 src3)));
12578 
12579   ins_cost(1.9 * INSN_COST);
12580   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12581 
12582   ins_encode %{
12583     __ sub(as_Register($dst$$reg),
12584               as_Register($src1$$reg),
12585               as_Register($src2$$reg),
12586               Assembler::ASR,
12587               $src3$$constant & 0x3f);
12588   %}
12589 
12590   ins_pipe(ialu_reg_reg_shift);
12591 %}
12592 
12593 // This pattern is automatically generated from aarch64_ad.m4
12594 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12595 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12596                          iRegIorL2I src1, iRegIorL2I src2,
12597                          immI src3) %{
12598   match(Set dst (SubI src1 (LShiftI src2 src3)));
12599 
12600   ins_cost(1.9 * INSN_COST);
12601   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12602 
12603   ins_encode %{
12604     __ subw(as_Register($dst$$reg),
12605               as_Register($src1$$reg),
12606               as_Register($src2$$reg),
12607               Assembler::LSL,
12608               $src3$$constant & 0x1f);
12609   %}
12610 
12611   ins_pipe(ialu_reg_reg_shift);
12612 %}
12613 
12614 // This pattern is automatically generated from aarch64_ad.m4
12615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12616 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12617                          iRegL src1, iRegL src2,
12618                          immI src3) %{
12619   match(Set dst (SubL src1 (LShiftL src2 src3)));
12620 
12621   ins_cost(1.9 * INSN_COST);
12622   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12623 
12624   ins_encode %{
12625     __ sub(as_Register($dst$$reg),
12626               as_Register($src1$$reg),
12627               as_Register($src2$$reg),
12628               Assembler::LSL,
12629               $src3$$constant & 0x3f);
12630   %}
12631 
12632   ins_pipe(ialu_reg_reg_shift);
12633 %}
12634 
12635 // This pattern is automatically generated from aarch64_ad.m4
12636 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12637 
12638 // Shift Left followed by Shift Right.
12639 // This idiom is used by the compiler for the i2b bytecode etc.
12640 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12641 %{
12642   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12643   ins_cost(INSN_COST * 2);
12644   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12645   ins_encode %{
12646     int lshift = $lshift_count$$constant & 63;
12647     int rshift = $rshift_count$$constant & 63;
12648     int s = 63 - lshift;
12649     int r = (rshift - lshift) & 63;
12650     __ sbfm(as_Register($dst$$reg),
12651             as_Register($src$$reg),
12652             r, s);
12653   %}
12654 
12655   ins_pipe(ialu_reg_shift);
12656 %}
12657 
12658 // This pattern is automatically generated from aarch64_ad.m4
12659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12660 
12661 // Shift Left followed by Shift Right.
12662 // This idiom is used by the compiler for the i2b bytecode etc.
12663 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12664 %{
12665   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12666   ins_cost(INSN_COST * 2);
12667   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12668   ins_encode %{
12669     int lshift = $lshift_count$$constant & 31;
12670     int rshift = $rshift_count$$constant & 31;
12671     int s = 31 - lshift;
12672     int r = (rshift - lshift) & 31;
12673     __ sbfmw(as_Register($dst$$reg),
12674             as_Register($src$$reg),
12675             r, s);
12676   %}
12677 
12678   ins_pipe(ialu_reg_shift);
12679 %}
12680 
12681 // This pattern is automatically generated from aarch64_ad.m4
12682 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12683 
12684 // Shift Left followed by Shift Right.
12685 // This idiom is used by the compiler for the i2b bytecode etc.
12686 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12687 %{
12688   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12689   ins_cost(INSN_COST * 2);
12690   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12691   ins_encode %{
12692     int lshift = $lshift_count$$constant & 63;
12693     int rshift = $rshift_count$$constant & 63;
12694     int s = 63 - lshift;
12695     int r = (rshift - lshift) & 63;
12696     __ ubfm(as_Register($dst$$reg),
12697             as_Register($src$$reg),
12698             r, s);
12699   %}
12700 
12701   ins_pipe(ialu_reg_shift);
12702 %}
12703 
12704 // This pattern is automatically generated from aarch64_ad.m4
12705 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12706 
12707 // Shift Left followed by Shift Right.
12708 // This idiom is used by the compiler for the i2b bytecode etc.
12709 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12710 %{
12711   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12712   ins_cost(INSN_COST * 2);
12713   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12714   ins_encode %{
12715     int lshift = $lshift_count$$constant & 31;
12716     int rshift = $rshift_count$$constant & 31;
12717     int s = 31 - lshift;
12718     int r = (rshift - lshift) & 31;
12719     __ ubfmw(as_Register($dst$$reg),
12720             as_Register($src$$reg),
12721             r, s);
12722   %}
12723 
12724   ins_pipe(ialu_reg_shift);
12725 %}
12726 
12727 // Bitfield extract with shift & mask
12728 
12729 // This pattern is automatically generated from aarch64_ad.m4
12730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12731 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12732 %{
12733   match(Set dst (AndI (URShiftI src rshift) mask));
12734   // Make sure we are not going to exceed what ubfxw can do.
12735   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12736 
12737   ins_cost(INSN_COST);
12738   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12739   ins_encode %{
12740     int rshift = $rshift$$constant & 31;
12741     intptr_t mask = $mask$$constant;
12742     int width = exact_log2(mask+1);
12743     __ ubfxw(as_Register($dst$$reg),
12744             as_Register($src$$reg), rshift, width);
12745   %}
12746   ins_pipe(ialu_reg_shift);
12747 %}
12748 
12749 // This pattern is automatically generated from aarch64_ad.m4
12750 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12751 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12752 %{
12753   match(Set dst (AndL (URShiftL src rshift) mask));
12754   // Make sure we are not going to exceed what ubfx can do.
12755   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12756 
12757   ins_cost(INSN_COST);
12758   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12759   ins_encode %{
12760     int rshift = $rshift$$constant & 63;
12761     intptr_t mask = $mask$$constant;
12762     int width = exact_log2_long(mask+1);
12763     __ ubfx(as_Register($dst$$reg),
12764             as_Register($src$$reg), rshift, width);
12765   %}
12766   ins_pipe(ialu_reg_shift);
12767 %}
12768 
12769 
12770 // This pattern is automatically generated from aarch64_ad.m4
12771 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12772 
12773 // We can use ubfx when extending an And with a mask when we know mask
12774 // is positive.  We know that because immI_bitmask guarantees it.
12775 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12776 %{
12777   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12778   // Make sure we are not going to exceed what ubfxw can do.
12779   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12780 
12781   ins_cost(INSN_COST * 2);
12782   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12783   ins_encode %{
12784     int rshift = $rshift$$constant & 31;
12785     intptr_t mask = $mask$$constant;
12786     int width = exact_log2(mask+1);
12787     __ ubfx(as_Register($dst$$reg),
12788             as_Register($src$$reg), rshift, width);
12789   %}
12790   ins_pipe(ialu_reg_shift);
12791 %}
12792 
12793 
12794 // This pattern is automatically generated from aarch64_ad.m4
12795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12796 
12797 // We can use ubfiz when masking by a positive number and then left shifting the result.
12798 // We know that the mask is positive because immI_bitmask guarantees it.
12799 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12800 %{
12801   match(Set dst (LShiftI (AndI src mask) lshift));
12802   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12803 
12804   ins_cost(INSN_COST);
12805   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12806   ins_encode %{
12807     int lshift = $lshift$$constant & 31;
12808     intptr_t mask = $mask$$constant;
12809     int width = exact_log2(mask+1);
12810     __ ubfizw(as_Register($dst$$reg),
12811           as_Register($src$$reg), lshift, width);
12812   %}
12813   ins_pipe(ialu_reg_shift);
12814 %}
12815 
12816 // This pattern is automatically generated from aarch64_ad.m4
12817 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12818 
12819 // We can use ubfiz when masking by a positive number and then left shifting the result.
12820 // We know that the mask is positive because immL_bitmask guarantees it.
12821 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12822 %{
12823   match(Set dst (LShiftL (AndL src mask) lshift));
12824   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12825 
12826   ins_cost(INSN_COST);
12827   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12828   ins_encode %{
12829     int lshift = $lshift$$constant & 63;
12830     intptr_t mask = $mask$$constant;
12831     int width = exact_log2_long(mask+1);
12832     __ ubfiz(as_Register($dst$$reg),
12833           as_Register($src$$reg), lshift, width);
12834   %}
12835   ins_pipe(ialu_reg_shift);
12836 %}
12837 
12838 // This pattern is automatically generated from aarch64_ad.m4
12839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12840 
12841 // We can use ubfiz when masking by a positive number and then left shifting the result.
12842 // We know that the mask is positive because immI_bitmask guarantees it.
12843 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12844 %{
12845   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12846   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12847 
12848   ins_cost(INSN_COST);
12849   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12850   ins_encode %{
12851     int lshift = $lshift$$constant & 31;
12852     intptr_t mask = $mask$$constant;
12853     int width = exact_log2(mask+1);
12854     __ ubfizw(as_Register($dst$$reg),
12855           as_Register($src$$reg), lshift, width);
12856   %}
12857   ins_pipe(ialu_reg_shift);
12858 %}
12859 
12860 // This pattern is automatically generated from aarch64_ad.m4
12861 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12862 
12863 // We can use ubfiz when masking by a positive number and then left shifting the result.
12864 // We know that the mask is positive because immL_bitmask guarantees it.
12865 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12866 %{
12867   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12868   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12869 
12870   ins_cost(INSN_COST);
12871   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12872   ins_encode %{
12873     int lshift = $lshift$$constant & 63;
12874     intptr_t mask = $mask$$constant;
12875     int width = exact_log2_long(mask+1);
12876     __ ubfiz(as_Register($dst$$reg),
12877           as_Register($src$$reg), lshift, width);
12878   %}
12879   ins_pipe(ialu_reg_shift);
12880 %}
12881 
12882 
12883 // This pattern is automatically generated from aarch64_ad.m4
12884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12885 
12886 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12887 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12888 %{
12889   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12890   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12891 
12892   ins_cost(INSN_COST);
12893   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12894   ins_encode %{
12895     int lshift = $lshift$$constant & 63;
12896     intptr_t mask = $mask$$constant;
12897     int width = exact_log2(mask+1);
12898     __ ubfiz(as_Register($dst$$reg),
12899              as_Register($src$$reg), lshift, width);
12900   %}
12901   ins_pipe(ialu_reg_shift);
12902 %}
12903 
12904 // This pattern is automatically generated from aarch64_ad.m4
12905 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12906 
12907 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12908 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12909 %{
12910   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12911   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12912 
12913   ins_cost(INSN_COST);
12914   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12915   ins_encode %{
12916     int lshift = $lshift$$constant & 31;
12917     intptr_t mask = $mask$$constant;
12918     int width = exact_log2(mask+1);
12919     __ ubfiz(as_Register($dst$$reg),
12920              as_Register($src$$reg), lshift, width);
12921   %}
12922   ins_pipe(ialu_reg_shift);
12923 %}
12924 
12925 // This pattern is automatically generated from aarch64_ad.m4
12926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12927 
12928 // Can skip int2long conversions after AND with small bitmask
12929 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12930 %{
12931   match(Set dst (ConvI2L (AndI src msk)));
12932   ins_cost(INSN_COST);
12933   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12934   ins_encode %{
12935     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12936   %}
12937   ins_pipe(ialu_reg_shift);
12938 %}
12939 
12940 
12941 // Rotations
12942 
12943 // This pattern is automatically generated from aarch64_ad.m4
12944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12945 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12946 %{
12947   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12948   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12949 
12950   ins_cost(INSN_COST);
12951   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12952 
12953   ins_encode %{
12954     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12955             $rshift$$constant & 63);
12956   %}
12957   ins_pipe(ialu_reg_reg_extr);
12958 %}
12959 
12960 
12961 // This pattern is automatically generated from aarch64_ad.m4
12962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12963 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12964 %{
12965   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12966   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12967 
12968   ins_cost(INSN_COST);
12969   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12970 
12971   ins_encode %{
12972     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12973             $rshift$$constant & 31);
12974   %}
12975   ins_pipe(ialu_reg_reg_extr);
12976 %}
12977 
12978 
12979 // This pattern is automatically generated from aarch64_ad.m4
12980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12981 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12982 %{
12983   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12984   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12985 
12986   ins_cost(INSN_COST);
12987   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12988 
12989   ins_encode %{
12990     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12991             $rshift$$constant & 63);
12992   %}
12993   ins_pipe(ialu_reg_reg_extr);
12994 %}
12995 
12996 
12997 // This pattern is automatically generated from aarch64_ad.m4
12998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12999 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13000 %{
13001   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13002   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13003 
13004   ins_cost(INSN_COST);
13005   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13006 
13007   ins_encode %{
13008     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13009             $rshift$$constant & 31);
13010   %}
13011   ins_pipe(ialu_reg_reg_extr);
13012 %}
13013 
13014 // This pattern is automatically generated from aarch64_ad.m4
13015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13016 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13017 %{
13018   match(Set dst (RotateRight src shift));
13019 
13020   ins_cost(INSN_COST);
13021   format %{ "ror    $dst, $src, $shift" %}
13022 
13023   ins_encode %{
13024      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13025                $shift$$constant & 0x1f);
13026   %}
13027   ins_pipe(ialu_reg_reg_vshift);
13028 %}
13029 
13030 // This pattern is automatically generated from aarch64_ad.m4
13031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13032 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13033 %{
13034   match(Set dst (RotateRight src shift));
13035 
13036   ins_cost(INSN_COST);
13037   format %{ "ror    $dst, $src, $shift" %}
13038 
13039   ins_encode %{
13040      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13041                $shift$$constant & 0x3f);
13042   %}
13043   ins_pipe(ialu_reg_reg_vshift);
13044 %}
13045 
13046 // This pattern is automatically generated from aarch64_ad.m4
13047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13048 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13049 %{
13050   match(Set dst (RotateRight src shift));
13051 
13052   ins_cost(INSN_COST);
13053   format %{ "ror    $dst, $src, $shift" %}
13054 
13055   ins_encode %{
13056      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13057   %}
13058   ins_pipe(ialu_reg_reg_vshift);
13059 %}
13060 
13061 // This pattern is automatically generated from aarch64_ad.m4
13062 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13063 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13064 %{
13065   match(Set dst (RotateRight src shift));
13066 
13067   ins_cost(INSN_COST);
13068   format %{ "ror    $dst, $src, $shift" %}
13069 
13070   ins_encode %{
13071      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13072   %}
13073   ins_pipe(ialu_reg_reg_vshift);
13074 %}
13075 
13076 // This pattern is automatically generated from aarch64_ad.m4
13077 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13078 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13079 %{
13080   match(Set dst (RotateLeft src shift));
13081 
13082   ins_cost(INSN_COST);
13083   format %{ "rol    $dst, $src, $shift" %}
13084 
13085   ins_encode %{
13086      __ subw(rscratch1, zr, as_Register($shift$$reg));
13087      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13088   %}
13089   ins_pipe(ialu_reg_reg_vshift);
13090 %}
13091 
13092 // This pattern is automatically generated from aarch64_ad.m4
13093 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13094 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13095 %{
13096   match(Set dst (RotateLeft src shift));
13097 
13098   ins_cost(INSN_COST);
13099   format %{ "rol    $dst, $src, $shift" %}
13100 
13101   ins_encode %{
13102      __ subw(rscratch1, zr, as_Register($shift$$reg));
13103      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13104   %}
13105   ins_pipe(ialu_reg_reg_vshift);
13106 %}
13107 
13108 
13109 // Add/subtract (extended)
13110 
13111 // This pattern is automatically generated from aarch64_ad.m4
13112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13113 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13114 %{
13115   match(Set dst (AddL src1 (ConvI2L src2)));
13116   ins_cost(INSN_COST);
13117   format %{ "add  $dst, $src1, $src2, sxtw" %}
13118 
13119    ins_encode %{
13120      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13121             as_Register($src2$$reg), ext::sxtw);
13122    %}
13123   ins_pipe(ialu_reg_reg);
13124 %}
13125 
13126 // This pattern is automatically generated from aarch64_ad.m4
13127 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13128 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13129 %{
13130   match(Set dst (SubL src1 (ConvI2L src2)));
13131   ins_cost(INSN_COST);
13132   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13133 
13134    ins_encode %{
13135      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13136             as_Register($src2$$reg), ext::sxtw);
13137    %}
13138   ins_pipe(ialu_reg_reg);
13139 %}
13140 
13141 // This pattern is automatically generated from aarch64_ad.m4
13142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13143 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13144 %{
13145   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13146   ins_cost(INSN_COST);
13147   format %{ "add  $dst, $src1, $src2, sxth" %}
13148 
13149    ins_encode %{
13150      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13151             as_Register($src2$$reg), ext::sxth);
13152    %}
13153   ins_pipe(ialu_reg_reg);
13154 %}
13155 
13156 // This pattern is automatically generated from aarch64_ad.m4
13157 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13158 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13159 %{
13160   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13161   ins_cost(INSN_COST);
13162   format %{ "add  $dst, $src1, $src2, sxtb" %}
13163 
13164    ins_encode %{
13165      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13166             as_Register($src2$$reg), ext::sxtb);
13167    %}
13168   ins_pipe(ialu_reg_reg);
13169 %}
13170 
13171 // This pattern is automatically generated from aarch64_ad.m4
13172 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13173 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13174 %{
13175   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13176   ins_cost(INSN_COST);
13177   format %{ "add  $dst, $src1, $src2, uxtb" %}
13178 
13179    ins_encode %{
13180      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13181             as_Register($src2$$reg), ext::uxtb);
13182    %}
13183   ins_pipe(ialu_reg_reg);
13184 %}
13185 
13186 // This pattern is automatically generated from aarch64_ad.m4
13187 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13188 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13189 %{
13190   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13191   ins_cost(INSN_COST);
13192   format %{ "add  $dst, $src1, $src2, sxth" %}
13193 
13194    ins_encode %{
13195      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13196             as_Register($src2$$reg), ext::sxth);
13197    %}
13198   ins_pipe(ialu_reg_reg);
13199 %}
13200 
13201 // This pattern is automatically generated from aarch64_ad.m4
13202 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13203 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13204 %{
13205   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13206   ins_cost(INSN_COST);
13207   format %{ "add  $dst, $src1, $src2, sxtw" %}
13208 
13209    ins_encode %{
13210      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13211             as_Register($src2$$reg), ext::sxtw);
13212    %}
13213   ins_pipe(ialu_reg_reg);
13214 %}
13215 
13216 // This pattern is automatically generated from aarch64_ad.m4
13217 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13218 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13219 %{
13220   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13221   ins_cost(INSN_COST);
13222   format %{ "add  $dst, $src1, $src2, sxtb" %}
13223 
13224    ins_encode %{
13225      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13226             as_Register($src2$$reg), ext::sxtb);
13227    %}
13228   ins_pipe(ialu_reg_reg);
13229 %}
13230 
13231 // This pattern is automatically generated from aarch64_ad.m4
13232 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13233 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13234 %{
13235   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13236   ins_cost(INSN_COST);
13237   format %{ "add  $dst, $src1, $src2, uxtb" %}
13238 
13239    ins_encode %{
13240      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13241             as_Register($src2$$reg), ext::uxtb);
13242    %}
13243   ins_pipe(ialu_reg_reg);
13244 %}
13245 
13246 // This pattern is automatically generated from aarch64_ad.m4
13247 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13248 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13249 %{
13250   match(Set dst (AddI src1 (AndI src2 mask)));
13251   ins_cost(INSN_COST);
13252   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13253 
13254    ins_encode %{
13255      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13256             as_Register($src2$$reg), ext::uxtb);
13257    %}
13258   ins_pipe(ialu_reg_reg);
13259 %}
13260 
13261 // This pattern is automatically generated from aarch64_ad.m4
13262 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13263 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13264 %{
13265   match(Set dst (AddI src1 (AndI src2 mask)));
13266   ins_cost(INSN_COST);
13267   format %{ "addw  $dst, $src1, $src2, uxth" %}
13268 
13269    ins_encode %{
13270      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13271             as_Register($src2$$reg), ext::uxth);
13272    %}
13273   ins_pipe(ialu_reg_reg);
13274 %}
13275 
13276 // This pattern is automatically generated from aarch64_ad.m4
13277 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13278 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13279 %{
13280   match(Set dst (AddL src1 (AndL src2 mask)));
13281   ins_cost(INSN_COST);
13282   format %{ "add  $dst, $src1, $src2, uxtb" %}
13283 
13284    ins_encode %{
13285      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13286             as_Register($src2$$reg), ext::uxtb);
13287    %}
13288   ins_pipe(ialu_reg_reg);
13289 %}
13290 
13291 // This pattern is automatically generated from aarch64_ad.m4
13292 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13293 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13294 %{
13295   match(Set dst (AddL src1 (AndL src2 mask)));
13296   ins_cost(INSN_COST);
13297   format %{ "add  $dst, $src1, $src2, uxth" %}
13298 
13299    ins_encode %{
13300      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13301             as_Register($src2$$reg), ext::uxth);
13302    %}
13303   ins_pipe(ialu_reg_reg);
13304 %}
13305 
13306 // This pattern is automatically generated from aarch64_ad.m4
13307 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13308 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13309 %{
13310   match(Set dst (AddL src1 (AndL src2 mask)));
13311   ins_cost(INSN_COST);
13312   format %{ "add  $dst, $src1, $src2, uxtw" %}
13313 
13314    ins_encode %{
13315      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13316             as_Register($src2$$reg), ext::uxtw);
13317    %}
13318   ins_pipe(ialu_reg_reg);
13319 %}
13320 
13321 // This pattern is automatically generated from aarch64_ad.m4
13322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13323 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13324 %{
13325   match(Set dst (SubI src1 (AndI src2 mask)));
13326   ins_cost(INSN_COST);
13327   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13328 
13329    ins_encode %{
13330      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13331             as_Register($src2$$reg), ext::uxtb);
13332    %}
13333   ins_pipe(ialu_reg_reg);
13334 %}
13335 
13336 // This pattern is automatically generated from aarch64_ad.m4
13337 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13338 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13339 %{
13340   match(Set dst (SubI src1 (AndI src2 mask)));
13341   ins_cost(INSN_COST);
13342   format %{ "subw  $dst, $src1, $src2, uxth" %}
13343 
13344    ins_encode %{
13345      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13346             as_Register($src2$$reg), ext::uxth);
13347    %}
13348   ins_pipe(ialu_reg_reg);
13349 %}
13350 
13351 // This pattern is automatically generated from aarch64_ad.m4
13352 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13353 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13354 %{
13355   match(Set dst (SubL src1 (AndL src2 mask)));
13356   ins_cost(INSN_COST);
13357   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13358 
13359    ins_encode %{
13360      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13361             as_Register($src2$$reg), ext::uxtb);
13362    %}
13363   ins_pipe(ialu_reg_reg);
13364 %}
13365 
13366 // This pattern is automatically generated from aarch64_ad.m4
13367 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13368 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13369 %{
13370   match(Set dst (SubL src1 (AndL src2 mask)));
13371   ins_cost(INSN_COST);
13372   format %{ "sub  $dst, $src1, $src2, uxth" %}
13373 
13374    ins_encode %{
13375      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13376             as_Register($src2$$reg), ext::uxth);
13377    %}
13378   ins_pipe(ialu_reg_reg);
13379 %}
13380 
13381 // This pattern is automatically generated from aarch64_ad.m4
13382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13383 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13384 %{
13385   match(Set dst (SubL src1 (AndL src2 mask)));
13386   ins_cost(INSN_COST);
13387   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13388 
13389    ins_encode %{
13390      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13391             as_Register($src2$$reg), ext::uxtw);
13392    %}
13393   ins_pipe(ialu_reg_reg);
13394 %}
13395 
13396 
13397 // This pattern is automatically generated from aarch64_ad.m4
13398 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13399 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13400 %{
13401   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13402   ins_cost(1.9 * INSN_COST);
13403   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13404 
13405    ins_encode %{
13406      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13407             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13408    %}
13409   ins_pipe(ialu_reg_reg_shift);
13410 %}
13411 
13412 // This pattern is automatically generated from aarch64_ad.m4
13413 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13414 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13415 %{
13416   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13417   ins_cost(1.9 * INSN_COST);
13418   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13419 
13420    ins_encode %{
13421      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13422             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13423    %}
13424   ins_pipe(ialu_reg_reg_shift);
13425 %}
13426 
13427 // This pattern is automatically generated from aarch64_ad.m4
13428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13429 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13430 %{
13431   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13432   ins_cost(1.9 * INSN_COST);
13433   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13434 
13435    ins_encode %{
13436      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13437             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13438    %}
13439   ins_pipe(ialu_reg_reg_shift);
13440 %}
13441 
13442 // This pattern is automatically generated from aarch64_ad.m4
13443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13444 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13445 %{
13446   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13447   ins_cost(1.9 * INSN_COST);
13448   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13449 
13450    ins_encode %{
13451      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13452             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13453    %}
13454   ins_pipe(ialu_reg_reg_shift);
13455 %}
13456 
13457 // This pattern is automatically generated from aarch64_ad.m4
13458 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13459 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13460 %{
13461   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13462   ins_cost(1.9 * INSN_COST);
13463   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13464 
13465    ins_encode %{
13466      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13467             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13468    %}
13469   ins_pipe(ialu_reg_reg_shift);
13470 %}
13471 
13472 // This pattern is automatically generated from aarch64_ad.m4
13473 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13474 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13475 %{
13476   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13477   ins_cost(1.9 * INSN_COST);
13478   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13479 
13480    ins_encode %{
13481      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13482             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13483    %}
13484   ins_pipe(ialu_reg_reg_shift);
13485 %}
13486 
13487 // This pattern is automatically generated from aarch64_ad.m4
13488 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13489 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13490 %{
13491   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13492   ins_cost(1.9 * INSN_COST);
13493   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13494 
13495    ins_encode %{
13496      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13497             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13498    %}
13499   ins_pipe(ialu_reg_reg_shift);
13500 %}
13501 
13502 // This pattern is automatically generated from aarch64_ad.m4
13503 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13504 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13505 %{
13506   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13507   ins_cost(1.9 * INSN_COST);
13508   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13509 
13510    ins_encode %{
13511      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13512             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13513    %}
13514   ins_pipe(ialu_reg_reg_shift);
13515 %}
13516 
13517 // This pattern is automatically generated from aarch64_ad.m4
13518 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13519 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13520 %{
13521   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13522   ins_cost(1.9 * INSN_COST);
13523   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13524 
13525    ins_encode %{
13526      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13527             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13528    %}
13529   ins_pipe(ialu_reg_reg_shift);
13530 %}
13531 
13532 // This pattern is automatically generated from aarch64_ad.m4
13533 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13534 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13535 %{
13536   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13537   ins_cost(1.9 * INSN_COST);
13538   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13539 
13540    ins_encode %{
13541      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13542             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13543    %}
13544   ins_pipe(ialu_reg_reg_shift);
13545 %}
13546 
13547 // This pattern is automatically generated from aarch64_ad.m4
13548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13549 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13550 %{
13551   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13552   ins_cost(1.9 * INSN_COST);
13553   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13554 
13555    ins_encode %{
13556      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13557             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13558    %}
13559   ins_pipe(ialu_reg_reg_shift);
13560 %}
13561 
13562 // This pattern is automatically generated from aarch64_ad.m4
13563 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13564 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13565 %{
13566   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13567   ins_cost(1.9 * INSN_COST);
13568   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13569 
13570    ins_encode %{
13571      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13572             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13573    %}
13574   ins_pipe(ialu_reg_reg_shift);
13575 %}
13576 
13577 // This pattern is automatically generated from aarch64_ad.m4
13578 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13579 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13580 %{
13581   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13582   ins_cost(1.9 * INSN_COST);
13583   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13584 
13585    ins_encode %{
13586      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13587             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13588    %}
13589   ins_pipe(ialu_reg_reg_shift);
13590 %}
13591 
13592 // This pattern is automatically generated from aarch64_ad.m4
13593 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13594 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13595 %{
13596   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13597   ins_cost(1.9 * INSN_COST);
13598   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13599 
13600    ins_encode %{
13601      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13602             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13603    %}
13604   ins_pipe(ialu_reg_reg_shift);
13605 %}
13606 
13607 // This pattern is automatically generated from aarch64_ad.m4
13608 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13609 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13610 %{
13611   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13612   ins_cost(1.9 * INSN_COST);
13613   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13614 
13615    ins_encode %{
13616      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13617             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13618    %}
13619   ins_pipe(ialu_reg_reg_shift);
13620 %}
13621 
13622 // This pattern is automatically generated from aarch64_ad.m4
13623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13624 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13625 %{
13626   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13627   ins_cost(1.9 * INSN_COST);
13628   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13629 
13630    ins_encode %{
13631      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13632             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13633    %}
13634   ins_pipe(ialu_reg_reg_shift);
13635 %}
13636 
13637 // This pattern is automatically generated from aarch64_ad.m4
13638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13639 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13640 %{
13641   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13642   ins_cost(1.9 * INSN_COST);
13643   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13644 
13645    ins_encode %{
13646      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13647             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13648    %}
13649   ins_pipe(ialu_reg_reg_shift);
13650 %}
13651 
13652 // This pattern is automatically generated from aarch64_ad.m4
13653 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13654 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13655 %{
13656   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13657   ins_cost(1.9 * INSN_COST);
13658   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13659 
13660    ins_encode %{
13661      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13662             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13663    %}
13664   ins_pipe(ialu_reg_reg_shift);
13665 %}
13666 
13667 // This pattern is automatically generated from aarch64_ad.m4
13668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13669 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13670 %{
13671   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13672   ins_cost(1.9 * INSN_COST);
13673   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13674 
13675    ins_encode %{
13676      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13677             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13678    %}
13679   ins_pipe(ialu_reg_reg_shift);
13680 %}
13681 
13682 // This pattern is automatically generated from aarch64_ad.m4
13683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13684 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13685 %{
13686   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13687   ins_cost(1.9 * INSN_COST);
13688   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13689 
13690    ins_encode %{
13691      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13692             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13693    %}
13694   ins_pipe(ialu_reg_reg_shift);
13695 %}
13696 
13697 // This pattern is automatically generated from aarch64_ad.m4
13698 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13699 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13700 %{
13701   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13702   ins_cost(1.9 * INSN_COST);
13703   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13704 
13705    ins_encode %{
13706      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13707             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13708    %}
13709   ins_pipe(ialu_reg_reg_shift);
13710 %}
13711 
13712 // This pattern is automatically generated from aarch64_ad.m4
13713 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13714 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13715 %{
13716   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13717   ins_cost(1.9 * INSN_COST);
13718   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13719 
13720    ins_encode %{
13721      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13722             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13723    %}
13724   ins_pipe(ialu_reg_reg_shift);
13725 %}
13726 
13727 // This pattern is automatically generated from aarch64_ad.m4
13728 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13729 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13730 %{
13731   effect(DEF dst, USE src1, USE src2, USE cr);
13732   ins_cost(INSN_COST * 2);
13733   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
13734 
13735   ins_encode %{
13736     __ cselw($dst$$Register,
13737              $src1$$Register,
13738              $src2$$Register,
13739              Assembler::LT);
13740   %}
13741   ins_pipe(icond_reg_reg);
13742 %}
13743 
13744 // This pattern is automatically generated from aarch64_ad.m4
13745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13746 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13747 %{
13748   effect(DEF dst, USE src1, USE src2, USE cr);
13749   ins_cost(INSN_COST * 2);
13750   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
13751 
13752   ins_encode %{
13753     __ cselw($dst$$Register,
13754              $src1$$Register,
13755              $src2$$Register,
13756              Assembler::GT);
13757   %}
13758   ins_pipe(icond_reg_reg);
13759 %}
13760 
13761 // This pattern is automatically generated from aarch64_ad.m4
13762 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13763 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13764 %{
13765   effect(DEF dst, USE src1, USE cr);
13766   ins_cost(INSN_COST * 2);
13767   format %{ "cselw $dst, $src1, zr lt\t"  %}
13768 
13769   ins_encode %{
13770     __ cselw($dst$$Register,
13771              $src1$$Register,
13772              zr,
13773              Assembler::LT);
13774   %}
13775   ins_pipe(icond_reg);
13776 %}
13777 
13778 // This pattern is automatically generated from aarch64_ad.m4
13779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13780 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13781 %{
13782   effect(DEF dst, USE src1, USE cr);
13783   ins_cost(INSN_COST * 2);
13784   format %{ "cselw $dst, $src1, zr gt\t"  %}
13785 
13786   ins_encode %{
13787     __ cselw($dst$$Register,
13788              $src1$$Register,
13789              zr,
13790              Assembler::GT);
13791   %}
13792   ins_pipe(icond_reg);
13793 %}
13794 
13795 // This pattern is automatically generated from aarch64_ad.m4
13796 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13797 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13798 %{
13799   effect(DEF dst, USE src1, USE cr);
13800   ins_cost(INSN_COST * 2);
13801   format %{ "csincw $dst, $src1, zr le\t"  %}
13802 
13803   ins_encode %{
13804     __ csincw($dst$$Register,
13805              $src1$$Register,
13806              zr,
13807              Assembler::LE);
13808   %}
13809   ins_pipe(icond_reg);
13810 %}
13811 
13812 // This pattern is automatically generated from aarch64_ad.m4
13813 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13814 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13815 %{
13816   effect(DEF dst, USE src1, USE cr);
13817   ins_cost(INSN_COST * 2);
13818   format %{ "csincw $dst, $src1, zr gt\t"  %}
13819 
13820   ins_encode %{
13821     __ csincw($dst$$Register,
13822              $src1$$Register,
13823              zr,
13824              Assembler::GT);
13825   %}
13826   ins_pipe(icond_reg);
13827 %}
13828 
13829 // This pattern is automatically generated from aarch64_ad.m4
13830 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13831 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13832 %{
13833   effect(DEF dst, USE src1, USE cr);
13834   ins_cost(INSN_COST * 2);
13835   format %{ "csinvw $dst, $src1, zr lt\t"  %}
13836 
13837   ins_encode %{
13838     __ csinvw($dst$$Register,
13839              $src1$$Register,
13840              zr,
13841              Assembler::LT);
13842   %}
13843   ins_pipe(icond_reg);
13844 %}
13845 
13846 // This pattern is automatically generated from aarch64_ad.m4
13847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13848 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13849 %{
13850   effect(DEF dst, USE src1, USE cr);
13851   ins_cost(INSN_COST * 2);
13852   format %{ "csinvw $dst, $src1, zr ge\t"  %}
13853 
13854   ins_encode %{
13855     __ csinvw($dst$$Register,
13856              $src1$$Register,
13857              zr,
13858              Assembler::GE);
13859   %}
13860   ins_pipe(icond_reg);
13861 %}
13862 
13863 // This pattern is automatically generated from aarch64_ad.m4
13864 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13865 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13866 %{
13867   match(Set dst (MinI src imm));
13868   ins_cost(INSN_COST * 3);
13869   expand %{
13870     rFlagsReg cr;
13871     compI_reg_imm0(cr, src);
13872     cmovI_reg_imm0_lt(dst, src, cr);
13873   %}
13874 %}
13875 
13876 // This pattern is automatically generated from aarch64_ad.m4
13877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13878 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13879 %{
13880   match(Set dst (MinI imm src));
13881   ins_cost(INSN_COST * 3);
13882   expand %{
13883     rFlagsReg cr;
13884     compI_reg_imm0(cr, src);
13885     cmovI_reg_imm0_lt(dst, src, cr);
13886   %}
13887 %}
13888 
13889 // This pattern is automatically generated from aarch64_ad.m4
13890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13891 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13892 %{
13893   match(Set dst (MinI src imm));
13894   ins_cost(INSN_COST * 3);
13895   expand %{
13896     rFlagsReg cr;
13897     compI_reg_imm0(cr, src);
13898     cmovI_reg_imm1_le(dst, src, cr);
13899   %}
13900 %}
13901 
13902 // This pattern is automatically generated from aarch64_ad.m4
13903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13904 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13905 %{
13906   match(Set dst (MinI imm src));
13907   ins_cost(INSN_COST * 3);
13908   expand %{
13909     rFlagsReg cr;
13910     compI_reg_imm0(cr, src);
13911     cmovI_reg_imm1_le(dst, src, cr);
13912   %}
13913 %}
13914 
13915 // This pattern is automatically generated from aarch64_ad.m4
13916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13917 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13918 %{
13919   match(Set dst (MinI src imm));
13920   ins_cost(INSN_COST * 3);
13921   expand %{
13922     rFlagsReg cr;
13923     compI_reg_imm0(cr, src);
13924     cmovI_reg_immM1_lt(dst, src, cr);
13925   %}
13926 %}
13927 
13928 // This pattern is automatically generated from aarch64_ad.m4
13929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13930 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13931 %{
13932   match(Set dst (MinI imm src));
13933   ins_cost(INSN_COST * 3);
13934   expand %{
13935     rFlagsReg cr;
13936     compI_reg_imm0(cr, src);
13937     cmovI_reg_immM1_lt(dst, src, cr);
13938   %}
13939 %}
13940 
13941 // This pattern is automatically generated from aarch64_ad.m4
13942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13943 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13944 %{
13945   match(Set dst (MaxI src imm));
13946   ins_cost(INSN_COST * 3);
13947   expand %{
13948     rFlagsReg cr;
13949     compI_reg_imm0(cr, src);
13950     cmovI_reg_imm0_gt(dst, src, cr);
13951   %}
13952 %}
13953 
13954 // This pattern is automatically generated from aarch64_ad.m4
13955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13956 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13957 %{
13958   match(Set dst (MaxI imm src));
13959   ins_cost(INSN_COST * 3);
13960   expand %{
13961     rFlagsReg cr;
13962     compI_reg_imm0(cr, src);
13963     cmovI_reg_imm0_gt(dst, src, cr);
13964   %}
13965 %}
13966 
13967 // This pattern is automatically generated from aarch64_ad.m4
13968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13969 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13970 %{
13971   match(Set dst (MaxI src imm));
13972   ins_cost(INSN_COST * 3);
13973   expand %{
13974     rFlagsReg cr;
13975     compI_reg_imm0(cr, src);
13976     cmovI_reg_imm1_gt(dst, src, cr);
13977   %}
13978 %}
13979 
13980 // This pattern is automatically generated from aarch64_ad.m4
13981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13982 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13983 %{
13984   match(Set dst (MaxI imm src));
13985   ins_cost(INSN_COST * 3);
13986   expand %{
13987     rFlagsReg cr;
13988     compI_reg_imm0(cr, src);
13989     cmovI_reg_imm1_gt(dst, src, cr);
13990   %}
13991 %}
13992 
13993 // This pattern is automatically generated from aarch64_ad.m4
13994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13995 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13996 %{
13997   match(Set dst (MaxI src imm));
13998   ins_cost(INSN_COST * 3);
13999   expand %{
14000     rFlagsReg cr;
14001     compI_reg_imm0(cr, src);
14002     cmovI_reg_immM1_ge(dst, src, cr);
14003   %}
14004 %}
14005 
14006 // This pattern is automatically generated from aarch64_ad.m4
14007 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14008 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14009 %{
14010   match(Set dst (MaxI imm src));
14011   ins_cost(INSN_COST * 3);
14012   expand %{
14013     rFlagsReg cr;
14014     compI_reg_imm0(cr, src);
14015     cmovI_reg_immM1_ge(dst, src, cr);
14016   %}
14017 %}
14018 
14019 // This pattern is automatically generated from aarch64_ad.m4
14020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14021 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
14022 %{
14023   match(Set dst (ReverseI src));
14024   ins_cost(INSN_COST);
14025   format %{ "rbitw  $dst, $src" %}
14026   ins_encode %{
14027     __ rbitw($dst$$Register, $src$$Register);
14028   %}
14029   ins_pipe(ialu_reg);
14030 %}
14031 
14032 // This pattern is automatically generated from aarch64_ad.m4
14033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14034 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
14035 %{
14036   match(Set dst (ReverseL src));
14037   ins_cost(INSN_COST);
14038   format %{ "rbit  $dst, $src" %}
14039   ins_encode %{
14040     __ rbit($dst$$Register, $src$$Register);
14041   %}
14042   ins_pipe(ialu_reg);
14043 %}
14044 
14045 
14046 // END This section of the file is automatically generated. Do not edit --------------
14047 
14048 
14049 // ============================================================================
14050 // Floating Point Arithmetic Instructions
14051 
14052 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14053   match(Set dst (AddF src1 src2));
14054 
14055   ins_cost(INSN_COST * 5);
14056   format %{ "fadds   $dst, $src1, $src2" %}
14057 
14058   ins_encode %{
14059     __ fadds(as_FloatRegister($dst$$reg),
14060              as_FloatRegister($src1$$reg),
14061              as_FloatRegister($src2$$reg));
14062   %}
14063 
14064   ins_pipe(fp_dop_reg_reg_s);
14065 %}
14066 
14067 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14068   match(Set dst (AddD src1 src2));
14069 
14070   ins_cost(INSN_COST * 5);
14071   format %{ "faddd   $dst, $src1, $src2" %}
14072 
14073   ins_encode %{
14074     __ faddd(as_FloatRegister($dst$$reg),
14075              as_FloatRegister($src1$$reg),
14076              as_FloatRegister($src2$$reg));
14077   %}
14078 
14079   ins_pipe(fp_dop_reg_reg_d);
14080 %}
14081 
14082 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14083   match(Set dst (SubF src1 src2));
14084 
14085   ins_cost(INSN_COST * 5);
14086   format %{ "fsubs   $dst, $src1, $src2" %}
14087 
14088   ins_encode %{
14089     __ fsubs(as_FloatRegister($dst$$reg),
14090              as_FloatRegister($src1$$reg),
14091              as_FloatRegister($src2$$reg));
14092   %}
14093 
14094   ins_pipe(fp_dop_reg_reg_s);
14095 %}
14096 
14097 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14098   match(Set dst (SubD src1 src2));
14099 
14100   ins_cost(INSN_COST * 5);
14101   format %{ "fsubd   $dst, $src1, $src2" %}
14102 
14103   ins_encode %{
14104     __ fsubd(as_FloatRegister($dst$$reg),
14105              as_FloatRegister($src1$$reg),
14106              as_FloatRegister($src2$$reg));
14107   %}
14108 
14109   ins_pipe(fp_dop_reg_reg_d);
14110 %}
14111 
14112 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14113   match(Set dst (MulF src1 src2));
14114 
14115   ins_cost(INSN_COST * 6);
14116   format %{ "fmuls   $dst, $src1, $src2" %}
14117 
14118   ins_encode %{
14119     __ fmuls(as_FloatRegister($dst$$reg),
14120              as_FloatRegister($src1$$reg),
14121              as_FloatRegister($src2$$reg));
14122   %}
14123 
14124   ins_pipe(fp_dop_reg_reg_s);
14125 %}
14126 
14127 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14128   match(Set dst (MulD src1 src2));
14129 
14130   ins_cost(INSN_COST * 6);
14131   format %{ "fmuld   $dst, $src1, $src2" %}
14132 
14133   ins_encode %{
14134     __ fmuld(as_FloatRegister($dst$$reg),
14135              as_FloatRegister($src1$$reg),
14136              as_FloatRegister($src2$$reg));
14137   %}
14138 
14139   ins_pipe(fp_dop_reg_reg_d);
14140 %}
14141 
14142 // src1 * src2 + src3
14143 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14144   match(Set dst (FmaF src3 (Binary src1 src2)));
14145 
14146   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
14147 
14148   ins_encode %{
14149     assert(UseFMA, "Needs FMA instructions support.");
14150     __ fmadds(as_FloatRegister($dst$$reg),
14151              as_FloatRegister($src1$$reg),
14152              as_FloatRegister($src2$$reg),
14153              as_FloatRegister($src3$$reg));
14154   %}
14155 
14156   ins_pipe(pipe_class_default);
14157 %}
14158 
14159 // src1 * src2 + src3
14160 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14161   match(Set dst (FmaD src3 (Binary src1 src2)));
14162 
14163   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
14164 
14165   ins_encode %{
14166     assert(UseFMA, "Needs FMA instructions support.");
14167     __ fmaddd(as_FloatRegister($dst$$reg),
14168              as_FloatRegister($src1$$reg),
14169              as_FloatRegister($src2$$reg),
14170              as_FloatRegister($src3$$reg));
14171   %}
14172 
14173   ins_pipe(pipe_class_default);
14174 %}
14175 
14176 // src1 * (-src2) + src3
14177 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
14178 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14179   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14180 
14181   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
14182 
14183   ins_encode %{
14184     assert(UseFMA, "Needs FMA instructions support.");
14185     __ fmsubs(as_FloatRegister($dst$$reg),
14186               as_FloatRegister($src1$$reg),
14187               as_FloatRegister($src2$$reg),
14188               as_FloatRegister($src3$$reg));
14189   %}
14190 
14191   ins_pipe(pipe_class_default);
14192 %}
14193 
14194 // src1 * (-src2) + src3
14195 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
14196 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14197   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14198 
14199   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
14200 
14201   ins_encode %{
14202     assert(UseFMA, "Needs FMA instructions support.");
14203     __ fmsubd(as_FloatRegister($dst$$reg),
14204               as_FloatRegister($src1$$reg),
14205               as_FloatRegister($src2$$reg),
14206               as_FloatRegister($src3$$reg));
14207   %}
14208 
14209   ins_pipe(pipe_class_default);
14210 %}
14211 
14212 // src1 * (-src2) - src3
14213 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14214 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14215   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14216 
14217   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14218 
14219   ins_encode %{
14220     assert(UseFMA, "Needs FMA instructions support.");
14221     __ fnmadds(as_FloatRegister($dst$$reg),
14222                as_FloatRegister($src1$$reg),
14223                as_FloatRegister($src2$$reg),
14224                as_FloatRegister($src3$$reg));
14225   %}
14226 
14227   ins_pipe(pipe_class_default);
14228 %}
14229 
14230 // src1 * (-src2) - src3
14231 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14232 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14233   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14234 
14235   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14236 
14237   ins_encode %{
14238     assert(UseFMA, "Needs FMA instructions support.");
14239     __ fnmaddd(as_FloatRegister($dst$$reg),
14240                as_FloatRegister($src1$$reg),
14241                as_FloatRegister($src2$$reg),
14242                as_FloatRegister($src3$$reg));
14243   %}
14244 
14245   ins_pipe(pipe_class_default);
14246 %}
14247 
14248 // src1 * src2 - src3
14249 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14250   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14251 
14252   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14253 
14254   ins_encode %{
14255     assert(UseFMA, "Needs FMA instructions support.");
14256     __ fnmsubs(as_FloatRegister($dst$$reg),
14257                as_FloatRegister($src1$$reg),
14258                as_FloatRegister($src2$$reg),
14259                as_FloatRegister($src3$$reg));
14260   %}
14261 
14262   ins_pipe(pipe_class_default);
14263 %}
14264 
14265 // src1 * src2 - src3
14266 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14267   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14268 
14269   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14270 
14271   ins_encode %{
14272     assert(UseFMA, "Needs FMA instructions support.");
14273     // n.b. insn name should be fnmsubd
14274     __ fnmsub(as_FloatRegister($dst$$reg),
14275               as_FloatRegister($src1$$reg),
14276               as_FloatRegister($src2$$reg),
14277               as_FloatRegister($src3$$reg));
14278   %}
14279 
14280   ins_pipe(pipe_class_default);
14281 %}
14282 
14283 
14284 // Math.max(FF)F
14285 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14286   match(Set dst (MaxF src1 src2));
14287 
14288   format %{ "fmaxs   $dst, $src1, $src2" %}
14289   ins_encode %{
14290     __ fmaxs(as_FloatRegister($dst$$reg),
14291              as_FloatRegister($src1$$reg),
14292              as_FloatRegister($src2$$reg));
14293   %}
14294 
14295   ins_pipe(fp_dop_reg_reg_s);
14296 %}
14297 
14298 // Math.min(FF)F
14299 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14300   match(Set dst (MinF src1 src2));
14301 
14302   format %{ "fmins   $dst, $src1, $src2" %}
14303   ins_encode %{
14304     __ fmins(as_FloatRegister($dst$$reg),
14305              as_FloatRegister($src1$$reg),
14306              as_FloatRegister($src2$$reg));
14307   %}
14308 
14309   ins_pipe(fp_dop_reg_reg_s);
14310 %}
14311 
14312 // Math.max(DD)D
14313 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14314   match(Set dst (MaxD src1 src2));
14315 
14316   format %{ "fmaxd   $dst, $src1, $src2" %}
14317   ins_encode %{
14318     __ fmaxd(as_FloatRegister($dst$$reg),
14319              as_FloatRegister($src1$$reg),
14320              as_FloatRegister($src2$$reg));
14321   %}
14322 
14323   ins_pipe(fp_dop_reg_reg_d);
14324 %}
14325 
14326 // Math.min(DD)D
14327 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14328   match(Set dst (MinD src1 src2));
14329 
14330   format %{ "fmind   $dst, $src1, $src2" %}
14331   ins_encode %{
14332     __ fmind(as_FloatRegister($dst$$reg),
14333              as_FloatRegister($src1$$reg),
14334              as_FloatRegister($src2$$reg));
14335   %}
14336 
14337   ins_pipe(fp_dop_reg_reg_d);
14338 %}
14339 
14340 
14341 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14342   match(Set dst (DivF src1  src2));
14343 
14344   ins_cost(INSN_COST * 18);
14345   format %{ "fdivs   $dst, $src1, $src2" %}
14346 
14347   ins_encode %{
14348     __ fdivs(as_FloatRegister($dst$$reg),
14349              as_FloatRegister($src1$$reg),
14350              as_FloatRegister($src2$$reg));
14351   %}
14352 
14353   ins_pipe(fp_div_s);
14354 %}
14355 
14356 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14357   match(Set dst (DivD src1  src2));
14358 
14359   ins_cost(INSN_COST * 32);
14360   format %{ "fdivd   $dst, $src1, $src2" %}
14361 
14362   ins_encode %{
14363     __ fdivd(as_FloatRegister($dst$$reg),
14364              as_FloatRegister($src1$$reg),
14365              as_FloatRegister($src2$$reg));
14366   %}
14367 
14368   ins_pipe(fp_div_d);
14369 %}
14370 
14371 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14372   match(Set dst (NegF src));
14373 
14374   ins_cost(INSN_COST * 3);
14375   format %{ "fneg   $dst, $src" %}
14376 
14377   ins_encode %{
14378     __ fnegs(as_FloatRegister($dst$$reg),
14379              as_FloatRegister($src$$reg));
14380   %}
14381 
14382   ins_pipe(fp_uop_s);
14383 %}
14384 
14385 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14386   match(Set dst (NegD src));
14387 
14388   ins_cost(INSN_COST * 3);
14389   format %{ "fnegd   $dst, $src" %}
14390 
14391   ins_encode %{
14392     __ fnegd(as_FloatRegister($dst$$reg),
14393              as_FloatRegister($src$$reg));
14394   %}
14395 
14396   ins_pipe(fp_uop_d);
14397 %}
14398 
14399 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14400 %{
14401   match(Set dst (AbsI src));
14402 
14403   effect(KILL cr);
14404   ins_cost(INSN_COST * 2);
14405   format %{ "cmpw  $src, zr\n\t"
14406             "cnegw $dst, $src, Assembler::LT\t# int abs"
14407   %}
14408 
14409   ins_encode %{
14410     __ cmpw(as_Register($src$$reg), zr);
14411     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14412   %}
14413   ins_pipe(pipe_class_default);
14414 %}
14415 
14416 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14417 %{
14418   match(Set dst (AbsL src));
14419 
14420   effect(KILL cr);
14421   ins_cost(INSN_COST * 2);
14422   format %{ "cmp  $src, zr\n\t"
14423             "cneg $dst, $src, Assembler::LT\t# long abs"
14424   %}
14425 
14426   ins_encode %{
14427     __ cmp(as_Register($src$$reg), zr);
14428     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14429   %}
14430   ins_pipe(pipe_class_default);
14431 %}
14432 
14433 instruct absF_reg(vRegF dst, vRegF src) %{
14434   match(Set dst (AbsF src));
14435 
14436   ins_cost(INSN_COST * 3);
14437   format %{ "fabss   $dst, $src" %}
14438   ins_encode %{
14439     __ fabss(as_FloatRegister($dst$$reg),
14440              as_FloatRegister($src$$reg));
14441   %}
14442 
14443   ins_pipe(fp_uop_s);
14444 %}
14445 
14446 instruct absD_reg(vRegD dst, vRegD src) %{
14447   match(Set dst (AbsD src));
14448 
14449   ins_cost(INSN_COST * 3);
14450   format %{ "fabsd   $dst, $src" %}
14451   ins_encode %{
14452     __ fabsd(as_FloatRegister($dst$$reg),
14453              as_FloatRegister($src$$reg));
14454   %}
14455 
14456   ins_pipe(fp_uop_d);
14457 %}
14458 
14459 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14460   match(Set dst (AbsF (SubF src1 src2)));
14461 
14462   ins_cost(INSN_COST * 3);
14463   format %{ "fabds   $dst, $src1, $src2" %}
14464   ins_encode %{
14465     __ fabds(as_FloatRegister($dst$$reg),
14466              as_FloatRegister($src1$$reg),
14467              as_FloatRegister($src2$$reg));
14468   %}
14469 
14470   ins_pipe(fp_uop_s);
14471 %}
14472 
14473 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14474   match(Set dst (AbsD (SubD src1 src2)));
14475 
14476   ins_cost(INSN_COST * 3);
14477   format %{ "fabdd   $dst, $src1, $src2" %}
14478   ins_encode %{
14479     __ fabdd(as_FloatRegister($dst$$reg),
14480              as_FloatRegister($src1$$reg),
14481              as_FloatRegister($src2$$reg));
14482   %}
14483 
14484   ins_pipe(fp_uop_d);
14485 %}
14486 
14487 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14488   match(Set dst (SqrtD src));
14489 
14490   ins_cost(INSN_COST * 50);
14491   format %{ "fsqrtd  $dst, $src" %}
14492   ins_encode %{
14493     __ fsqrtd(as_FloatRegister($dst$$reg),
14494              as_FloatRegister($src$$reg));
14495   %}
14496 
14497   ins_pipe(fp_div_s);
14498 %}
14499 
14500 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14501   match(Set dst (SqrtF src));
14502 
14503   ins_cost(INSN_COST * 50);
14504   format %{ "fsqrts  $dst, $src" %}
14505   ins_encode %{
14506     __ fsqrts(as_FloatRegister($dst$$reg),
14507              as_FloatRegister($src$$reg));
14508   %}
14509 
14510   ins_pipe(fp_div_d);
14511 %}
14512 
14513 // Math.rint, floor, ceil
14514 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14515   match(Set dst (RoundDoubleMode src rmode));
14516   format %{ "frint  $dst, $src, $rmode" %}
14517   ins_encode %{
14518     switch ($rmode$$constant) {
14519       case RoundDoubleModeNode::rmode_rint:
14520         __ frintnd(as_FloatRegister($dst$$reg),
14521                    as_FloatRegister($src$$reg));
14522         break;
14523       case RoundDoubleModeNode::rmode_floor:
14524         __ frintmd(as_FloatRegister($dst$$reg),
14525                    as_FloatRegister($src$$reg));
14526         break;
14527       case RoundDoubleModeNode::rmode_ceil:
14528         __ frintpd(as_FloatRegister($dst$$reg),
14529                    as_FloatRegister($src$$reg));
14530         break;
14531     }
14532   %}
14533   ins_pipe(fp_uop_d);
14534 %}
14535 
14536 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14537   match(Set dst (CopySignD src1 (Binary src2 zero)));
14538   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14539   format %{ "CopySignD  $dst $src1 $src2" %}
14540   ins_encode %{
14541     FloatRegister dst = as_FloatRegister($dst$$reg),
14542                   src1 = as_FloatRegister($src1$$reg),
14543                   src2 = as_FloatRegister($src2$$reg),
14544                   zero = as_FloatRegister($zero$$reg);
14545     __ fnegd(dst, zero);
14546     __ bsl(dst, __ T8B, src2, src1);
14547   %}
14548   ins_pipe(fp_uop_d);
14549 %}
14550 
14551 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14552   match(Set dst (CopySignF src1 src2));
14553   effect(TEMP_DEF dst, USE src1, USE src2);
14554   format %{ "CopySignF  $dst $src1 $src2" %}
14555   ins_encode %{
14556     FloatRegister dst = as_FloatRegister($dst$$reg),
14557                   src1 = as_FloatRegister($src1$$reg),
14558                   src2 = as_FloatRegister($src2$$reg);
14559     __ movi(dst, __ T2S, 0x80, 24);
14560     __ bsl(dst, __ T8B, src2, src1);
14561   %}
14562   ins_pipe(fp_uop_d);
14563 %}
14564 
14565 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14566   match(Set dst (SignumD src (Binary zero one)));
14567   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14568   format %{ "signumD  $dst, $src" %}
14569   ins_encode %{
14570     FloatRegister src = as_FloatRegister($src$$reg),
14571                   dst = as_FloatRegister($dst$$reg),
14572                   zero = as_FloatRegister($zero$$reg),
14573                   one = as_FloatRegister($one$$reg);
14574     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14575     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14576     // Bit selection instruction gets bit from "one" for each enabled bit in
14577     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14578     // NaN the whole "src" will be copied because "dst" is zero. For all other
14579     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14580     // from "src", and all other bits are copied from 1.0.
14581     __ bsl(dst, __ T8B, one, src);
14582   %}
14583   ins_pipe(fp_uop_d);
14584 %}
14585 
14586 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14587   match(Set dst (SignumF src (Binary zero one)));
14588   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14589   format %{ "signumF  $dst, $src" %}
14590   ins_encode %{
14591     FloatRegister src = as_FloatRegister($src$$reg),
14592                   dst = as_FloatRegister($dst$$reg),
14593                   zero = as_FloatRegister($zero$$reg),
14594                   one = as_FloatRegister($one$$reg);
14595     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14596     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14597     // Bit selection instruction gets bit from "one" for each enabled bit in
14598     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14599     // NaN the whole "src" will be copied because "dst" is zero. For all other
14600     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14601     // from "src", and all other bits are copied from 1.0.
14602     __ bsl(dst, __ T8B, one, src);
14603   %}
14604   ins_pipe(fp_uop_d);
14605 %}
14606 
14607 instruct onspinwait() %{
14608   match(OnSpinWait);
14609   ins_cost(INSN_COST);
14610 
14611   format %{ "onspinwait" %}
14612 
14613   ins_encode %{
14614     __ spin_wait();
14615   %}
14616   ins_pipe(pipe_class_empty);
14617 %}
14618 
14619 // ============================================================================
14620 // Logical Instructions
14621 
14622 // Integer Logical Instructions
14623 
14624 // And Instructions
14625 
14626 
14627 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14628   match(Set dst (AndI src1 src2));
14629 
14630   format %{ "andw  $dst, $src1, $src2\t# int" %}
14631 
14632   ins_cost(INSN_COST);
14633   ins_encode %{
14634     __ andw(as_Register($dst$$reg),
14635             as_Register($src1$$reg),
14636             as_Register($src2$$reg));
14637   %}
14638 
14639   ins_pipe(ialu_reg_reg);
14640 %}
14641 
14642 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14643   match(Set dst (AndI src1 src2));
14644 
14645   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14646 
14647   ins_cost(INSN_COST);
14648   ins_encode %{
14649     __ andw(as_Register($dst$$reg),
14650             as_Register($src1$$reg),
14651             (uint64_t)($src2$$constant));
14652   %}
14653 
14654   ins_pipe(ialu_reg_imm);
14655 %}
14656 
14657 // Or Instructions
14658 
14659 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14660   match(Set dst (OrI src1 src2));
14661 
14662   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14663 
14664   ins_cost(INSN_COST);
14665   ins_encode %{
14666     __ orrw(as_Register($dst$$reg),
14667             as_Register($src1$$reg),
14668             as_Register($src2$$reg));
14669   %}
14670 
14671   ins_pipe(ialu_reg_reg);
14672 %}
14673 
14674 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14675   match(Set dst (OrI src1 src2));
14676 
14677   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14678 
14679   ins_cost(INSN_COST);
14680   ins_encode %{
14681     __ orrw(as_Register($dst$$reg),
14682             as_Register($src1$$reg),
14683             (uint64_t)($src2$$constant));
14684   %}
14685 
14686   ins_pipe(ialu_reg_imm);
14687 %}
14688 
14689 // Xor Instructions
14690 
14691 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14692   match(Set dst (XorI src1 src2));
14693 
14694   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14695 
14696   ins_cost(INSN_COST);
14697   ins_encode %{
14698     __ eorw(as_Register($dst$$reg),
14699             as_Register($src1$$reg),
14700             as_Register($src2$$reg));
14701   %}
14702 
14703   ins_pipe(ialu_reg_reg);
14704 %}
14705 
14706 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14707   match(Set dst (XorI src1 src2));
14708 
14709   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14710 
14711   ins_cost(INSN_COST);
14712   ins_encode %{
14713     __ eorw(as_Register($dst$$reg),
14714             as_Register($src1$$reg),
14715             (uint64_t)($src2$$constant));
14716   %}
14717 
14718   ins_pipe(ialu_reg_imm);
14719 %}
14720 
14721 // Long Logical Instructions
14722 // TODO
14723 
14724 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14725   match(Set dst (AndL src1 src2));
14726 
14727   format %{ "and  $dst, $src1, $src2\t# int" %}
14728 
14729   ins_cost(INSN_COST);
14730   ins_encode %{
14731     __ andr(as_Register($dst$$reg),
14732             as_Register($src1$$reg),
14733             as_Register($src2$$reg));
14734   %}
14735 
14736   ins_pipe(ialu_reg_reg);
14737 %}
14738 
14739 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14740   match(Set dst (AndL src1 src2));
14741 
14742   format %{ "and  $dst, $src1, $src2\t# int" %}
14743 
14744   ins_cost(INSN_COST);
14745   ins_encode %{
14746     __ andr(as_Register($dst$$reg),
14747             as_Register($src1$$reg),
14748             (uint64_t)($src2$$constant));
14749   %}
14750 
14751   ins_pipe(ialu_reg_imm);
14752 %}
14753 
14754 // Or Instructions
14755 
14756 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14757   match(Set dst (OrL src1 src2));
14758 
14759   format %{ "orr  $dst, $src1, $src2\t# int" %}
14760 
14761   ins_cost(INSN_COST);
14762   ins_encode %{
14763     __ orr(as_Register($dst$$reg),
14764            as_Register($src1$$reg),
14765            as_Register($src2$$reg));
14766   %}
14767 
14768   ins_pipe(ialu_reg_reg);
14769 %}
14770 
14771 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14772   match(Set dst (OrL src1 src2));
14773 
14774   format %{ "orr  $dst, $src1, $src2\t# int" %}
14775 
14776   ins_cost(INSN_COST);
14777   ins_encode %{
14778     __ orr(as_Register($dst$$reg),
14779            as_Register($src1$$reg),
14780            (uint64_t)($src2$$constant));
14781   %}
14782 
14783   ins_pipe(ialu_reg_imm);
14784 %}
14785 
14786 // Xor Instructions
14787 
14788 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14789   match(Set dst (XorL src1 src2));
14790 
14791   format %{ "eor  $dst, $src1, $src2\t# int" %}
14792 
14793   ins_cost(INSN_COST);
14794   ins_encode %{
14795     __ eor(as_Register($dst$$reg),
14796            as_Register($src1$$reg),
14797            as_Register($src2$$reg));
14798   %}
14799 
14800   ins_pipe(ialu_reg_reg);
14801 %}
14802 
14803 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14804   match(Set dst (XorL src1 src2));
14805 
14806   ins_cost(INSN_COST);
14807   format %{ "eor  $dst, $src1, $src2\t# int" %}
14808 
14809   ins_encode %{
14810     __ eor(as_Register($dst$$reg),
14811            as_Register($src1$$reg),
14812            (uint64_t)($src2$$constant));
14813   %}
14814 
14815   ins_pipe(ialu_reg_imm);
14816 %}
14817 
14818 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14819 %{
14820   match(Set dst (ConvI2L src));
14821 
14822   ins_cost(INSN_COST);
14823   format %{ "sxtw  $dst, $src\t# i2l" %}
14824   ins_encode %{
14825     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14826   %}
14827   ins_pipe(ialu_reg_shift);
14828 %}
14829 
14830 // this pattern occurs in bigmath arithmetic
14831 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14832 %{
14833   match(Set dst (AndL (ConvI2L src) mask));
14834 
14835   ins_cost(INSN_COST);
14836   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14837   ins_encode %{
14838     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14839   %}
14840 
14841   ins_pipe(ialu_reg_shift);
14842 %}
14843 
14844 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14845   match(Set dst (ConvL2I src));
14846 
14847   ins_cost(INSN_COST);
14848   format %{ "movw  $dst, $src \t// l2i" %}
14849 
14850   ins_encode %{
14851     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14852   %}
14853 
14854   ins_pipe(ialu_reg);
14855 %}
14856 
14857 instruct convD2F_reg(vRegF dst, vRegD src) %{
14858   match(Set dst (ConvD2F src));
14859 
14860   ins_cost(INSN_COST * 5);
14861   format %{ "fcvtd  $dst, $src \t// d2f" %}
14862 
14863   ins_encode %{
14864     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14865   %}
14866 
14867   ins_pipe(fp_d2f);
14868 %}
14869 
14870 instruct convF2D_reg(vRegD dst, vRegF src) %{
14871   match(Set dst (ConvF2D src));
14872 
14873   ins_cost(INSN_COST * 5);
14874   format %{ "fcvts  $dst, $src \t// f2d" %}
14875 
14876   ins_encode %{
14877     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14878   %}
14879 
14880   ins_pipe(fp_f2d);
14881 %}
14882 
14883 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14884   match(Set dst (ConvF2I src));
14885 
14886   ins_cost(INSN_COST * 5);
14887   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14888 
14889   ins_encode %{
14890     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14891   %}
14892 
14893   ins_pipe(fp_f2i);
14894 %}
14895 
14896 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14897   match(Set dst (ConvF2L src));
14898 
14899   ins_cost(INSN_COST * 5);
14900   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14901 
14902   ins_encode %{
14903     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14904   %}
14905 
14906   ins_pipe(fp_f2l);
14907 %}
14908 
14909 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14910   match(Set dst (ConvF2HF src));
14911   format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14912             "smov $dst, $tmp\t# move result from $tmp to $dst"
14913   %}
14914   effect(TEMP tmp);
14915   ins_encode %{
14916       __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14917   %}
14918   ins_pipe(pipe_slow);
14919 %}
14920 
14921 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14922   match(Set dst (ConvHF2F src));
14923   format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14924             "fcvt $dst, $tmp\t# convert half to single precision"
14925   %}
14926   effect(TEMP tmp);
14927   ins_encode %{
14928       __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14929   %}
14930   ins_pipe(pipe_slow);
14931 %}
14932 
14933 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14934   match(Set dst (ConvI2F src));
14935 
14936   ins_cost(INSN_COST * 5);
14937   format %{ "scvtfws  $dst, $src \t// i2f" %}
14938 
14939   ins_encode %{
14940     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14941   %}
14942 
14943   ins_pipe(fp_i2f);
14944 %}
14945 
14946 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14947   match(Set dst (ConvL2F src));
14948 
14949   ins_cost(INSN_COST * 5);
14950   format %{ "scvtfs  $dst, $src \t// l2f" %}
14951 
14952   ins_encode %{
14953     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14954   %}
14955 
14956   ins_pipe(fp_l2f);
14957 %}
14958 
14959 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14960   match(Set dst (ConvD2I src));
14961 
14962   ins_cost(INSN_COST * 5);
14963   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14964 
14965   ins_encode %{
14966     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14967   %}
14968 
14969   ins_pipe(fp_d2i);
14970 %}
14971 
14972 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14973   match(Set dst (ConvD2L src));
14974 
14975   ins_cost(INSN_COST * 5);
14976   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14977 
14978   ins_encode %{
14979     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14980   %}
14981 
14982   ins_pipe(fp_d2l);
14983 %}
14984 
14985 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14986   match(Set dst (ConvI2D src));
14987 
14988   ins_cost(INSN_COST * 5);
14989   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14990 
14991   ins_encode %{
14992     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14993   %}
14994 
14995   ins_pipe(fp_i2d);
14996 %}
14997 
14998 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14999   match(Set dst (ConvL2D src));
15000 
15001   ins_cost(INSN_COST * 5);
15002   format %{ "scvtfd  $dst, $src \t// l2d" %}
15003 
15004   ins_encode %{
15005     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15006   %}
15007 
15008   ins_pipe(fp_l2d);
15009 %}
15010 
15011 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
15012 %{
15013   match(Set dst (RoundD src));
15014   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15015   format %{ "java_round_double $dst,$src"%}
15016   ins_encode %{
15017     __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
15018                          as_FloatRegister($ftmp$$reg));
15019   %}
15020   ins_pipe(pipe_slow);
15021 %}
15022 
15023 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
15024 %{
15025   match(Set dst (RoundF src));
15026   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15027   format %{ "java_round_float $dst,$src"%}
15028   ins_encode %{
15029     __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
15030                         as_FloatRegister($ftmp$$reg));
15031   %}
15032   ins_pipe(pipe_slow);
15033 %}
15034 
15035 // stack <-> reg and reg <-> reg shuffles with no conversion
15036 
15037 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15038 
15039   match(Set dst (MoveF2I src));
15040 
15041   effect(DEF dst, USE src);
15042 
15043   ins_cost(4 * INSN_COST);
15044 
15045   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15046 
15047   ins_encode %{
15048     __ ldrw($dst$$Register, Address(sp, $src$$disp));
15049   %}
15050 
15051   ins_pipe(iload_reg_reg);
15052 
15053 %}
15054 
15055 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15056 
15057   match(Set dst (MoveI2F src));
15058 
15059   effect(DEF dst, USE src);
15060 
15061   ins_cost(4 * INSN_COST);
15062 
15063   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15064 
15065   ins_encode %{
15066     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15067   %}
15068 
15069   ins_pipe(pipe_class_memory);
15070 
15071 %}
15072 
15073 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15074 
15075   match(Set dst (MoveD2L src));
15076 
15077   effect(DEF dst, USE src);
15078 
15079   ins_cost(4 * INSN_COST);
15080 
15081   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15082 
15083   ins_encode %{
15084     __ ldr($dst$$Register, Address(sp, $src$$disp));
15085   %}
15086 
15087   ins_pipe(iload_reg_reg);
15088 
15089 %}
15090 
15091 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15092 
15093   match(Set dst (MoveL2D src));
15094 
15095   effect(DEF dst, USE src);
15096 
15097   ins_cost(4 * INSN_COST);
15098 
15099   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15100 
15101   ins_encode %{
15102     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15103   %}
15104 
15105   ins_pipe(pipe_class_memory);
15106 
15107 %}
15108 
15109 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15110 
15111   match(Set dst (MoveF2I src));
15112 
15113   effect(DEF dst, USE src);
15114 
15115   ins_cost(INSN_COST);
15116 
15117   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15118 
15119   ins_encode %{
15120     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15121   %}
15122 
15123   ins_pipe(pipe_class_memory);
15124 
15125 %}
15126 
15127 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15128 
15129   match(Set dst (MoveI2F src));
15130 
15131   effect(DEF dst, USE src);
15132 
15133   ins_cost(INSN_COST);
15134 
15135   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15136 
15137   ins_encode %{
15138     __ strw($src$$Register, Address(sp, $dst$$disp));
15139   %}
15140 
15141   ins_pipe(istore_reg_reg);
15142 
15143 %}
15144 
15145 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15146 
15147   match(Set dst (MoveD2L src));
15148 
15149   effect(DEF dst, USE src);
15150 
15151   ins_cost(INSN_COST);
15152 
15153   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15154 
15155   ins_encode %{
15156     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15157   %}
15158 
15159   ins_pipe(pipe_class_memory);
15160 
15161 %}
15162 
15163 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15164 
15165   match(Set dst (MoveL2D src));
15166 
15167   effect(DEF dst, USE src);
15168 
15169   ins_cost(INSN_COST);
15170 
15171   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15172 
15173   ins_encode %{
15174     __ str($src$$Register, Address(sp, $dst$$disp));
15175   %}
15176 
15177   ins_pipe(istore_reg_reg);
15178 
15179 %}
15180 
15181 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15182 
15183   match(Set dst (MoveF2I src));
15184 
15185   effect(DEF dst, USE src);
15186 
15187   ins_cost(INSN_COST);
15188 
15189   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15190 
15191   ins_encode %{
15192     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15193   %}
15194 
15195   ins_pipe(fp_f2i);
15196 
15197 %}
15198 
15199 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15200 
15201   match(Set dst (MoveI2F src));
15202 
15203   effect(DEF dst, USE src);
15204 
15205   ins_cost(INSN_COST);
15206 
15207   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15208 
15209   ins_encode %{
15210     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15211   %}
15212 
15213   ins_pipe(fp_i2f);
15214 
15215 %}
15216 
15217 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15218 
15219   match(Set dst (MoveD2L src));
15220 
15221   effect(DEF dst, USE src);
15222 
15223   ins_cost(INSN_COST);
15224 
15225   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15226 
15227   ins_encode %{
15228     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15229   %}
15230 
15231   ins_pipe(fp_d2l);
15232 
15233 %}
15234 
15235 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15236 
15237   match(Set dst (MoveL2D src));
15238 
15239   effect(DEF dst, USE src);
15240 
15241   ins_cost(INSN_COST);
15242 
15243   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15244 
15245   ins_encode %{
15246     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15247   %}
15248 
15249   ins_pipe(fp_l2d);
15250 
15251 %}
15252 
15253 // ============================================================================
15254 // clearing of an array
15255 
15256 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15257 %{
15258   match(Set dummy (ClearArray cnt base));
15259   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15260 
15261   ins_cost(4 * INSN_COST);
15262   format %{ "ClearArray $cnt, $base" %}
15263 
15264   ins_encode %{
15265     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15266     if (tpc == nullptr) {
15267       ciEnv::current()->record_failure("CodeCache is full");
15268       return;
15269     }
15270   %}
15271 
15272   ins_pipe(pipe_class_memory);
15273 %}
15274 
15275 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15276 %{
15277   predicate((uint64_t)n->in(2)->get_long()
15278             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15279   match(Set dummy (ClearArray cnt base));
15280   effect(TEMP temp, USE_KILL base, KILL cr);
15281 
15282   ins_cost(4 * INSN_COST);
15283   format %{ "ClearArray $cnt, $base" %}
15284 
15285   ins_encode %{
15286     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15287     if (tpc == nullptr) {
15288       ciEnv::current()->record_failure("CodeCache is full");
15289       return;
15290     }
15291   %}
15292 
15293   ins_pipe(pipe_class_memory);
15294 %}
15295 
15296 // ============================================================================
15297 // Overflow Math Instructions
15298 
15299 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15300 %{
15301   match(Set cr (OverflowAddI op1 op2));
15302 
15303   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15304   ins_cost(INSN_COST);
15305   ins_encode %{
15306     __ cmnw($op1$$Register, $op2$$Register);
15307   %}
15308 
15309   ins_pipe(icmp_reg_reg);
15310 %}
15311 
15312 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15313 %{
15314   match(Set cr (OverflowAddI op1 op2));
15315 
15316   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15317   ins_cost(INSN_COST);
15318   ins_encode %{
15319     __ cmnw($op1$$Register, $op2$$constant);
15320   %}
15321 
15322   ins_pipe(icmp_reg_imm);
15323 %}
15324 
15325 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15326 %{
15327   match(Set cr (OverflowAddL op1 op2));
15328 
15329   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15330   ins_cost(INSN_COST);
15331   ins_encode %{
15332     __ cmn($op1$$Register, $op2$$Register);
15333   %}
15334 
15335   ins_pipe(icmp_reg_reg);
15336 %}
15337 
15338 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15339 %{
15340   match(Set cr (OverflowAddL op1 op2));
15341 
15342   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
15343   ins_cost(INSN_COST);
15344   ins_encode %{
15345     __ adds(zr, $op1$$Register, $op2$$constant);
15346   %}
15347 
15348   ins_pipe(icmp_reg_imm);
15349 %}
15350 
15351 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15352 %{
15353   match(Set cr (OverflowSubI op1 op2));
15354 
15355   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15356   ins_cost(INSN_COST);
15357   ins_encode %{
15358     __ cmpw($op1$$Register, $op2$$Register);
15359   %}
15360 
15361   ins_pipe(icmp_reg_reg);
15362 %}
15363 
15364 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15365 %{
15366   match(Set cr (OverflowSubI op1 op2));
15367 
15368   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15369   ins_cost(INSN_COST);
15370   ins_encode %{
15371     __ cmpw($op1$$Register, $op2$$constant);
15372   %}
15373 
15374   ins_pipe(icmp_reg_imm);
15375 %}
15376 
15377 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15378 %{
15379   match(Set cr (OverflowSubL op1 op2));
15380 
15381   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15382   ins_cost(INSN_COST);
15383   ins_encode %{
15384     __ cmp($op1$$Register, $op2$$Register);
15385   %}
15386 
15387   ins_pipe(icmp_reg_reg);
15388 %}
15389 
15390 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15391 %{
15392   match(Set cr (OverflowSubL op1 op2));
15393 
15394   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15395   ins_cost(INSN_COST);
15396   ins_encode %{
15397     __ subs(zr, $op1$$Register, $op2$$constant);
15398   %}
15399 
15400   ins_pipe(icmp_reg_imm);
15401 %}
15402 
15403 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15404 %{
15405   match(Set cr (OverflowSubI zero op1));
15406 
15407   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15408   ins_cost(INSN_COST);
15409   ins_encode %{
15410     __ cmpw(zr, $op1$$Register);
15411   %}
15412 
15413   ins_pipe(icmp_reg_imm);
15414 %}
15415 
15416 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15417 %{
15418   match(Set cr (OverflowSubL zero op1));
15419 
15420   format %{ "cmp   zr, $op1\t# overflow check long" %}
15421   ins_cost(INSN_COST);
15422   ins_encode %{
15423     __ cmp(zr, $op1$$Register);
15424   %}
15425 
15426   ins_pipe(icmp_reg_imm);
15427 %}
15428 
15429 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15430 %{
15431   match(Set cr (OverflowMulI op1 op2));
15432 
15433   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15434             "cmp   rscratch1, rscratch1, sxtw\n\t"
15435             "movw  rscratch1, #0x80000000\n\t"
15436             "cselw rscratch1, rscratch1, zr, NE\n\t"
15437             "cmpw  rscratch1, #1" %}
15438   ins_cost(5 * INSN_COST);
15439   ins_encode %{
15440     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15441     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15442     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15443     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15444     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15445   %}
15446 
15447   ins_pipe(pipe_slow);
15448 %}
15449 
15450 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15451 %{
15452   match(If cmp (OverflowMulI op1 op2));
15453   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15454             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15455   effect(USE labl, KILL cr);
15456 
15457   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15458             "cmp   rscratch1, rscratch1, sxtw\n\t"
15459             "b$cmp   $labl" %}
15460   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15461   ins_encode %{
15462     Label* L = $labl$$label;
15463     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15464     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15465     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15466     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15467   %}
15468 
15469   ins_pipe(pipe_serial);
15470 %}
15471 
15472 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15473 %{
15474   match(Set cr (OverflowMulL op1 op2));
15475 
15476   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15477             "smulh rscratch2, $op1, $op2\n\t"
15478             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15479             "movw  rscratch1, #0x80000000\n\t"
15480             "cselw rscratch1, rscratch1, zr, NE\n\t"
15481             "cmpw  rscratch1, #1" %}
15482   ins_cost(6 * INSN_COST);
15483   ins_encode %{
15484     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15485     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15486     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15487     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15488     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15489     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15490   %}
15491 
15492   ins_pipe(pipe_slow);
15493 %}
15494 
15495 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15496 %{
15497   match(If cmp (OverflowMulL op1 op2));
15498   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15499             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15500   effect(USE labl, KILL cr);
15501 
15502   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15503             "smulh rscratch2, $op1, $op2\n\t"
15504             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15505             "b$cmp $labl" %}
15506   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15507   ins_encode %{
15508     Label* L = $labl$$label;
15509     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15510     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15511     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15512     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15513     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15514   %}
15515 
15516   ins_pipe(pipe_serial);
15517 %}
15518 
15519 // ============================================================================
15520 // Compare Instructions
15521 
15522 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15523 %{
15524   match(Set cr (CmpI op1 op2));
15525 
15526   effect(DEF cr, USE op1, USE op2);
15527 
15528   ins_cost(INSN_COST);
15529   format %{ "cmpw  $op1, $op2" %}
15530 
15531   ins_encode(aarch64_enc_cmpw(op1, op2));
15532 
15533   ins_pipe(icmp_reg_reg);
15534 %}
15535 
15536 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15537 %{
15538   match(Set cr (CmpI op1 zero));
15539 
15540   effect(DEF cr, USE op1);
15541 
15542   ins_cost(INSN_COST);
15543   format %{ "cmpw $op1, 0" %}
15544 
15545   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15546 
15547   ins_pipe(icmp_reg_imm);
15548 %}
15549 
15550 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15551 %{
15552   match(Set cr (CmpI op1 op2));
15553 
15554   effect(DEF cr, USE op1);
15555 
15556   ins_cost(INSN_COST);
15557   format %{ "cmpw  $op1, $op2" %}
15558 
15559   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15560 
15561   ins_pipe(icmp_reg_imm);
15562 %}
15563 
15564 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15565 %{
15566   match(Set cr (CmpI op1 op2));
15567 
15568   effect(DEF cr, USE op1);
15569 
15570   ins_cost(INSN_COST * 2);
15571   format %{ "cmpw  $op1, $op2" %}
15572 
15573   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15574 
15575   ins_pipe(icmp_reg_imm);
15576 %}
15577 
15578 // Unsigned compare Instructions; really, same as signed compare
15579 // except it should only be used to feed an If or a CMovI which takes a
15580 // cmpOpU.
15581 
15582 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15583 %{
15584   match(Set cr (CmpU op1 op2));
15585 
15586   effect(DEF cr, USE op1, USE op2);
15587 
15588   ins_cost(INSN_COST);
15589   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15590 
15591   ins_encode(aarch64_enc_cmpw(op1, op2));
15592 
15593   ins_pipe(icmp_reg_reg);
15594 %}
15595 
15596 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15597 %{
15598   match(Set cr (CmpU op1 zero));
15599 
15600   effect(DEF cr, USE op1);
15601 
15602   ins_cost(INSN_COST);
15603   format %{ "cmpw $op1, #0\t# unsigned" %}
15604 
15605   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15606 
15607   ins_pipe(icmp_reg_imm);
15608 %}
15609 
15610 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15611 %{
15612   match(Set cr (CmpU op1 op2));
15613 
15614   effect(DEF cr, USE op1);
15615 
15616   ins_cost(INSN_COST);
15617   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15618 
15619   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15620 
15621   ins_pipe(icmp_reg_imm);
15622 %}
15623 
15624 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15625 %{
15626   match(Set cr (CmpU op1 op2));
15627 
15628   effect(DEF cr, USE op1);
15629 
15630   ins_cost(INSN_COST * 2);
15631   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15632 
15633   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15634 
15635   ins_pipe(icmp_reg_imm);
15636 %}
15637 
15638 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15639 %{
15640   match(Set cr (CmpL op1 op2));
15641 
15642   effect(DEF cr, USE op1, USE op2);
15643 
15644   ins_cost(INSN_COST);
15645   format %{ "cmp  $op1, $op2" %}
15646 
15647   ins_encode(aarch64_enc_cmp(op1, op2));
15648 
15649   ins_pipe(icmp_reg_reg);
15650 %}
15651 
15652 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15653 %{
15654   match(Set cr (CmpL op1 zero));
15655 
15656   effect(DEF cr, USE op1);
15657 
15658   ins_cost(INSN_COST);
15659   format %{ "tst  $op1" %}
15660 
15661   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15662 
15663   ins_pipe(icmp_reg_imm);
15664 %}
15665 
15666 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15667 %{
15668   match(Set cr (CmpL op1 op2));
15669 
15670   effect(DEF cr, USE op1);
15671 
15672   ins_cost(INSN_COST);
15673   format %{ "cmp  $op1, $op2" %}
15674 
15675   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15676 
15677   ins_pipe(icmp_reg_imm);
15678 %}
15679 
15680 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15681 %{
15682   match(Set cr (CmpL op1 op2));
15683 
15684   effect(DEF cr, USE op1);
15685 
15686   ins_cost(INSN_COST * 2);
15687   format %{ "cmp  $op1, $op2" %}
15688 
15689   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15690 
15691   ins_pipe(icmp_reg_imm);
15692 %}
15693 
15694 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15695 %{
15696   match(Set cr (CmpUL op1 op2));
15697 
15698   effect(DEF cr, USE op1, USE op2);
15699 
15700   ins_cost(INSN_COST);
15701   format %{ "cmp  $op1, $op2" %}
15702 
15703   ins_encode(aarch64_enc_cmp(op1, op2));
15704 
15705   ins_pipe(icmp_reg_reg);
15706 %}
15707 
15708 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15709 %{
15710   match(Set cr (CmpUL op1 zero));
15711 
15712   effect(DEF cr, USE op1);
15713 
15714   ins_cost(INSN_COST);
15715   format %{ "tst  $op1" %}
15716 
15717   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15718 
15719   ins_pipe(icmp_reg_imm);
15720 %}
15721 
15722 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15723 %{
15724   match(Set cr (CmpUL op1 op2));
15725 
15726   effect(DEF cr, USE op1);
15727 
15728   ins_cost(INSN_COST);
15729   format %{ "cmp  $op1, $op2" %}
15730 
15731   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15732 
15733   ins_pipe(icmp_reg_imm);
15734 %}
15735 
15736 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15737 %{
15738   match(Set cr (CmpUL op1 op2));
15739 
15740   effect(DEF cr, USE op1);
15741 
15742   ins_cost(INSN_COST * 2);
15743   format %{ "cmp  $op1, $op2" %}
15744 
15745   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15746 
15747   ins_pipe(icmp_reg_imm);
15748 %}
15749 
15750 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15751 %{
15752   match(Set cr (CmpP op1 op2));
15753 
15754   effect(DEF cr, USE op1, USE op2);
15755 
15756   ins_cost(INSN_COST);
15757   format %{ "cmp  $op1, $op2\t // ptr" %}
15758 
15759   ins_encode(aarch64_enc_cmpp(op1, op2));
15760 
15761   ins_pipe(icmp_reg_reg);
15762 %}
15763 
15764 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15765 %{
15766   match(Set cr (CmpN op1 op2));
15767 
15768   effect(DEF cr, USE op1, USE op2);
15769 
15770   ins_cost(INSN_COST);
15771   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15772 
15773   ins_encode(aarch64_enc_cmpn(op1, op2));
15774 
15775   ins_pipe(icmp_reg_reg);
15776 %}
15777 
15778 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15779 %{
15780   match(Set cr (CmpP op1 zero));
15781 
15782   effect(DEF cr, USE op1, USE zero);
15783 
15784   ins_cost(INSN_COST);
15785   format %{ "cmp  $op1, 0\t // ptr" %}
15786 
15787   ins_encode(aarch64_enc_testp(op1));
15788 
15789   ins_pipe(icmp_reg_imm);
15790 %}
15791 
15792 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15793 %{
15794   match(Set cr (CmpN op1 zero));
15795 
15796   effect(DEF cr, USE op1, USE zero);
15797 
15798   ins_cost(INSN_COST);
15799   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15800 
15801   ins_encode(aarch64_enc_testn(op1));
15802 
15803   ins_pipe(icmp_reg_imm);
15804 %}
15805 
15806 // FP comparisons
15807 //
15808 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15809 // using normal cmpOp. See declaration of rFlagsReg for details.
15810 
15811 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15812 %{
15813   match(Set cr (CmpF src1 src2));
15814 
15815   ins_cost(3 * INSN_COST);
15816   format %{ "fcmps $src1, $src2" %}
15817 
15818   ins_encode %{
15819     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15820   %}
15821 
15822   ins_pipe(pipe_class_compare);
15823 %}
15824 
15825 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15826 %{
15827   match(Set cr (CmpF src1 src2));
15828 
15829   ins_cost(3 * INSN_COST);
15830   format %{ "fcmps $src1, 0.0" %}
15831 
15832   ins_encode %{
15833     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15834   %}
15835 
15836   ins_pipe(pipe_class_compare);
15837 %}
15838 // FROM HERE
15839 
15840 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15841 %{
15842   match(Set cr (CmpD src1 src2));
15843 
15844   ins_cost(3 * INSN_COST);
15845   format %{ "fcmpd $src1, $src2" %}
15846 
15847   ins_encode %{
15848     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15849   %}
15850 
15851   ins_pipe(pipe_class_compare);
15852 %}
15853 
15854 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15855 %{
15856   match(Set cr (CmpD src1 src2));
15857 
15858   ins_cost(3 * INSN_COST);
15859   format %{ "fcmpd $src1, 0.0" %}
15860 
15861   ins_encode %{
15862     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15863   %}
15864 
15865   ins_pipe(pipe_class_compare);
15866 %}
15867 
15868 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15869 %{
15870   match(Set dst (CmpF3 src1 src2));
15871   effect(KILL cr);
15872 
15873   ins_cost(5 * INSN_COST);
15874   format %{ "fcmps $src1, $src2\n\t"
15875             "csinvw($dst, zr, zr, eq\n\t"
15876             "csnegw($dst, $dst, $dst, lt)"
15877   %}
15878 
15879   ins_encode %{
15880     Label done;
15881     FloatRegister s1 = as_FloatRegister($src1$$reg);
15882     FloatRegister s2 = as_FloatRegister($src2$$reg);
15883     Register d = as_Register($dst$$reg);
15884     __ fcmps(s1, s2);
15885     // installs 0 if EQ else -1
15886     __ csinvw(d, zr, zr, Assembler::EQ);
15887     // keeps -1 if less or unordered else installs 1
15888     __ csnegw(d, d, d, Assembler::LT);
15889     __ bind(done);
15890   %}
15891 
15892   ins_pipe(pipe_class_default);
15893 
15894 %}
15895 
15896 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15897 %{
15898   match(Set dst (CmpD3 src1 src2));
15899   effect(KILL cr);
15900 
15901   ins_cost(5 * INSN_COST);
15902   format %{ "fcmpd $src1, $src2\n\t"
15903             "csinvw($dst, zr, zr, eq\n\t"
15904             "csnegw($dst, $dst, $dst, lt)"
15905   %}
15906 
15907   ins_encode %{
15908     Label done;
15909     FloatRegister s1 = as_FloatRegister($src1$$reg);
15910     FloatRegister s2 = as_FloatRegister($src2$$reg);
15911     Register d = as_Register($dst$$reg);
15912     __ fcmpd(s1, s2);
15913     // installs 0 if EQ else -1
15914     __ csinvw(d, zr, zr, Assembler::EQ);
15915     // keeps -1 if less or unordered else installs 1
15916     __ csnegw(d, d, d, Assembler::LT);
15917     __ bind(done);
15918   %}
15919   ins_pipe(pipe_class_default);
15920 
15921 %}
15922 
15923 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15924 %{
15925   match(Set dst (CmpF3 src1 zero));
15926   effect(KILL cr);
15927 
15928   ins_cost(5 * INSN_COST);
15929   format %{ "fcmps $src1, 0.0\n\t"
15930             "csinvw($dst, zr, zr, eq\n\t"
15931             "csnegw($dst, $dst, $dst, lt)"
15932   %}
15933 
15934   ins_encode %{
15935     Label done;
15936     FloatRegister s1 = as_FloatRegister($src1$$reg);
15937     Register d = as_Register($dst$$reg);
15938     __ fcmps(s1, 0.0);
15939     // installs 0 if EQ else -1
15940     __ csinvw(d, zr, zr, Assembler::EQ);
15941     // keeps -1 if less or unordered else installs 1
15942     __ csnegw(d, d, d, Assembler::LT);
15943     __ bind(done);
15944   %}
15945 
15946   ins_pipe(pipe_class_default);
15947 
15948 %}
15949 
15950 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15951 %{
15952   match(Set dst (CmpD3 src1 zero));
15953   effect(KILL cr);
15954 
15955   ins_cost(5 * INSN_COST);
15956   format %{ "fcmpd $src1, 0.0\n\t"
15957             "csinvw($dst, zr, zr, eq\n\t"
15958             "csnegw($dst, $dst, $dst, lt)"
15959   %}
15960 
15961   ins_encode %{
15962     Label done;
15963     FloatRegister s1 = as_FloatRegister($src1$$reg);
15964     Register d = as_Register($dst$$reg);
15965     __ fcmpd(s1, 0.0);
15966     // installs 0 if EQ else -1
15967     __ csinvw(d, zr, zr, Assembler::EQ);
15968     // keeps -1 if less or unordered else installs 1
15969     __ csnegw(d, d, d, Assembler::LT);
15970     __ bind(done);
15971   %}
15972   ins_pipe(pipe_class_default);
15973 
15974 %}
15975 
15976 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15977 %{
15978   match(Set dst (CmpLTMask p q));
15979   effect(KILL cr);
15980 
15981   ins_cost(3 * INSN_COST);
15982 
15983   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15984             "csetw $dst, lt\n\t"
15985             "subw $dst, zr, $dst"
15986   %}
15987 
15988   ins_encode %{
15989     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15990     __ csetw(as_Register($dst$$reg), Assembler::LT);
15991     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15992   %}
15993 
15994   ins_pipe(ialu_reg_reg);
15995 %}
15996 
15997 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15998 %{
15999   match(Set dst (CmpLTMask src zero));
16000   effect(KILL cr);
16001 
16002   ins_cost(INSN_COST);
16003 
16004   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
16005 
16006   ins_encode %{
16007     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
16008   %}
16009 
16010   ins_pipe(ialu_reg_shift);
16011 %}
16012 
16013 // ============================================================================
16014 // Max and Min
16015 
16016 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
16017 
16018 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
16019 %{
16020   effect(DEF cr, USE src);
16021   ins_cost(INSN_COST);
16022   format %{ "cmpw $src, 0" %}
16023 
16024   ins_encode %{
16025     __ cmpw($src$$Register, 0);
16026   %}
16027   ins_pipe(icmp_reg_imm);
16028 %}
16029 
16030 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16031 %{
16032   match(Set dst (MinI src1 src2));
16033   ins_cost(INSN_COST * 3);
16034 
16035   expand %{
16036     rFlagsReg cr;
16037     compI_reg_reg(cr, src1, src2);
16038     cmovI_reg_reg_lt(dst, src1, src2, cr);
16039   %}
16040 %}
16041 
16042 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16043 %{
16044   match(Set dst (MaxI src1 src2));
16045   ins_cost(INSN_COST * 3);
16046 
16047   expand %{
16048     rFlagsReg cr;
16049     compI_reg_reg(cr, src1, src2);
16050     cmovI_reg_reg_gt(dst, src1, src2, cr);
16051   %}
16052 %}
16053 
16054 
16055 // ============================================================================
16056 // Branch Instructions
16057 
16058 // Direct Branch.
16059 instruct branch(label lbl)
16060 %{
16061   match(Goto);
16062 
16063   effect(USE lbl);
16064 
16065   ins_cost(BRANCH_COST);
16066   format %{ "b  $lbl" %}
16067 
16068   ins_encode(aarch64_enc_b(lbl));
16069 
16070   ins_pipe(pipe_branch);
16071 %}
16072 
16073 // Conditional Near Branch
16074 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16075 %{
16076   // Same match rule as `branchConFar'.
16077   match(If cmp cr);
16078 
16079   effect(USE lbl);
16080 
16081   ins_cost(BRANCH_COST);
16082   // If set to 1 this indicates that the current instruction is a
16083   // short variant of a long branch. This avoids using this
16084   // instruction in first-pass matching. It will then only be used in
16085   // the `Shorten_branches' pass.
16086   // ins_short_branch(1);
16087   format %{ "b$cmp  $lbl" %}
16088 
16089   ins_encode(aarch64_enc_br_con(cmp, lbl));
16090 
16091   ins_pipe(pipe_branch_cond);
16092 %}
16093 
16094 // Conditional Near Branch Unsigned
16095 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16096 %{
16097   // Same match rule as `branchConFar'.
16098   match(If cmp cr);
16099 
16100   effect(USE lbl);
16101 
16102   ins_cost(BRANCH_COST);
16103   // If set to 1 this indicates that the current instruction is a
16104   // short variant of a long branch. This avoids using this
16105   // instruction in first-pass matching. It will then only be used in
16106   // the `Shorten_branches' pass.
16107   // ins_short_branch(1);
16108   format %{ "b$cmp  $lbl\t# unsigned" %}
16109 
16110   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16111 
16112   ins_pipe(pipe_branch_cond);
16113 %}
16114 
16115 // Make use of CBZ and CBNZ.  These instructions, as well as being
16116 // shorter than (cmp; branch), have the additional benefit of not
16117 // killing the flags.
16118 
16119 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16120   match(If cmp (CmpI op1 op2));
16121   effect(USE labl);
16122 
16123   ins_cost(BRANCH_COST);
16124   format %{ "cbw$cmp   $op1, $labl" %}
16125   ins_encode %{
16126     Label* L = $labl$$label;
16127     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16128     if (cond == Assembler::EQ)
16129       __ cbzw($op1$$Register, *L);
16130     else
16131       __ cbnzw($op1$$Register, *L);
16132   %}
16133   ins_pipe(pipe_cmp_branch);
16134 %}
16135 
16136 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16137   match(If cmp (CmpL op1 op2));
16138   effect(USE labl);
16139 
16140   ins_cost(BRANCH_COST);
16141   format %{ "cb$cmp   $op1, $labl" %}
16142   ins_encode %{
16143     Label* L = $labl$$label;
16144     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16145     if (cond == Assembler::EQ)
16146       __ cbz($op1$$Register, *L);
16147     else
16148       __ cbnz($op1$$Register, *L);
16149   %}
16150   ins_pipe(pipe_cmp_branch);
16151 %}
16152 
16153 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16154   match(If cmp (CmpP op1 op2));
16155   effect(USE labl);
16156 
16157   ins_cost(BRANCH_COST);
16158   format %{ "cb$cmp   $op1, $labl" %}
16159   ins_encode %{
16160     Label* L = $labl$$label;
16161     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16162     if (cond == Assembler::EQ)
16163       __ cbz($op1$$Register, *L);
16164     else
16165       __ cbnz($op1$$Register, *L);
16166   %}
16167   ins_pipe(pipe_cmp_branch);
16168 %}
16169 
16170 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16171   match(If cmp (CmpN op1 op2));
16172   effect(USE labl);
16173 
16174   ins_cost(BRANCH_COST);
16175   format %{ "cbw$cmp   $op1, $labl" %}
16176   ins_encode %{
16177     Label* L = $labl$$label;
16178     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16179     if (cond == Assembler::EQ)
16180       __ cbzw($op1$$Register, *L);
16181     else
16182       __ cbnzw($op1$$Register, *L);
16183   %}
16184   ins_pipe(pipe_cmp_branch);
16185 %}
16186 
16187 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16188   match(If cmp (CmpP (DecodeN oop) zero));
16189   effect(USE labl);
16190 
16191   ins_cost(BRANCH_COST);
16192   format %{ "cb$cmp   $oop, $labl" %}
16193   ins_encode %{
16194     Label* L = $labl$$label;
16195     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16196     if (cond == Assembler::EQ)
16197       __ cbzw($oop$$Register, *L);
16198     else
16199       __ cbnzw($oop$$Register, *L);
16200   %}
16201   ins_pipe(pipe_cmp_branch);
16202 %}
16203 
16204 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16205   match(If cmp (CmpU op1 op2));
16206   effect(USE labl);
16207 
16208   ins_cost(BRANCH_COST);
16209   format %{ "cbw$cmp   $op1, $labl" %}
16210   ins_encode %{
16211     Label* L = $labl$$label;
16212     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16213     if (cond == Assembler::EQ || cond == Assembler::LS)
16214       __ cbzw($op1$$Register, *L);
16215     else
16216       __ cbnzw($op1$$Register, *L);
16217   %}
16218   ins_pipe(pipe_cmp_branch);
16219 %}
16220 
16221 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16222   match(If cmp (CmpUL op1 op2));
16223   effect(USE labl);
16224 
16225   ins_cost(BRANCH_COST);
16226   format %{ "cb$cmp   $op1, $labl" %}
16227   ins_encode %{
16228     Label* L = $labl$$label;
16229     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16230     if (cond == Assembler::EQ || cond == Assembler::LS)
16231       __ cbz($op1$$Register, *L);
16232     else
16233       __ cbnz($op1$$Register, *L);
16234   %}
16235   ins_pipe(pipe_cmp_branch);
16236 %}
16237 
16238 // Test bit and Branch
16239 
16240 // Patterns for short (< 32KiB) variants
16241 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16242   match(If cmp (CmpL op1 op2));
16243   effect(USE labl);
16244 
16245   ins_cost(BRANCH_COST);
16246   format %{ "cb$cmp   $op1, $labl # long" %}
16247   ins_encode %{
16248     Label* L = $labl$$label;
16249     Assembler::Condition cond =
16250       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16251     __ tbr(cond, $op1$$Register, 63, *L);
16252   %}
16253   ins_pipe(pipe_cmp_branch);
16254   ins_short_branch(1);
16255 %}
16256 
16257 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16258   match(If cmp (CmpI op1 op2));
16259   effect(USE labl);
16260 
16261   ins_cost(BRANCH_COST);
16262   format %{ "cb$cmp   $op1, $labl # int" %}
16263   ins_encode %{
16264     Label* L = $labl$$label;
16265     Assembler::Condition cond =
16266       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16267     __ tbr(cond, $op1$$Register, 31, *L);
16268   %}
16269   ins_pipe(pipe_cmp_branch);
16270   ins_short_branch(1);
16271 %}
16272 
16273 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16274   match(If cmp (CmpL (AndL op1 op2) op3));
16275   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16276   effect(USE labl);
16277 
16278   ins_cost(BRANCH_COST);
16279   format %{ "tb$cmp   $op1, $op2, $labl" %}
16280   ins_encode %{
16281     Label* L = $labl$$label;
16282     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16283     int bit = exact_log2_long($op2$$constant);
16284     __ tbr(cond, $op1$$Register, bit, *L);
16285   %}
16286   ins_pipe(pipe_cmp_branch);
16287   ins_short_branch(1);
16288 %}
16289 
16290 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16291   match(If cmp (CmpI (AndI op1 op2) op3));
16292   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16293   effect(USE labl);
16294 
16295   ins_cost(BRANCH_COST);
16296   format %{ "tb$cmp   $op1, $op2, $labl" %}
16297   ins_encode %{
16298     Label* L = $labl$$label;
16299     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16300     int bit = exact_log2((juint)$op2$$constant);
16301     __ tbr(cond, $op1$$Register, bit, *L);
16302   %}
16303   ins_pipe(pipe_cmp_branch);
16304   ins_short_branch(1);
16305 %}
16306 
16307 // And far variants
16308 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16309   match(If cmp (CmpL op1 op2));
16310   effect(USE labl);
16311 
16312   ins_cost(BRANCH_COST);
16313   format %{ "cb$cmp   $op1, $labl # long" %}
16314   ins_encode %{
16315     Label* L = $labl$$label;
16316     Assembler::Condition cond =
16317       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16318     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16319   %}
16320   ins_pipe(pipe_cmp_branch);
16321 %}
16322 
16323 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16324   match(If cmp (CmpI op1 op2));
16325   effect(USE labl);
16326 
16327   ins_cost(BRANCH_COST);
16328   format %{ "cb$cmp   $op1, $labl # int" %}
16329   ins_encode %{
16330     Label* L = $labl$$label;
16331     Assembler::Condition cond =
16332       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16333     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16334   %}
16335   ins_pipe(pipe_cmp_branch);
16336 %}
16337 
16338 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16339   match(If cmp (CmpL (AndL op1 op2) op3));
16340   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16341   effect(USE labl);
16342 
16343   ins_cost(BRANCH_COST);
16344   format %{ "tb$cmp   $op1, $op2, $labl" %}
16345   ins_encode %{
16346     Label* L = $labl$$label;
16347     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16348     int bit = exact_log2_long($op2$$constant);
16349     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16350   %}
16351   ins_pipe(pipe_cmp_branch);
16352 %}
16353 
16354 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16355   match(If cmp (CmpI (AndI op1 op2) op3));
16356   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16357   effect(USE labl);
16358 
16359   ins_cost(BRANCH_COST);
16360   format %{ "tb$cmp   $op1, $op2, $labl" %}
16361   ins_encode %{
16362     Label* L = $labl$$label;
16363     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16364     int bit = exact_log2((juint)$op2$$constant);
16365     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16366   %}
16367   ins_pipe(pipe_cmp_branch);
16368 %}
16369 
16370 // Test bits
16371 
16372 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16373   match(Set cr (CmpL (AndL op1 op2) op3));
16374   predicate(Assembler::operand_valid_for_logical_immediate
16375             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16376 
16377   ins_cost(INSN_COST);
16378   format %{ "tst $op1, $op2 # long" %}
16379   ins_encode %{
16380     __ tst($op1$$Register, $op2$$constant);
16381   %}
16382   ins_pipe(ialu_reg_reg);
16383 %}
16384 
16385 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16386   match(Set cr (CmpI (AndI op1 op2) op3));
16387   predicate(Assembler::operand_valid_for_logical_immediate
16388             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16389 
16390   ins_cost(INSN_COST);
16391   format %{ "tst $op1, $op2 # int" %}
16392   ins_encode %{
16393     __ tstw($op1$$Register, $op2$$constant);
16394   %}
16395   ins_pipe(ialu_reg_reg);
16396 %}
16397 
16398 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16399   match(Set cr (CmpL (AndL op1 op2) op3));
16400 
16401   ins_cost(INSN_COST);
16402   format %{ "tst $op1, $op2 # long" %}
16403   ins_encode %{
16404     __ tst($op1$$Register, $op2$$Register);
16405   %}
16406   ins_pipe(ialu_reg_reg);
16407 %}
16408 
16409 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16410   match(Set cr (CmpI (AndI op1 op2) op3));
16411 
16412   ins_cost(INSN_COST);
16413   format %{ "tstw $op1, $op2 # int" %}
16414   ins_encode %{
16415     __ tstw($op1$$Register, $op2$$Register);
16416   %}
16417   ins_pipe(ialu_reg_reg);
16418 %}
16419 
16420 
16421 // Conditional Far Branch
16422 // Conditional Far Branch Unsigned
16423 // TODO: fixme
16424 
16425 // counted loop end branch near
16426 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16427 %{
16428   match(CountedLoopEnd cmp cr);
16429 
16430   effect(USE lbl);
16431 
16432   ins_cost(BRANCH_COST);
16433   // short variant.
16434   // ins_short_branch(1);
16435   format %{ "b$cmp $lbl \t// counted loop end" %}
16436 
16437   ins_encode(aarch64_enc_br_con(cmp, lbl));
16438 
16439   ins_pipe(pipe_branch);
16440 %}
16441 
16442 // counted loop end branch far
16443 // TODO: fixme
16444 
16445 // ============================================================================
16446 // inlined locking and unlocking
16447 
16448 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16449 %{
16450   predicate(LockingMode != LM_LIGHTWEIGHT);
16451   match(Set cr (FastLock object box));
16452   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16453 
16454   ins_cost(5 * INSN_COST);
16455   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16456 
16457   ins_encode %{
16458     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16459   %}
16460 
16461   ins_pipe(pipe_serial);
16462 %}
16463 
16464 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16465 %{
16466   predicate(LockingMode != LM_LIGHTWEIGHT);
16467   match(Set cr (FastUnlock object box));
16468   effect(TEMP tmp, TEMP tmp2);
16469 
16470   ins_cost(5 * INSN_COST);
16471   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16472 
16473   ins_encode %{
16474     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16475   %}
16476 
16477   ins_pipe(pipe_serial);
16478 %}
16479 
16480 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16481 %{
16482   predicate(LockingMode == LM_LIGHTWEIGHT);
16483   match(Set cr (FastLock object box));
16484   effect(TEMP tmp, TEMP tmp2);
16485 
16486   ins_cost(5 * INSN_COST);
16487   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16488 
16489   ins_encode %{
16490     __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16491   %}
16492 
16493   ins_pipe(pipe_serial);
16494 %}
16495 
16496 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16497 %{
16498   predicate(LockingMode == LM_LIGHTWEIGHT);
16499   match(Set cr (FastUnlock object box));
16500   effect(TEMP tmp, TEMP tmp2);
16501 
16502   ins_cost(5 * INSN_COST);
16503   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16504 
16505   ins_encode %{
16506     __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16507   %}
16508 
16509   ins_pipe(pipe_serial);
16510 %}
16511 
16512 // ============================================================================
16513 // Safepoint Instructions
16514 
16515 // TODO
16516 // provide a near and far version of this code
16517 
16518 instruct safePoint(rFlagsReg cr, iRegP poll)
16519 %{
16520   match(SafePoint poll);
16521   effect(KILL cr);
16522 
16523   format %{
16524     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16525   %}
16526   ins_encode %{
16527     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16528   %}
16529   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16530 %}
16531 
16532 
16533 // ============================================================================
16534 // Procedure Call/Return Instructions
16535 
16536 // Call Java Static Instruction
16537 
16538 instruct CallStaticJavaDirect(method meth)
16539 %{
16540   match(CallStaticJava);
16541 
16542   effect(USE meth);
16543 
16544   ins_cost(CALL_COST);
16545 
16546   format %{ "call,static $meth \t// ==> " %}
16547 
16548   ins_encode(aarch64_enc_java_static_call(meth),
16549              aarch64_enc_call_epilog);
16550 
16551   ins_pipe(pipe_class_call);
16552 %}
16553 
16554 // TO HERE
16555 
16556 // Call Java Dynamic Instruction
16557 instruct CallDynamicJavaDirect(method meth)
16558 %{
16559   match(CallDynamicJava);
16560 
16561   effect(USE meth);
16562 
16563   ins_cost(CALL_COST);
16564 
16565   format %{ "CALL,dynamic $meth \t// ==> " %}
16566 
16567   ins_encode(aarch64_enc_java_dynamic_call(meth),
16568              aarch64_enc_call_epilog);
16569 
16570   ins_pipe(pipe_class_call);
16571 %}
16572 
16573 // Call Runtime Instruction
16574 
16575 instruct CallRuntimeDirect(method meth)
16576 %{
16577   match(CallRuntime);
16578 
16579   effect(USE meth);
16580 
16581   ins_cost(CALL_COST);
16582 
16583   format %{ "CALL, runtime $meth" %}
16584 
16585   ins_encode( aarch64_enc_java_to_runtime(meth) );
16586 
16587   ins_pipe(pipe_class_call);
16588 %}
16589 
16590 // Call Runtime Instruction
16591 
16592 instruct CallLeafDirect(method meth)
16593 %{
16594   match(CallLeaf);
16595 
16596   effect(USE meth);
16597 
16598   ins_cost(CALL_COST);
16599 
16600   format %{ "CALL, runtime leaf $meth" %}
16601 
16602   ins_encode( aarch64_enc_java_to_runtime(meth) );
16603 
16604   ins_pipe(pipe_class_call);
16605 %}
16606 
16607 // Call Runtime Instruction
16608 
16609 instruct CallLeafNoFPDirect(method meth)
16610 %{
16611   match(CallLeafNoFP);
16612 
16613   effect(USE meth);
16614 
16615   ins_cost(CALL_COST);
16616 
16617   format %{ "CALL, runtime leaf nofp $meth" %}
16618 
16619   ins_encode( aarch64_enc_java_to_runtime(meth) );
16620 
16621   ins_pipe(pipe_class_call);
16622 %}
16623 
16624 // Tail Call; Jump from runtime stub to Java code.
16625 // Also known as an 'interprocedural jump'.
16626 // Target of jump will eventually return to caller.
16627 // TailJump below removes the return address.
16628 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16629 %{
16630   match(TailCall jump_target method_ptr);
16631 
16632   ins_cost(CALL_COST);
16633 
16634   format %{ "br $jump_target\t# $method_ptr holds method" %}
16635 
16636   ins_encode(aarch64_enc_tail_call(jump_target));
16637 
16638   ins_pipe(pipe_class_call);
16639 %}
16640 
16641 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16642 %{
16643   match(TailJump jump_target ex_oop);
16644 
16645   ins_cost(CALL_COST);
16646 
16647   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16648 
16649   ins_encode(aarch64_enc_tail_jmp(jump_target));
16650 
16651   ins_pipe(pipe_class_call);
16652 %}
16653 
16654 // Create exception oop: created by stack-crawling runtime code.
16655 // Created exception is now available to this handler, and is setup
16656 // just prior to jumping to this handler. No code emitted.
16657 // TODO check
16658 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16659 instruct CreateException(iRegP_R0 ex_oop)
16660 %{
16661   match(Set ex_oop (CreateEx));
16662 
16663   format %{ " -- \t// exception oop; no code emitted" %}
16664 
16665   size(0);
16666 
16667   ins_encode( /*empty*/ );
16668 
16669   ins_pipe(pipe_class_empty);
16670 %}
16671 
16672 // Rethrow exception: The exception oop will come in the first
16673 // argument position. Then JUMP (not call) to the rethrow stub code.
16674 instruct RethrowException() %{
16675   match(Rethrow);
16676   ins_cost(CALL_COST);
16677 
16678   format %{ "b rethrow_stub" %}
16679 
16680   ins_encode( aarch64_enc_rethrow() );
16681 
16682   ins_pipe(pipe_class_call);
16683 %}
16684 
16685 
16686 // Return Instruction
16687 // epilog node loads ret address into lr as part of frame pop
16688 instruct Ret()
16689 %{
16690   match(Return);
16691 
16692   format %{ "ret\t// return register" %}
16693 
16694   ins_encode( aarch64_enc_ret() );
16695 
16696   ins_pipe(pipe_branch);
16697 %}
16698 
16699 // Die now.
16700 instruct ShouldNotReachHere() %{
16701   match(Halt);
16702 
16703   ins_cost(CALL_COST);
16704   format %{ "ShouldNotReachHere" %}
16705 
16706   ins_encode %{
16707     if (is_reachable()) {
16708       __ stop(_halt_reason);
16709     }
16710   %}
16711 
16712   ins_pipe(pipe_class_default);
16713 %}
16714 
16715 // ============================================================================
16716 // Partial Subtype Check
16717 //
16718 // superklass array for an instance of the superklass.  Set a hidden
16719 // internal cache on a hit (cache is checked with exposed code in
16720 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16721 // encoding ALSO sets flags.
16722 
16723 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16724 %{
16725   match(Set result (PartialSubtypeCheck sub super));
16726   effect(KILL cr, KILL temp);
16727 
16728   ins_cost(1100);  // slightly larger than the next version
16729   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16730 
16731   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16732 
16733   opcode(0x1); // Force zero of result reg on hit
16734 
16735   ins_pipe(pipe_class_memory);
16736 %}
16737 
16738 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16739 %{
16740   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16741   effect(KILL temp, KILL result);
16742 
16743   ins_cost(1100);  // slightly larger than the next version
16744   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16745 
16746   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16747 
16748   opcode(0x0); // Don't zero result reg on hit
16749 
16750   ins_pipe(pipe_class_memory);
16751 %}
16752 
16753 // Intrisics for String.compareTo()
16754 
16755 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16756                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16757 %{
16758   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16759   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16760   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16761 
16762   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16763   ins_encode %{
16764     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16765     __ string_compare($str1$$Register, $str2$$Register,
16766                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16767                       $tmp1$$Register, $tmp2$$Register,
16768                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16769   %}
16770   ins_pipe(pipe_class_memory);
16771 %}
16772 
16773 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16774                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16775 %{
16776   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16777   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16778   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16779 
16780   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16781   ins_encode %{
16782     __ string_compare($str1$$Register, $str2$$Register,
16783                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16784                       $tmp1$$Register, $tmp2$$Register,
16785                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16786   %}
16787   ins_pipe(pipe_class_memory);
16788 %}
16789 
16790 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16791                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16792                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16793 %{
16794   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16795   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16796   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16797          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16798 
16799   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16800   ins_encode %{
16801     __ string_compare($str1$$Register, $str2$$Register,
16802                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16803                       $tmp1$$Register, $tmp2$$Register,
16804                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16805                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16806   %}
16807   ins_pipe(pipe_class_memory);
16808 %}
16809 
16810 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16811                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16812                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16813 %{
16814   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16815   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16816   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16817          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16818 
16819   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16820   ins_encode %{
16821     __ string_compare($str1$$Register, $str2$$Register,
16822                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16823                       $tmp1$$Register, $tmp2$$Register,
16824                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16825                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16826   %}
16827   ins_pipe(pipe_class_memory);
16828 %}
16829 
16830 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16831 // these string_compare variants as NEON register type for convenience so that the prototype of
16832 // string_compare can be shared with all variants.
16833 
16834 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16835                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16836                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16837                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16838 %{
16839   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16840   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16841   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16842          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16843 
16844   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16845   ins_encode %{
16846     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16847     __ string_compare($str1$$Register, $str2$$Register,
16848                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16849                       $tmp1$$Register, $tmp2$$Register,
16850                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16851                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16852                       StrIntrinsicNode::LL);
16853   %}
16854   ins_pipe(pipe_class_memory);
16855 %}
16856 
16857 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16858                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16859                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16860                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16861 %{
16862   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16863   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16864   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16865          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16866 
16867   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16868   ins_encode %{
16869     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16870     __ string_compare($str1$$Register, $str2$$Register,
16871                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16872                       $tmp1$$Register, $tmp2$$Register,
16873                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16874                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16875                       StrIntrinsicNode::LU);
16876   %}
16877   ins_pipe(pipe_class_memory);
16878 %}
16879 
16880 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16881                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16882                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16883                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16884 %{
16885   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16886   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16887   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16888          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16889 
16890   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16891   ins_encode %{
16892     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16893     __ string_compare($str1$$Register, $str2$$Register,
16894                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16895                       $tmp1$$Register, $tmp2$$Register,
16896                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16897                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16898                       StrIntrinsicNode::UL);
16899   %}
16900   ins_pipe(pipe_class_memory);
16901 %}
16902 
16903 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16904                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16905                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16906                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16907 %{
16908   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16909   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16910   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16911          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16912 
16913   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16914   ins_encode %{
16915     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16916     __ string_compare($str1$$Register, $str2$$Register,
16917                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16918                       $tmp1$$Register, $tmp2$$Register,
16919                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16920                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16921                       StrIntrinsicNode::UU);
16922   %}
16923   ins_pipe(pipe_class_memory);
16924 %}
16925 
16926 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16927                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16928                           iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16929                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16930 %{
16931   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16932   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16933   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16934          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16935          TEMP vtmp0, TEMP vtmp1, KILL cr);
16936   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16937             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16938 
16939   ins_encode %{
16940     __ string_indexof($str1$$Register, $str2$$Register,
16941                       $cnt1$$Register, $cnt2$$Register,
16942                       $tmp1$$Register, $tmp2$$Register,
16943                       $tmp3$$Register, $tmp4$$Register,
16944                       $tmp5$$Register, $tmp6$$Register,
16945                       -1, $result$$Register, StrIntrinsicNode::UU);
16946   %}
16947   ins_pipe(pipe_class_memory);
16948 %}
16949 
16950 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16951                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16952                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16953                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16954 %{
16955   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16956   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16957   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16958          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16959          TEMP vtmp0, TEMP vtmp1, KILL cr);
16960   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16961             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16962 
16963   ins_encode %{
16964     __ string_indexof($str1$$Register, $str2$$Register,
16965                       $cnt1$$Register, $cnt2$$Register,
16966                       $tmp1$$Register, $tmp2$$Register,
16967                       $tmp3$$Register, $tmp4$$Register,
16968                       $tmp5$$Register, $tmp6$$Register,
16969                       -1, $result$$Register, StrIntrinsicNode::LL);
16970   %}
16971   ins_pipe(pipe_class_memory);
16972 %}
16973 
16974 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16975                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16976                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16977                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16978 %{
16979   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16980   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16981   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16982          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16983          TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16984   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16985             "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16986 
16987   ins_encode %{
16988     __ string_indexof($str1$$Register, $str2$$Register,
16989                       $cnt1$$Register, $cnt2$$Register,
16990                       $tmp1$$Register, $tmp2$$Register,
16991                       $tmp3$$Register, $tmp4$$Register,
16992                       $tmp5$$Register, $tmp6$$Register,
16993                       -1, $result$$Register, StrIntrinsicNode::UL);
16994   %}
16995   ins_pipe(pipe_class_memory);
16996 %}
16997 
16998 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16999                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17000                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17001 %{
17002   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17003   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17004   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17005          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17006   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
17007             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17008 
17009   ins_encode %{
17010     int icnt2 = (int)$int_cnt2$$constant;
17011     __ string_indexof($str1$$Register, $str2$$Register,
17012                       $cnt1$$Register, zr,
17013                       $tmp1$$Register, $tmp2$$Register,
17014                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17015                       icnt2, $result$$Register, StrIntrinsicNode::UU);
17016   %}
17017   ins_pipe(pipe_class_memory);
17018 %}
17019 
17020 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17021                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17022                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17023 %{
17024   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17025   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17026   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17027          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17028   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
17029             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17030 
17031   ins_encode %{
17032     int icnt2 = (int)$int_cnt2$$constant;
17033     __ string_indexof($str1$$Register, $str2$$Register,
17034                       $cnt1$$Register, zr,
17035                       $tmp1$$Register, $tmp2$$Register,
17036                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17037                       icnt2, $result$$Register, StrIntrinsicNode::LL);
17038   %}
17039   ins_pipe(pipe_class_memory);
17040 %}
17041 
17042 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17043                               immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17044                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17045 %{
17046   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17047   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17048   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17049          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17050   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
17051             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17052 
17053   ins_encode %{
17054     int icnt2 = (int)$int_cnt2$$constant;
17055     __ string_indexof($str1$$Register, $str2$$Register,
17056                       $cnt1$$Register, zr,
17057                       $tmp1$$Register, $tmp2$$Register,
17058                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17059                       icnt2, $result$$Register, StrIntrinsicNode::UL);
17060   %}
17061   ins_pipe(pipe_class_memory);
17062 %}
17063 
17064 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17065                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17066                              iRegINoSp tmp3, rFlagsReg cr)
17067 %{
17068   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17069   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17070   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17071          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17072 
17073   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17074 
17075   ins_encode %{
17076     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17077                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
17078                            $tmp3$$Register);
17079   %}
17080   ins_pipe(pipe_class_memory);
17081 %}
17082 
17083 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17084                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17085                               iRegINoSp tmp3, rFlagsReg cr)
17086 %{
17087   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17088   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17089   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17090          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17091 
17092   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17093 
17094   ins_encode %{
17095     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17096                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
17097                             $tmp3$$Register);
17098   %}
17099   ins_pipe(pipe_class_memory);
17100 %}
17101 
17102 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17103                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17104                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17105   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17106   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17107   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17108   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17109   ins_encode %{
17110     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17111                                $result$$Register, $ztmp1$$FloatRegister,
17112                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17113                                $ptmp$$PRegister, true /* isL */);
17114   %}
17115   ins_pipe(pipe_class_memory);
17116 %}
17117 
17118 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17119                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17120                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17121   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17122   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17123   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17124   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17125   ins_encode %{
17126     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17127                                $result$$Register, $ztmp1$$FloatRegister,
17128                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17129                                $ptmp$$PRegister, false /* isL */);
17130   %}
17131   ins_pipe(pipe_class_memory);
17132 %}
17133 
17134 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17135                         iRegI_R0 result, rFlagsReg cr)
17136 %{
17137   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17138   match(Set result (StrEquals (Binary str1 str2) cnt));
17139   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17140 
17141   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17142   ins_encode %{
17143     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17144     __ string_equals($str1$$Register, $str2$$Register,
17145                      $result$$Register, $cnt$$Register);
17146   %}
17147   ins_pipe(pipe_class_memory);
17148 %}
17149 
17150 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17151                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17152                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17153                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17154                        iRegP_R10 tmp, rFlagsReg cr)
17155 %{
17156   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17157   match(Set result (AryEq ary1 ary2));
17158   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17159          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17160          TEMP vtmp6, TEMP vtmp7, KILL cr);
17161 
17162   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17163   ins_encode %{
17164     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17165                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17166                                    $result$$Register, $tmp$$Register, 1);
17167     if (tpc == nullptr) {
17168       ciEnv::current()->record_failure("CodeCache is full");
17169       return;
17170     }
17171   %}
17172   ins_pipe(pipe_class_memory);
17173 %}
17174 
17175 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17176                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17177                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17178                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17179                        iRegP_R10 tmp, rFlagsReg cr)
17180 %{
17181   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17182   match(Set result (AryEq ary1 ary2));
17183   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17184          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17185          TEMP vtmp6, TEMP vtmp7, KILL cr);
17186 
17187   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17188   ins_encode %{
17189     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17190                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17191                                    $result$$Register, $tmp$$Register, 2);
17192     if (tpc == nullptr) {
17193       ciEnv::current()->record_failure("CodeCache is full");
17194       return;
17195     }
17196   %}
17197   ins_pipe(pipe_class_memory);
17198 %}
17199 
17200 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17201 %{
17202   match(Set result (CountPositives ary1 len));
17203   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17204   format %{ "count positives byte[] $ary1,$len -> $result" %}
17205   ins_encode %{
17206     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17207     if (tpc == nullptr) {
17208       ciEnv::current()->record_failure("CodeCache is full");
17209       return;
17210     }
17211   %}
17212   ins_pipe( pipe_slow );
17213 %}
17214 
17215 // fast char[] to byte[] compression
17216 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17217                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17218                          vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17219                          iRegI_R0 result, rFlagsReg cr)
17220 %{
17221   match(Set result (StrCompressedCopy src (Binary dst len)));
17222   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17223          USE_KILL src, USE_KILL dst, USE len, KILL cr);
17224 
17225   format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17226   ins_encode %{
17227     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17228                            $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17229                            $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17230                            $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17231   %}
17232   ins_pipe(pipe_slow);
17233 %}
17234 
17235 // fast byte[] to char[] inflation
17236 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17237                         vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17238                         vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17239 %{
17240   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17241   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17242          TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17243          USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17244 
17245   format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17246   ins_encode %{
17247     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17248                                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17249                                         $vtmp2$$FloatRegister, $tmp$$Register);
17250     if (tpc == nullptr) {
17251       ciEnv::current()->record_failure("CodeCache is full");
17252       return;
17253     }
17254   %}
17255   ins_pipe(pipe_class_memory);
17256 %}
17257 
17258 // encode char[] to byte[] in ISO_8859_1
17259 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17260                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17261                           vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17262                           iRegI_R0 result, rFlagsReg cr)
17263 %{
17264   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17265   match(Set result (EncodeISOArray src (Binary dst len)));
17266   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17267          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17268 
17269   format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17270   ins_encode %{
17271     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17272                         $result$$Register, false,
17273                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17274                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17275                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17276   %}
17277   ins_pipe(pipe_class_memory);
17278 %}
17279 
17280 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17281                             vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17282                             vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17283                             iRegI_R0 result, rFlagsReg cr)
17284 %{
17285   predicate(((EncodeISOArrayNode*)n)->is_ascii());
17286   match(Set result (EncodeISOArray src (Binary dst len)));
17287   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17288          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17289 
17290   format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17291   ins_encode %{
17292     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17293                         $result$$Register, true,
17294                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17295                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17296                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17297   %}
17298   ins_pipe(pipe_class_memory);
17299 %}
17300 
17301 //----------------------------- CompressBits/ExpandBits ------------------------
17302 
17303 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17304                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17305   match(Set dst (CompressBits src mask));
17306   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17307   format %{ "mov    $tsrc, $src\n\t"
17308             "mov    $tmask, $mask\n\t"
17309             "bext   $tdst, $tsrc, $tmask\n\t"
17310             "mov    $dst, $tdst"
17311           %}
17312   ins_encode %{
17313     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17314     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17315     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17316     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17317   %}
17318   ins_pipe(pipe_slow);
17319 %}
17320 
17321 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17322                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17323   match(Set dst (CompressBits (LoadI mem) mask));
17324   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17325   format %{ "ldrs   $tsrc, $mem\n\t"
17326             "ldrs   $tmask, $mask\n\t"
17327             "bext   $tdst, $tsrc, $tmask\n\t"
17328             "mov    $dst, $tdst"
17329           %}
17330   ins_encode %{
17331     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17332               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17333     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17334     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17335     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17336   %}
17337   ins_pipe(pipe_slow);
17338 %}
17339 
17340 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17341                            vRegD tdst, vRegD tsrc, vRegD tmask) %{
17342   match(Set dst (CompressBits src mask));
17343   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17344   format %{ "mov    $tsrc, $src\n\t"
17345             "mov    $tmask, $mask\n\t"
17346             "bext   $tdst, $tsrc, $tmask\n\t"
17347             "mov    $dst, $tdst"
17348           %}
17349   ins_encode %{
17350     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17351     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17352     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17353     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17354   %}
17355   ins_pipe(pipe_slow);
17356 %}
17357 
17358 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17359                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17360   match(Set dst (CompressBits (LoadL mem) mask));
17361   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17362   format %{ "ldrd   $tsrc, $mem\n\t"
17363             "ldrd   $tmask, $mask\n\t"
17364             "bext   $tdst, $tsrc, $tmask\n\t"
17365             "mov    $dst, $tdst"
17366           %}
17367   ins_encode %{
17368     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17369               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17370     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17371     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17372     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17373   %}
17374   ins_pipe(pipe_slow);
17375 %}
17376 
17377 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17378                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17379   match(Set dst (ExpandBits src mask));
17380   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17381   format %{ "mov    $tsrc, $src\n\t"
17382             "mov    $tmask, $mask\n\t"
17383             "bdep   $tdst, $tsrc, $tmask\n\t"
17384             "mov    $dst, $tdst"
17385           %}
17386   ins_encode %{
17387     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17388     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17389     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17390     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17391   %}
17392   ins_pipe(pipe_slow);
17393 %}
17394 
17395 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17396                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17397   match(Set dst (ExpandBits (LoadI mem) mask));
17398   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17399   format %{ "ldrs   $tsrc, $mem\n\t"
17400             "ldrs   $tmask, $mask\n\t"
17401             "bdep   $tdst, $tsrc, $tmask\n\t"
17402             "mov    $dst, $tdst"
17403           %}
17404   ins_encode %{
17405     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17406               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17407     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17408     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17409     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17410   %}
17411   ins_pipe(pipe_slow);
17412 %}
17413 
17414 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17415                          vRegD tdst, vRegD tsrc, vRegD tmask) %{
17416   match(Set dst (ExpandBits src mask));
17417   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17418   format %{ "mov    $tsrc, $src\n\t"
17419             "mov    $tmask, $mask\n\t"
17420             "bdep   $tdst, $tsrc, $tmask\n\t"
17421             "mov    $dst, $tdst"
17422           %}
17423   ins_encode %{
17424     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17425     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17426     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17427     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17428   %}
17429   ins_pipe(pipe_slow);
17430 %}
17431 
17432 
17433 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17434                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17435   match(Set dst (ExpandBits (LoadL mem) mask));
17436   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17437   format %{ "ldrd   $tsrc, $mem\n\t"
17438             "ldrd   $tmask, $mask\n\t"
17439             "bdep   $tdst, $tsrc, $tmask\n\t"
17440             "mov    $dst, $tdst"
17441           %}
17442   ins_encode %{
17443     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17444               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17445     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17446     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17447     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17448   %}
17449   ins_pipe(pipe_slow);
17450 %}
17451 
17452 // ============================================================================
17453 // This name is KNOWN by the ADLC and cannot be changed.
17454 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17455 // for this guy.
17456 instruct tlsLoadP(thread_RegP dst)
17457 %{
17458   match(Set dst (ThreadLocal));
17459 
17460   ins_cost(0);
17461 
17462   format %{ " -- \t// $dst=Thread::current(), empty" %}
17463 
17464   size(0);
17465 
17466   ins_encode( /*empty*/ );
17467 
17468   ins_pipe(pipe_class_empty);
17469 %}
17470 
17471 //----------PEEPHOLE RULES-----------------------------------------------------
17472 // These must follow all instruction definitions as they use the names
17473 // defined in the instructions definitions.
17474 //
17475 // peepmatch ( root_instr_name [preceding_instruction]* );
17476 //
17477 // peepconstraint %{
17478 // (instruction_number.operand_name relational_op instruction_number.operand_name
17479 //  [, ...] );
17480 // // instruction numbers are zero-based using left to right order in peepmatch
17481 //
17482 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17483 // // provide an instruction_number.operand_name for each operand that appears
17484 // // in the replacement instruction's match rule
17485 //
17486 // ---------VM FLAGS---------------------------------------------------------
17487 //
17488 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17489 //
17490 // Each peephole rule is given an identifying number starting with zero and
17491 // increasing by one in the order seen by the parser.  An individual peephole
17492 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17493 // on the command-line.
17494 //
17495 // ---------CURRENT LIMITATIONS----------------------------------------------
17496 //
17497 // Only match adjacent instructions in same basic block
17498 // Only equality constraints
17499 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17500 // Only one replacement instruction
17501 //
17502 // ---------EXAMPLE----------------------------------------------------------
17503 //
17504 // // pertinent parts of existing instructions in architecture description
17505 // instruct movI(iRegINoSp dst, iRegI src)
17506 // %{
17507 //   match(Set dst (CopyI src));
17508 // %}
17509 //
17510 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17511 // %{
17512 //   match(Set dst (AddI dst src));
17513 //   effect(KILL cr);
17514 // %}
17515 //
17516 // // Change (inc mov) to lea
17517 // peephole %{
17518 //   // increment preceded by register-register move
17519 //   peepmatch ( incI_iReg movI );
17520 //   // require that the destination register of the increment
17521 //   // match the destination register of the move
17522 //   peepconstraint ( 0.dst == 1.dst );
17523 //   // construct a replacement instruction that sets
17524 //   // the destination to ( move's source register + one )
17525 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17526 // %}
17527 //
17528 
17529 // Implementation no longer uses movX instructions since
17530 // machine-independent system no longer uses CopyX nodes.
17531 //
17532 // peephole
17533 // %{
17534 //   peepmatch (incI_iReg movI);
17535 //   peepconstraint (0.dst == 1.dst);
17536 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17537 // %}
17538 
17539 // peephole
17540 // %{
17541 //   peepmatch (decI_iReg movI);
17542 //   peepconstraint (0.dst == 1.dst);
17543 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17544 // %}
17545 
17546 // peephole
17547 // %{
17548 //   peepmatch (addI_iReg_imm movI);
17549 //   peepconstraint (0.dst == 1.dst);
17550 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17551 // %}
17552 
17553 // peephole
17554 // %{
17555 //   peepmatch (incL_iReg movL);
17556 //   peepconstraint (0.dst == 1.dst);
17557 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17558 // %}
17559 
17560 // peephole
17561 // %{
17562 //   peepmatch (decL_iReg movL);
17563 //   peepconstraint (0.dst == 1.dst);
17564 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17565 // %}
17566 
17567 // peephole
17568 // %{
17569 //   peepmatch (addL_iReg_imm movL);
17570 //   peepconstraint (0.dst == 1.dst);
17571 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17572 // %}
17573 
17574 // peephole
17575 // %{
17576 //   peepmatch (addP_iReg_imm movP);
17577 //   peepconstraint (0.dst == 1.dst);
17578 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17579 // %}
17580 
17581 // // Change load of spilled value to only a spill
17582 // instruct storeI(memory mem, iRegI src)
17583 // %{
17584 //   match(Set mem (StoreI mem src));
17585 // %}
17586 //
17587 // instruct loadI(iRegINoSp dst, memory mem)
17588 // %{
17589 //   match(Set dst (LoadI mem));
17590 // %}
17591 //
17592 
17593 //----------SMARTSPILL RULES---------------------------------------------------
17594 // These must follow all instruction definitions as they use the names
17595 // defined in the instructions definitions.
17596 
17597 // Local Variables:
17598 // mode: c++
17599 // End: