1 //
    2 // Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // architecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182 
  183   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  184   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  185   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  186   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  187 
  188   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  189   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  190   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  191   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  192 
  193   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  194   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  195   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  196   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  197 
  198   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  199   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  200   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  201   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  202 
  203   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  204   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  205   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  206   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  207 
  208   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  209   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  210   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  211   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  212 
  213   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  214   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  215   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  216   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  217 
  218   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  219   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  220   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  221   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  222 
  223   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  224   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  225   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  226   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  227 
  228   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  229   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  230   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  231   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  232 
  233   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  234   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  235   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  236   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  237 
  238   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  239   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  240   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  241   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  242 
  243   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  244   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  245   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  246   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  247 
  248   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  249   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  250   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  251   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  252 
  253   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  254   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  255   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  256   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  257 
  258   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  259   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  260   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  261   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  262 
  263   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  264   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  265   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  266   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  267 
  268   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  269   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  270   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  271   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  272 
  273   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  274   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  275   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  276   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  277 
  278   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  279   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  280   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  281   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  282 
  283   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  284   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  285   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  286   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  287 
  288   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  289   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  290   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  291   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  292 
  293   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  294   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  295   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  296   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  297 
  298   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  299   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  300   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  301   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  302 
  303   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  304   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  305   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  306   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  307 
  308   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  309   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  310   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  311   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  312 
  313   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  314   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  315   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  316   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  317 
  318   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  319   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  320   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  321   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  322 
  323   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  324   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  325   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  326   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  327 
  328   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  329   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  330   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  331   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  332 
  333   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  334   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  335   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  336   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  337 
  338 // ----------------------------
  339 // SVE Predicate Registers
  340 // ----------------------------
  341   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  342   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  343   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  344   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  345   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  346   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  347   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  348   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  349   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  350   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  351   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  352   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  353   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  354   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  355   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  356   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  357 
  358 // ----------------------------
  359 // Special Registers
  360 // ----------------------------
  361 
  362 // the AArch64 CSPR status flag register is not directly accessible as
  363 // instruction operand. the FPSR status flag register is a system
  364 // register which can be written/read using MSR/MRS but again does not
  365 // appear as an operand (a code identifying the FSPR occurs as an
  366 // immediate value in the instruction).
  367 
  368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  369 
  370 // Specify priority of register selection within phases of register
  371 // allocation.  Highest priority is first.  A useful heuristic is to
  372 // give registers a low priority when they are required by machine
  373 // instructions, like EAX and EDX on I486, and choose no-save registers
  374 // before save-on-call, & save-on-call before save-on-entry.  Registers
  375 // which participate in fixed calling sequences should come last.
  376 // Registers which are used as pairs must fall on an even boundary.
  377 
  378 alloc_class chunk0(
  379     // volatiles
  380     R10, R10_H,
  381     R11, R11_H,
  382     R12, R12_H,
  383     R13, R13_H,
  384     R14, R14_H,
  385     R15, R15_H,
  386     R16, R16_H,
  387     R17, R17_H,
  388     R18, R18_H,
  389 
  390     // arg registers
  391     R0, R0_H,
  392     R1, R1_H,
  393     R2, R2_H,
  394     R3, R3_H,
  395     R4, R4_H,
  396     R5, R5_H,
  397     R6, R6_H,
  398     R7, R7_H,
  399 
  400     // non-volatiles
  401     R19, R19_H,
  402     R20, R20_H,
  403     R21, R21_H,
  404     R22, R22_H,
  405     R23, R23_H,
  406     R24, R24_H,
  407     R25, R25_H,
  408     R26, R26_H,
  409 
  410     // non-allocatable registers
  411 
  412     R27, R27_H, // heapbase
  413     R28, R28_H, // thread
  414     R29, R29_H, // fp
  415     R30, R30_H, // lr
  416     R31, R31_H, // sp
  417     R8, R8_H,   // rscratch1
  418     R9, R9_H,   // rscratch2
  419 );
  420 
  421 alloc_class chunk1(
  422 
  423     // no save
  424     V16, V16_H, V16_J, V16_K,
  425     V17, V17_H, V17_J, V17_K,
  426     V18, V18_H, V18_J, V18_K,
  427     V19, V19_H, V19_J, V19_K,
  428     V20, V20_H, V20_J, V20_K,
  429     V21, V21_H, V21_J, V21_K,
  430     V22, V22_H, V22_J, V22_K,
  431     V23, V23_H, V23_J, V23_K,
  432     V24, V24_H, V24_J, V24_K,
  433     V25, V25_H, V25_J, V25_K,
  434     V26, V26_H, V26_J, V26_K,
  435     V27, V27_H, V27_J, V27_K,
  436     V28, V28_H, V28_J, V28_K,
  437     V29, V29_H, V29_J, V29_K,
  438     V30, V30_H, V30_J, V30_K,
  439     V31, V31_H, V31_J, V31_K,
  440 
  441     // arg registers
  442     V0, V0_H, V0_J, V0_K,
  443     V1, V1_H, V1_J, V1_K,
  444     V2, V2_H, V2_J, V2_K,
  445     V3, V3_H, V3_J, V3_K,
  446     V4, V4_H, V4_J, V4_K,
  447     V5, V5_H, V5_J, V5_K,
  448     V6, V6_H, V6_J, V6_K,
  449     V7, V7_H, V7_J, V7_K,
  450 
  451     // non-volatiles
  452     V8, V8_H, V8_J, V8_K,
  453     V9, V9_H, V9_J, V9_K,
  454     V10, V10_H, V10_J, V10_K,
  455     V11, V11_H, V11_J, V11_K,
  456     V12, V12_H, V12_J, V12_K,
  457     V13, V13_H, V13_J, V13_K,
  458     V14, V14_H, V14_J, V14_K,
  459     V15, V15_H, V15_J, V15_K,
  460 );
  461 
  462 alloc_class chunk2 (
  463     P0,
  464     P1,
  465     P2,
  466     P3,
  467     P4,
  468     P5,
  469     P6,
  470     P7,
  471 
  472     P8,
  473     P9,
  474     P10,
  475     P11,
  476     P12,
  477     P13,
  478     P14,
  479     P15,
  480 );
  481 
  482 alloc_class chunk3(RFLAGS);
  483 
  484 //----------Architecture Description Register Classes--------------------------
  485 // Several register classes are automatically defined based upon information in
  486 // this architecture description.
  487 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  488 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  489 //
  490 
  491 // Class for all 32 bit general purpose registers
  492 reg_class all_reg32(
  493     R0,
  494     R1,
  495     R2,
  496     R3,
  497     R4,
  498     R5,
  499     R6,
  500     R7,
  501     R10,
  502     R11,
  503     R12,
  504     R13,
  505     R14,
  506     R15,
  507     R16,
  508     R17,
  509     R18,
  510     R19,
  511     R20,
  512     R21,
  513     R22,
  514     R23,
  515     R24,
  516     R25,
  517     R26,
  518     R27,
  519     R28,
  520     R29,
  521     R30,
  522     R31
  523 );
  524 
  525 
  526 // Class for all 32 bit integer registers (excluding SP which
  527 // will never be used as an integer register)
  528 reg_class any_reg32 %{
  529   return _ANY_REG32_mask;
  530 %}
  531 
  532 // Singleton class for R0 int register
  533 reg_class int_r0_reg(R0);
  534 
  535 // Singleton class for R2 int register
  536 reg_class int_r2_reg(R2);
  537 
  538 // Singleton class for R3 int register
  539 reg_class int_r3_reg(R3);
  540 
  541 // Singleton class for R4 int register
  542 reg_class int_r4_reg(R4);
  543 
  544 // Singleton class for R31 int register
  545 reg_class int_r31_reg(R31);
  546 
  547 // Class for all 64 bit general purpose registers
  548 reg_class all_reg(
  549     R0, R0_H,
  550     R1, R1_H,
  551     R2, R2_H,
  552     R3, R3_H,
  553     R4, R4_H,
  554     R5, R5_H,
  555     R6, R6_H,
  556     R7, R7_H,
  557     R10, R10_H,
  558     R11, R11_H,
  559     R12, R12_H,
  560     R13, R13_H,
  561     R14, R14_H,
  562     R15, R15_H,
  563     R16, R16_H,
  564     R17, R17_H,
  565     R18, R18_H,
  566     R19, R19_H,
  567     R20, R20_H,
  568     R21, R21_H,
  569     R22, R22_H,
  570     R23, R23_H,
  571     R24, R24_H,
  572     R25, R25_H,
  573     R26, R26_H,
  574     R27, R27_H,
  575     R28, R28_H,
  576     R29, R29_H,
  577     R30, R30_H,
  578     R31, R31_H
  579 );
  580 
  581 // Class for all long integer registers (including SP)
  582 reg_class any_reg %{
  583   return _ANY_REG_mask;
  584 %}
  585 
  586 // Class for non-allocatable 32 bit registers
  587 reg_class non_allocatable_reg32(
  588 #ifdef R18_RESERVED
  589     // See comment in register_aarch64.hpp
  590     R18,                        // tls on Windows
  591 #endif
  592     R28,                        // thread
  593     R30,                        // lr
  594     R31                         // sp
  595 );
  596 
  597 // Class for non-allocatable 64 bit registers
  598 reg_class non_allocatable_reg(
  599 #ifdef R18_RESERVED
  600     // See comment in register_aarch64.hpp
  601     R18, R18_H,                 // tls on Windows, platform register on macOS
  602 #endif
  603     R28, R28_H,                 // thread
  604     R30, R30_H,                 // lr
  605     R31, R31_H                  // sp
  606 );
  607 
  608 // Class for all non-special integer registers
  609 reg_class no_special_reg32 %{
  610   return _NO_SPECIAL_REG32_mask;
  611 %}
  612 
  613 // Class for all non-special long integer registers
  614 reg_class no_special_reg %{
  615   return _NO_SPECIAL_REG_mask;
  616 %}
  617 
  618 // Class for 64 bit register r0
  619 reg_class r0_reg(
  620     R0, R0_H
  621 );
  622 
  623 // Class for 64 bit register r1
  624 reg_class r1_reg(
  625     R1, R1_H
  626 );
  627 
  628 // Class for 64 bit register r2
  629 reg_class r2_reg(
  630     R2, R2_H
  631 );
  632 
  633 // Class for 64 bit register r3
  634 reg_class r3_reg(
  635     R3, R3_H
  636 );
  637 
  638 // Class for 64 bit register r4
  639 reg_class r4_reg(
  640     R4, R4_H
  641 );
  642 
  643 // Class for 64 bit register r5
  644 reg_class r5_reg(
  645     R5, R5_H
  646 );
  647 
  648 // Class for 64 bit register r10
  649 reg_class r10_reg(
  650     R10, R10_H
  651 );
  652 
  653 // Class for 64 bit register r11
  654 reg_class r11_reg(
  655     R11, R11_H
  656 );
  657 
  658 // Class for method register
  659 reg_class method_reg(
  660     R12, R12_H
  661 );
  662 
  663 // Class for heapbase register
  664 reg_class heapbase_reg(
  665     R27, R27_H
  666 );
  667 
  668 // Class for thread register
  669 reg_class thread_reg(
  670     R28, R28_H
  671 );
  672 
  673 // Class for frame pointer register
  674 reg_class fp_reg(
  675     R29, R29_H
  676 );
  677 
  678 // Class for link register
  679 reg_class lr_reg(
  680     R30, R30_H
  681 );
  682 
  683 // Class for long sp register
  684 reg_class sp_reg(
  685   R31, R31_H
  686 );
  687 
  688 // Class for all pointer registers
  689 reg_class ptr_reg %{
  690   return _PTR_REG_mask;
  691 %}
  692 
  693 // Class for all non_special pointer registers
  694 reg_class no_special_ptr_reg %{
  695   return _NO_SPECIAL_PTR_REG_mask;
  696 %}
  697 
  698 // Class for all float registers
  699 reg_class float_reg(
  700     V0,
  701     V1,
  702     V2,
  703     V3,
  704     V4,
  705     V5,
  706     V6,
  707     V7,
  708     V8,
  709     V9,
  710     V10,
  711     V11,
  712     V12,
  713     V13,
  714     V14,
  715     V15,
  716     V16,
  717     V17,
  718     V18,
  719     V19,
  720     V20,
  721     V21,
  722     V22,
  723     V23,
  724     V24,
  725     V25,
  726     V26,
  727     V27,
  728     V28,
  729     V29,
  730     V30,
  731     V31
  732 );
  733 
  734 // Double precision float registers have virtual `high halves' that
  735 // are needed by the allocator.
  736 // Class for all double registers
  737 reg_class double_reg(
  738     V0, V0_H,
  739     V1, V1_H,
  740     V2, V2_H,
  741     V3, V3_H,
  742     V4, V4_H,
  743     V5, V5_H,
  744     V6, V6_H,
  745     V7, V7_H,
  746     V8, V8_H,
  747     V9, V9_H,
  748     V10, V10_H,
  749     V11, V11_H,
  750     V12, V12_H,
  751     V13, V13_H,
  752     V14, V14_H,
  753     V15, V15_H,
  754     V16, V16_H,
  755     V17, V17_H,
  756     V18, V18_H,
  757     V19, V19_H,
  758     V20, V20_H,
  759     V21, V21_H,
  760     V22, V22_H,
  761     V23, V23_H,
  762     V24, V24_H,
  763     V25, V25_H,
  764     V26, V26_H,
  765     V27, V27_H,
  766     V28, V28_H,
  767     V29, V29_H,
  768     V30, V30_H,
  769     V31, V31_H
  770 );
  771 
  772 // Class for all SVE vector registers.
  773 reg_class vectora_reg (
  774     V0, V0_H, V0_J, V0_K,
  775     V1, V1_H, V1_J, V1_K,
  776     V2, V2_H, V2_J, V2_K,
  777     V3, V3_H, V3_J, V3_K,
  778     V4, V4_H, V4_J, V4_K,
  779     V5, V5_H, V5_J, V5_K,
  780     V6, V6_H, V6_J, V6_K,
  781     V7, V7_H, V7_J, V7_K,
  782     V8, V8_H, V8_J, V8_K,
  783     V9, V9_H, V9_J, V9_K,
  784     V10, V10_H, V10_J, V10_K,
  785     V11, V11_H, V11_J, V11_K,
  786     V12, V12_H, V12_J, V12_K,
  787     V13, V13_H, V13_J, V13_K,
  788     V14, V14_H, V14_J, V14_K,
  789     V15, V15_H, V15_J, V15_K,
  790     V16, V16_H, V16_J, V16_K,
  791     V17, V17_H, V17_J, V17_K,
  792     V18, V18_H, V18_J, V18_K,
  793     V19, V19_H, V19_J, V19_K,
  794     V20, V20_H, V20_J, V20_K,
  795     V21, V21_H, V21_J, V21_K,
  796     V22, V22_H, V22_J, V22_K,
  797     V23, V23_H, V23_J, V23_K,
  798     V24, V24_H, V24_J, V24_K,
  799     V25, V25_H, V25_J, V25_K,
  800     V26, V26_H, V26_J, V26_K,
  801     V27, V27_H, V27_J, V27_K,
  802     V28, V28_H, V28_J, V28_K,
  803     V29, V29_H, V29_J, V29_K,
  804     V30, V30_H, V30_J, V30_K,
  805     V31, V31_H, V31_J, V31_K,
  806 );
  807 
  808 // Class for all 64bit vector registers
  809 reg_class vectord_reg(
  810     V0, V0_H,
  811     V1, V1_H,
  812     V2, V2_H,
  813     V3, V3_H,
  814     V4, V4_H,
  815     V5, V5_H,
  816     V6, V6_H,
  817     V7, V7_H,
  818     V8, V8_H,
  819     V9, V9_H,
  820     V10, V10_H,
  821     V11, V11_H,
  822     V12, V12_H,
  823     V13, V13_H,
  824     V14, V14_H,
  825     V15, V15_H,
  826     V16, V16_H,
  827     V17, V17_H,
  828     V18, V18_H,
  829     V19, V19_H,
  830     V20, V20_H,
  831     V21, V21_H,
  832     V22, V22_H,
  833     V23, V23_H,
  834     V24, V24_H,
  835     V25, V25_H,
  836     V26, V26_H,
  837     V27, V27_H,
  838     V28, V28_H,
  839     V29, V29_H,
  840     V30, V30_H,
  841     V31, V31_H
  842 );
  843 
  844 // Class for all 128bit vector registers
  845 reg_class vectorx_reg(
  846     V0, V0_H, V0_J, V0_K,
  847     V1, V1_H, V1_J, V1_K,
  848     V2, V2_H, V2_J, V2_K,
  849     V3, V3_H, V3_J, V3_K,
  850     V4, V4_H, V4_J, V4_K,
  851     V5, V5_H, V5_J, V5_K,
  852     V6, V6_H, V6_J, V6_K,
  853     V7, V7_H, V7_J, V7_K,
  854     V8, V8_H, V8_J, V8_K,
  855     V9, V9_H, V9_J, V9_K,
  856     V10, V10_H, V10_J, V10_K,
  857     V11, V11_H, V11_J, V11_K,
  858     V12, V12_H, V12_J, V12_K,
  859     V13, V13_H, V13_J, V13_K,
  860     V14, V14_H, V14_J, V14_K,
  861     V15, V15_H, V15_J, V15_K,
  862     V16, V16_H, V16_J, V16_K,
  863     V17, V17_H, V17_J, V17_K,
  864     V18, V18_H, V18_J, V18_K,
  865     V19, V19_H, V19_J, V19_K,
  866     V20, V20_H, V20_J, V20_K,
  867     V21, V21_H, V21_J, V21_K,
  868     V22, V22_H, V22_J, V22_K,
  869     V23, V23_H, V23_J, V23_K,
  870     V24, V24_H, V24_J, V24_K,
  871     V25, V25_H, V25_J, V25_K,
  872     V26, V26_H, V26_J, V26_K,
  873     V27, V27_H, V27_J, V27_K,
  874     V28, V28_H, V28_J, V28_K,
  875     V29, V29_H, V29_J, V29_K,
  876     V30, V30_H, V30_J, V30_K,
  877     V31, V31_H, V31_J, V31_K
  878 );
  879 
  880 // Class for 128 bit register v0
  881 reg_class v0_reg(
  882     V0, V0_H
  883 );
  884 
  885 // Class for 128 bit register v1
  886 reg_class v1_reg(
  887     V1, V1_H
  888 );
  889 
  890 // Class for 128 bit register v2
  891 reg_class v2_reg(
  892     V2, V2_H
  893 );
  894 
  895 // Class for 128 bit register v3
  896 reg_class v3_reg(
  897     V3, V3_H
  898 );
  899 
  900 // Class for 128 bit register v4
  901 reg_class v4_reg(
  902     V4, V4_H
  903 );
  904 
  905 // Class for 128 bit register v5
  906 reg_class v5_reg(
  907     V5, V5_H
  908 );
  909 
  910 // Class for 128 bit register v6
  911 reg_class v6_reg(
  912     V6, V6_H
  913 );
  914 
  915 // Class for 128 bit register v7
  916 reg_class v7_reg(
  917     V7, V7_H
  918 );
  919 
  920 // Class for 128 bit register v8
  921 reg_class v8_reg(
  922     V8, V8_H
  923 );
  924 
  925 // Class for 128 bit register v9
  926 reg_class v9_reg(
  927     V9, V9_H
  928 );
  929 
  930 // Class for 128 bit register v10
  931 reg_class v10_reg(
  932     V10, V10_H
  933 );
  934 
  935 // Class for 128 bit register v11
  936 reg_class v11_reg(
  937     V11, V11_H
  938 );
  939 
  940 // Class for 128 bit register v12
  941 reg_class v12_reg(
  942     V12, V12_H
  943 );
  944 
  945 // Class for 128 bit register v13
  946 reg_class v13_reg(
  947     V13, V13_H
  948 );
  949 
  950 // Class for 128 bit register v14
  951 reg_class v14_reg(
  952     V14, V14_H
  953 );
  954 
  955 // Class for 128 bit register v15
  956 reg_class v15_reg(
  957     V15, V15_H
  958 );
  959 
  960 // Class for 128 bit register v16
  961 reg_class v16_reg(
  962     V16, V16_H
  963 );
  964 
  965 // Class for 128 bit register v17
  966 reg_class v17_reg(
  967     V17, V17_H
  968 );
  969 
  970 // Class for 128 bit register v18
  971 reg_class v18_reg(
  972     V18, V18_H
  973 );
  974 
  975 // Class for 128 bit register v19
  976 reg_class v19_reg(
  977     V19, V19_H
  978 );
  979 
  980 // Class for 128 bit register v20
  981 reg_class v20_reg(
  982     V20, V20_H
  983 );
  984 
  985 // Class for 128 bit register v21
  986 reg_class v21_reg(
  987     V21, V21_H
  988 );
  989 
  990 // Class for 128 bit register v22
  991 reg_class v22_reg(
  992     V22, V22_H
  993 );
  994 
  995 // Class for 128 bit register v23
  996 reg_class v23_reg(
  997     V23, V23_H
  998 );
  999 
 1000 // Class for 128 bit register v24
 1001 reg_class v24_reg(
 1002     V24, V24_H
 1003 );
 1004 
 1005 // Class for 128 bit register v25
 1006 reg_class v25_reg(
 1007     V25, V25_H
 1008 );
 1009 
 1010 // Class for 128 bit register v26
 1011 reg_class v26_reg(
 1012     V26, V26_H
 1013 );
 1014 
 1015 // Class for 128 bit register v27
 1016 reg_class v27_reg(
 1017     V27, V27_H
 1018 );
 1019 
 1020 // Class for 128 bit register v28
 1021 reg_class v28_reg(
 1022     V28, V28_H
 1023 );
 1024 
 1025 // Class for 128 bit register v29
 1026 reg_class v29_reg(
 1027     V29, V29_H
 1028 );
 1029 
 1030 // Class for 128 bit register v30
 1031 reg_class v30_reg(
 1032     V30, V30_H
 1033 );
 1034 
 1035 // Class for 128 bit register v31
 1036 reg_class v31_reg(
 1037     V31, V31_H
 1038 );
 1039 
 1040 // Class for all SVE predicate registers.
 1041 reg_class pr_reg (
 1042     P0,
 1043     P1,
 1044     P2,
 1045     P3,
 1046     P4,
 1047     P5,
 1048     P6,
 1049     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1050     P8,
 1051     P9,
 1052     P10,
 1053     P11,
 1054     P12,
 1055     P13,
 1056     P14,
 1057     P15
 1058 );
 1059 
 1060 // Class for SVE governing predicate registers, which are used
 1061 // to determine the active elements of a predicated instruction.
 1062 reg_class gov_pr (
 1063     P0,
 1064     P1,
 1065     P2,
 1066     P3,
 1067     P4,
 1068     P5,
 1069     P6,
 1070     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1071 );
 1072 
 1073 reg_class p0_reg(P0);
 1074 reg_class p1_reg(P1);
 1075 
 1076 // Singleton class for condition codes
 1077 reg_class int_flags(RFLAGS);
 1078 
 1079 %}
 1080 
 1081 //----------DEFINITION BLOCK---------------------------------------------------
 1082 // Define name --> value mappings to inform the ADLC of an integer valued name
 1083 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1084 // Format:
 1085 //        int_def  <name>         ( <int_value>, <expression>);
 1086 // Generated Code in ad_<arch>.hpp
 1087 //        #define  <name>   (<expression>)
 1088 //        // value == <int_value>
 1089 // Generated code in ad_<arch>.cpp adlc_verification()
 1090 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1091 //
 1092 
 1093 // we follow the ppc-aix port in using a simple cost model which ranks
 1094 // register operations as cheap, memory ops as more expensive and
 1095 // branches as most expensive. the first two have a low as well as a
 1096 // normal cost. huge cost appears to be a way of saying don't do
 1097 // something
 1098 
 1099 definitions %{
 1100   // The default cost (of a register move instruction).
 1101   int_def INSN_COST            (    100,     100);
 1102   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1103   int_def CALL_COST            (    200,     2 * INSN_COST);
 1104   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1105 %}
 1106 
 1107 
 1108 //----------SOURCE BLOCK-------------------------------------------------------
 1109 // This is a block of C++ code which provides values, functions, and
 1110 // definitions necessary in the rest of the architecture description
 1111 
 1112 source_hpp %{
 1113 
 1114 #include "asm/macroAssembler.hpp"
 1115 #include "gc/shared/barrierSetAssembler.hpp"
 1116 #include "gc/shared/cardTable.hpp"
 1117 #include "gc/shared/cardTableBarrierSet.hpp"
 1118 #include "gc/shared/collectedHeap.hpp"
 1119 #include "opto/addnode.hpp"
 1120 #include "opto/convertnode.hpp"
 1121 #include "runtime/objectMonitor.hpp"
 1122 
 1123 extern RegMask _ANY_REG32_mask;
 1124 extern RegMask _ANY_REG_mask;
 1125 extern RegMask _PTR_REG_mask;
 1126 extern RegMask _NO_SPECIAL_REG32_mask;
 1127 extern RegMask _NO_SPECIAL_REG_mask;
 1128 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1129 
 1130 class CallStubImpl {
 1131 
 1132   //--------------------------------------------------------------
 1133   //---<  Used for optimization in Compile::shorten_branches  >---
 1134   //--------------------------------------------------------------
 1135 
 1136  public:
 1137   // Size of call trampoline stub.
 1138   static uint size_call_trampoline() {
 1139     return 0; // no call trampolines on this platform
 1140   }
 1141 
 1142   // number of relocations needed by a call trampoline stub
 1143   static uint reloc_call_trampoline() {
 1144     return 0; // no call trampolines on this platform
 1145   }
 1146 };
 1147 
 1148 class HandlerImpl {
 1149 
 1150  public:
 1151 
 1152   static int emit_exception_handler(CodeBuffer &cbuf);
 1153   static int emit_deopt_handler(CodeBuffer& cbuf);
 1154 
 1155   static uint size_exception_handler() {
 1156     return MacroAssembler::far_codestub_branch_size();
 1157   }
 1158 
 1159   static uint size_deopt_handler() {
 1160     // count one adr and one far branch instruction
 1161     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1162   }
 1163 };
 1164 
 1165 class Node::PD {
 1166 public:
 1167   enum NodeFlags {
 1168     _last_flag = Node::_last_flag
 1169   };
 1170 };
 1171 
 1172   bool is_CAS(int opcode, bool maybe_volatile);
 1173 
 1174   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1175 
 1176   bool unnecessary_acquire(const Node *barrier);
 1177   bool needs_acquiring_load(const Node *load);
 1178 
 1179   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1180 
 1181   bool unnecessary_release(const Node *barrier);
 1182   bool unnecessary_volatile(const Node *barrier);
 1183   bool needs_releasing_store(const Node *store);
 1184 
 1185   // predicate controlling translation of CompareAndSwapX
 1186   bool needs_acquiring_load_exclusive(const Node *load);
 1187 
 1188   // predicate controlling addressing modes
 1189   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1190 %}
 1191 
 1192 source %{
 1193 
 1194   // Derived RegMask with conditionally allocatable registers
 1195 
 1196   void PhaseOutput::pd_perform_mach_node_analysis() {
 1197   }
 1198 
 1199   int MachNode::pd_alignment_required() const {
 1200     return 1;
 1201   }
 1202 
 1203   int MachNode::compute_padding(int current_offset) const {
 1204     return 0;
 1205   }
 1206 
 1207   RegMask _ANY_REG32_mask;
 1208   RegMask _ANY_REG_mask;
 1209   RegMask _PTR_REG_mask;
 1210   RegMask _NO_SPECIAL_REG32_mask;
 1211   RegMask _NO_SPECIAL_REG_mask;
 1212   RegMask _NO_SPECIAL_PTR_REG_mask;
 1213 
 1214   void reg_mask_init() {
 1215     // We derive below RegMask(s) from the ones which are auto-generated from
 1216     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1217     // registers conditionally reserved.
 1218 
 1219     _ANY_REG32_mask = _ALL_REG32_mask;
 1220     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1221 
 1222     _ANY_REG_mask = _ALL_REG_mask;
 1223 
 1224     _PTR_REG_mask = _ALL_REG_mask;
 1225 
 1226     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1227     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1228 
 1229     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1230     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1231 
 1232     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1233     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1234 
 1235     // r27 is not allocatable when compressed oops is on and heapbase is not
 1236     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1237     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1238       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1239       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1240       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1241     }
 1242 
 1243     // r29 is not allocatable when PreserveFramePointer is on
 1244     if (PreserveFramePointer) {
 1245       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1246       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1247       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1248     }
 1249   }
 1250 
 1251   // Optimizaton of volatile gets and puts
 1252   // -------------------------------------
 1253   //
 1254   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1255   // use to implement volatile reads and writes. For a volatile read
 1256   // we simply need
 1257   //
 1258   //   ldar<x>
 1259   //
 1260   // and for a volatile write we need
 1261   //
 1262   //   stlr<x>
 1263   //
 1264   // Alternatively, we can implement them by pairing a normal
 1265   // load/store with a memory barrier. For a volatile read we need
 1266   //
 1267   //   ldr<x>
 1268   //   dmb ishld
 1269   //
 1270   // for a volatile write
 1271   //
 1272   //   dmb ish
 1273   //   str<x>
 1274   //   dmb ish
 1275   //
 1276   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1277   // sequences. These are normally translated to an instruction
 1278   // sequence like the following
 1279   //
 1280   //   dmb      ish
 1281   // retry:
 1282   //   ldxr<x>   rval raddr
 1283   //   cmp       rval rold
 1284   //   b.ne done
 1285   //   stlxr<x>  rval, rnew, rold
 1286   //   cbnz      rval retry
 1287   // done:
 1288   //   cset      r0, eq
 1289   //   dmb ishld
 1290   //
 1291   // Note that the exclusive store is already using an stlxr
 1292   // instruction. That is required to ensure visibility to other
 1293   // threads of the exclusive write (assuming it succeeds) before that
 1294   // of any subsequent writes.
 1295   //
 1296   // The following instruction sequence is an improvement on the above
 1297   //
 1298   // retry:
 1299   //   ldaxr<x>  rval raddr
 1300   //   cmp       rval rold
 1301   //   b.ne done
 1302   //   stlxr<x>  rval, rnew, rold
 1303   //   cbnz      rval retry
 1304   // done:
 1305   //   cset      r0, eq
 1306   //
 1307   // We don't need the leading dmb ish since the stlxr guarantees
 1308   // visibility of prior writes in the case that the swap is
 1309   // successful. Crucially we don't have to worry about the case where
 1310   // the swap is not successful since no valid program should be
 1311   // relying on visibility of prior changes by the attempting thread
 1312   // in the case where the CAS fails.
 1313   //
 1314   // Similarly, we don't need the trailing dmb ishld if we substitute
 1315   // an ldaxr instruction since that will provide all the guarantees we
 1316   // require regarding observation of changes made by other threads
 1317   // before any change to the CAS address observed by the load.
 1318   //
 1319   // In order to generate the desired instruction sequence we need to
 1320   // be able to identify specific 'signature' ideal graph node
 1321   // sequences which i) occur as a translation of a volatile reads or
 1322   // writes or CAS operations and ii) do not occur through any other
 1323   // translation or graph transformation. We can then provide
 1324   // alternative aldc matching rules which translate these node
 1325   // sequences to the desired machine code sequences. Selection of the
 1326   // alternative rules can be implemented by predicates which identify
 1327   // the relevant node sequences.
 1328   //
 1329   // The ideal graph generator translates a volatile read to the node
 1330   // sequence
 1331   //
 1332   //   LoadX[mo_acquire]
 1333   //   MemBarAcquire
 1334   //
 1335   // As a special case when using the compressed oops optimization we
 1336   // may also see this variant
 1337   //
 1338   //   LoadN[mo_acquire]
 1339   //   DecodeN
 1340   //   MemBarAcquire
 1341   //
 1342   // A volatile write is translated to the node sequence
 1343   //
 1344   //   MemBarRelease
 1345   //   StoreX[mo_release] {CardMark}-optional
 1346   //   MemBarVolatile
 1347   //
 1348   // n.b. the above node patterns are generated with a strict
 1349   // 'signature' configuration of input and output dependencies (see
 1350   // the predicates below for exact details). The card mark may be as
 1351   // simple as a few extra nodes or, in a few GC configurations, may
 1352   // include more complex control flow between the leading and
 1353   // trailing memory barriers. However, whatever the card mark
 1354   // configuration these signatures are unique to translated volatile
 1355   // reads/stores -- they will not appear as a result of any other
 1356   // bytecode translation or inlining nor as a consequence of
 1357   // optimizing transforms.
 1358   //
 1359   // We also want to catch inlined unsafe volatile gets and puts and
 1360   // be able to implement them using either ldar<x>/stlr<x> or some
 1361   // combination of ldr<x>/stlr<x> and dmb instructions.
 1362   //
 1363   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1364   // normal volatile put node sequence containing an extra cpuorder
 1365   // membar
 1366   //
 1367   //   MemBarRelease
 1368   //   MemBarCPUOrder
 1369   //   StoreX[mo_release] {CardMark}-optional
 1370   //   MemBarCPUOrder
 1371   //   MemBarVolatile
 1372   //
 1373   // n.b. as an aside, a cpuorder membar is not itself subject to
 1374   // matching and translation by adlc rules.  However, the rule
 1375   // predicates need to detect its presence in order to correctly
 1376   // select the desired adlc rules.
 1377   //
 1378   // Inlined unsafe volatile gets manifest as a slightly different
 1379   // node sequence to a normal volatile get because of the
 1380   // introduction of some CPUOrder memory barriers to bracket the
 1381   // Load. However, but the same basic skeleton of a LoadX feeding a
 1382   // MemBarAcquire, possibly through an optional DecodeN, is still
 1383   // present
 1384   //
 1385   //   MemBarCPUOrder
 1386   //        ||       \\
 1387   //   MemBarCPUOrder LoadX[mo_acquire]
 1388   //        ||            |
 1389   //        ||       {DecodeN} optional
 1390   //        ||       /
 1391   //     MemBarAcquire
 1392   //
 1393   // In this case the acquire membar does not directly depend on the
 1394   // load. However, we can be sure that the load is generated from an
 1395   // inlined unsafe volatile get if we see it dependent on this unique
 1396   // sequence of membar nodes. Similarly, given an acquire membar we
 1397   // can know that it was added because of an inlined unsafe volatile
 1398   // get if it is fed and feeds a cpuorder membar and if its feed
 1399   // membar also feeds an acquiring load.
 1400   //
 1401   // Finally an inlined (Unsafe) CAS operation is translated to the
 1402   // following ideal graph
 1403   //
 1404   //   MemBarRelease
 1405   //   MemBarCPUOrder
 1406   //   CompareAndSwapX {CardMark}-optional
 1407   //   MemBarCPUOrder
 1408   //   MemBarAcquire
 1409   //
 1410   // So, where we can identify these volatile read and write
 1411   // signatures we can choose to plant either of the above two code
 1412   // sequences. For a volatile read we can simply plant a normal
 1413   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1414   // also choose to inhibit translation of the MemBarAcquire and
 1415   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1416   //
 1417   // When we recognise a volatile store signature we can choose to
 1418   // plant at a dmb ish as a translation for the MemBarRelease, a
 1419   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1420   // Alternatively, we can inhibit translation of the MemBarRelease
 1421   // and MemBarVolatile and instead plant a simple stlr<x>
 1422   // instruction.
 1423   //
 1424   // when we recognise a CAS signature we can choose to plant a dmb
 1425   // ish as a translation for the MemBarRelease, the conventional
 1426   // macro-instruction sequence for the CompareAndSwap node (which
 1427   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1428   // Alternatively, we can elide generation of the dmb instructions
 1429   // and plant the alternative CompareAndSwap macro-instruction
 1430   // sequence (which uses ldaxr<x>).
 1431   //
 1432   // Of course, the above only applies when we see these signature
 1433   // configurations. We still want to plant dmb instructions in any
 1434   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1435   // MemBarVolatile. For example, at the end of a constructor which
 1436   // writes final/volatile fields we will see a MemBarRelease
 1437   // instruction and this needs a 'dmb ish' lest we risk the
 1438   // constructed object being visible without making the
 1439   // final/volatile field writes visible.
 1440   //
 1441   // n.b. the translation rules below which rely on detection of the
 1442   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1443   // If we see anything other than the signature configurations we
 1444   // always just translate the loads and stores to ldr<x> and str<x>
 1445   // and translate acquire, release and volatile membars to the
 1446   // relevant dmb instructions.
 1447   //
 1448 
 1449   // is_CAS(int opcode, bool maybe_volatile)
 1450   //
 1451   // return true if opcode is one of the possible CompareAndSwapX
 1452   // values otherwise false.
 1453 
 1454   bool is_CAS(int opcode, bool maybe_volatile)
 1455   {
 1456     switch(opcode) {
 1457       // We handle these
 1458     case Op_CompareAndSwapI:
 1459     case Op_CompareAndSwapL:
 1460     case Op_CompareAndSwapP:
 1461     case Op_CompareAndSwapN:
 1462     case Op_ShenandoahCompareAndSwapP:
 1463     case Op_ShenandoahCompareAndSwapN:
 1464     case Op_CompareAndSwapB:
 1465     case Op_CompareAndSwapS:
 1466     case Op_GetAndSetI:
 1467     case Op_GetAndSetL:
 1468     case Op_GetAndSetP:
 1469     case Op_GetAndSetN:
 1470     case Op_GetAndAddI:
 1471     case Op_GetAndAddL:
 1472       return true;
 1473     case Op_CompareAndExchangeI:
 1474     case Op_CompareAndExchangeN:
 1475     case Op_CompareAndExchangeB:
 1476     case Op_CompareAndExchangeS:
 1477     case Op_CompareAndExchangeL:
 1478     case Op_CompareAndExchangeP:
 1479     case Op_WeakCompareAndSwapB:
 1480     case Op_WeakCompareAndSwapS:
 1481     case Op_WeakCompareAndSwapI:
 1482     case Op_WeakCompareAndSwapL:
 1483     case Op_WeakCompareAndSwapP:
 1484     case Op_WeakCompareAndSwapN:
 1485     case Op_ShenandoahWeakCompareAndSwapP:
 1486     case Op_ShenandoahWeakCompareAndSwapN:
 1487     case Op_ShenandoahCompareAndExchangeP:
 1488     case Op_ShenandoahCompareAndExchangeN:
 1489       return maybe_volatile;
 1490     default:
 1491       return false;
 1492     }
 1493   }
 1494 
 1495   // helper to determine the maximum number of Phi nodes we may need to
 1496   // traverse when searching from a card mark membar for the merge mem
 1497   // feeding a trailing membar or vice versa
 1498 
 1499 // predicates controlling emit of ldr<x>/ldar<x>
 1500 
 1501 bool unnecessary_acquire(const Node *barrier)
 1502 {
 1503   assert(barrier->is_MemBar(), "expecting a membar");
 1504 
 1505   MemBarNode* mb = barrier->as_MemBar();
 1506 
 1507   if (mb->trailing_load()) {
 1508     return true;
 1509   }
 1510 
 1511   if (mb->trailing_load_store()) {
 1512     Node* load_store = mb->in(MemBarNode::Precedent);
 1513     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1514     return is_CAS(load_store->Opcode(), true);
 1515   }
 1516 
 1517   return false;
 1518 }
 1519 
 1520 bool needs_acquiring_load(const Node *n)
 1521 {
 1522   assert(n->is_Load(), "expecting a load");
 1523   LoadNode *ld = n->as_Load();
 1524   return ld->is_acquire();
 1525 }
 1526 
 1527 bool unnecessary_release(const Node *n)
 1528 {
 1529   assert((n->is_MemBar() &&
 1530           n->Opcode() == Op_MemBarRelease),
 1531          "expecting a release membar");
 1532 
 1533   MemBarNode *barrier = n->as_MemBar();
 1534   if (!barrier->leading()) {
 1535     return false;
 1536   } else {
 1537     Node* trailing = barrier->trailing_membar();
 1538     MemBarNode* trailing_mb = trailing->as_MemBar();
 1539     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1540     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1541 
 1542     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1543     if (mem->is_Store()) {
 1544       assert(mem->as_Store()->is_release(), "");
 1545       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1546       return true;
 1547     } else {
 1548       assert(mem->is_LoadStore(), "");
 1549       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1550       return is_CAS(mem->Opcode(), true);
 1551     }
 1552   }
 1553   return false;
 1554 }
 1555 
 1556 bool unnecessary_volatile(const Node *n)
 1557 {
 1558   // assert n->is_MemBar();
 1559   MemBarNode *mbvol = n->as_MemBar();
 1560 
 1561   bool release = mbvol->trailing_store();
 1562   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1563 #ifdef ASSERT
 1564   if (release) {
 1565     Node* leading = mbvol->leading_membar();
 1566     assert(leading->Opcode() == Op_MemBarRelease, "");
 1567     assert(leading->as_MemBar()->leading_store(), "");
 1568     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1569   }
 1570 #endif
 1571 
 1572   return release;
 1573 }
 1574 
 1575 // predicates controlling emit of str<x>/stlr<x>
 1576 
 1577 bool needs_releasing_store(const Node *n)
 1578 {
 1579   // assert n->is_Store();
 1580   StoreNode *st = n->as_Store();
 1581   return st->trailing_membar() != NULL;
 1582 }
 1583 
 1584 // predicate controlling translation of CAS
 1585 //
 1586 // returns true if CAS needs to use an acquiring load otherwise false
 1587 
 1588 bool needs_acquiring_load_exclusive(const Node *n)
 1589 {
 1590   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1591   LoadStoreNode* ldst = n->as_LoadStore();
 1592   if (is_CAS(n->Opcode(), false)) {
 1593     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1594   } else {
 1595     return ldst->trailing_membar() != NULL;
 1596   }
 1597 
 1598   // so we can just return true here
 1599   return true;
 1600 }
 1601 
 1602 #define __ _masm.
 1603 
 1604 // advance declarations for helper functions to convert register
 1605 // indices to register objects
 1606 
 1607 // the ad file has to provide implementations of certain methods
 1608 // expected by the generic code
 1609 //
 1610 // REQUIRED FUNCTIONALITY
 1611 
 1612 //=============================================================================
 1613 
 1614 // !!!!! Special hack to get all types of calls to specify the byte offset
 1615 //       from the start of the call to the point where the return address
 1616 //       will point.
 1617 
 1618 int MachCallStaticJavaNode::ret_addr_offset()
 1619 {
 1620   // call should be a simple bl
 1621   int off = 4;
 1622   return off;
 1623 }
 1624 
 1625 int MachCallDynamicJavaNode::ret_addr_offset()
 1626 {
 1627   return 16; // movz, movk, movk, bl
 1628 }
 1629 
 1630 int MachCallRuntimeNode::ret_addr_offset() {
 1631   // for generated stubs the call will be
 1632   //   bl(addr)
 1633   // or with far branches
 1634   //   bl(trampoline_stub)
 1635   // for real runtime callouts it will be six instructions
 1636   // see aarch64_enc_java_to_runtime
 1637   //   adr(rscratch2, retaddr)
 1638   //   lea(rscratch1, RuntimeAddress(addr)
 1639   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1640   //   blr(rscratch1)
 1641   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1642   if (cb) {
 1643     return 1 * NativeInstruction::instruction_size;
 1644   } else {
 1645     return 6 * NativeInstruction::instruction_size;
 1646   }
 1647 }
 1648 
 1649 //=============================================================================
 1650 
 1651 #ifndef PRODUCT
 1652 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1653   st->print("BREAKPOINT");
 1654 }
 1655 #endif
 1656 
 1657 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1658   C2_MacroAssembler _masm(&cbuf);
 1659   __ brk(0);
 1660 }
 1661 
 1662 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1663   return MachNode::size(ra_);
 1664 }
 1665 
 1666 //=============================================================================
 1667 
 1668 #ifndef PRODUCT
 1669   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1670     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1671   }
 1672 #endif
 1673 
 1674   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1675     C2_MacroAssembler _masm(&cbuf);
 1676     for (int i = 0; i < _count; i++) {
 1677       __ nop();
 1678     }
 1679   }
 1680 
 1681   uint MachNopNode::size(PhaseRegAlloc*) const {
 1682     return _count * NativeInstruction::instruction_size;
 1683   }
 1684 
 1685 //=============================================================================
 1686 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1687 
 1688 int ConstantTable::calculate_table_base_offset() const {
 1689   return 0;  // absolute addressing, no offset
 1690 }
 1691 
 1692 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1693 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1694   ShouldNotReachHere();
 1695 }
 1696 
 1697 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1698   // Empty encoding
 1699 }
 1700 
 1701 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1702   return 0;
 1703 }
 1704 
 1705 #ifndef PRODUCT
 1706 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1707   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1708 }
 1709 #endif
 1710 
 1711 #ifndef PRODUCT
 1712 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1713   Compile* C = ra_->C;
 1714 
 1715   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1716 
 1717   if (C->output()->need_stack_bang(framesize))
 1718     st->print("# stack bang size=%d\n\t", framesize);
 1719 
 1720   if (VM_Version::use_rop_protection()) {
 1721     st->print("ldr zr, [lr]\n\t");
 1722     st->print("pacia  lr, rfp\n\t");
 1723   }
 1724   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1725     st->print("sub  sp, sp, #%d\n\t", framesize);
 1726     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1727     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1728   } else {
 1729     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1730     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1731     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1732     st->print("sub  sp, sp, rscratch1");
 1733   }
 1734   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1735     st->print("\n\t");
 1736     st->print("ldr  rscratch1, [guard]\n\t");
 1737     st->print("dmb ishld\n\t");
 1738     st->print("ldr  rscratch2, [rthread, #thread_disarmed_offset]\n\t");
 1739     st->print("cmp  rscratch1, rscratch2\n\t");
 1740     st->print("b.eq skip");
 1741     st->print("\n\t");
 1742     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1743     st->print("b skip\n\t");
 1744     st->print("guard: int\n\t");
 1745     st->print("\n\t");
 1746     st->print("skip:\n\t");
 1747   }
 1748 }
 1749 #endif
 1750 
 1751 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1752   Compile* C = ra_->C;
 1753   C2_MacroAssembler _masm(&cbuf);
 1754 
 1755   // n.b. frame size includes space for return pc and rfp
 1756   const int framesize = C->output()->frame_size_in_bytes();
 1757 
 1758   // insert a nop at the start of the prolog so we can patch in a
 1759   // branch if we need to invalidate the method later
 1760   __ nop();
 1761 
 1762   if (C->clinit_barrier_on_entry()) {
 1763     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1764 
 1765     Label L_skip_barrier;
 1766 
 1767     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1768     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1769     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1770     __ bind(L_skip_barrier);
 1771   }
 1772 
 1773   if (C->max_vector_size() > 0) {
 1774     __ reinitialize_ptrue();
 1775   }
 1776 
 1777   int bangsize = C->output()->bang_size_in_bytes();
 1778   if (C->output()->need_stack_bang(bangsize))
 1779     __ generate_stack_overflow_check(bangsize);
 1780 
 1781   __ build_frame(framesize);
 1782 
 1783   if (C->stub_function() == NULL) {
 1784     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1785     if (BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1786       // Dummy labels for just measuring the code size
 1787       Label dummy_slow_path;
 1788       Label dummy_continuation;
 1789       Label dummy_guard;
 1790       Label* slow_path = &dummy_slow_path;
 1791       Label* continuation = &dummy_continuation;
 1792       Label* guard = &dummy_guard;
 1793       if (!Compile::current()->output()->in_scratch_emit_size()) {
 1794         // Use real labels from actual stub when not emitting code for the purpose of measuring its size
 1795         C2EntryBarrierStub* stub = Compile::current()->output()->entry_barrier_table()->add_entry_barrier();
 1796         slow_path = &stub->slow_path();
 1797         continuation = &stub->continuation();
 1798         guard = &stub->guard();
 1799       }
 1800       // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
 1801       bs->nmethod_entry_barrier(&_masm, slow_path, continuation, guard);
 1802     }
 1803   }
 1804 
 1805   if (VerifyStackAtCalls) {
 1806     Unimplemented();
 1807   }
 1808 
 1809   C->output()->set_frame_complete(cbuf.insts_size());
 1810 
 1811   if (C->has_mach_constant_base_node()) {
 1812     // NOTE: We set the table base offset here because users might be
 1813     // emitted before MachConstantBaseNode.
 1814     ConstantTable& constant_table = C->output()->constant_table();
 1815     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1816   }
 1817 }
 1818 
 1819 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1820 {
 1821   return MachNode::size(ra_); // too many variables; just compute it
 1822                               // the hard way
 1823 }
 1824 
 1825 int MachPrologNode::reloc() const
 1826 {
 1827   return 0;
 1828 }
 1829 
 1830 //=============================================================================
 1831 
 1832 #ifndef PRODUCT
 1833 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1834   Compile* C = ra_->C;
 1835   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1836 
 1837   st->print("# pop frame %d\n\t",framesize);
 1838 
 1839   if (framesize == 0) {
 1840     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1841   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1842     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1843     st->print("add  sp, sp, #%d\n\t", framesize);
 1844   } else {
 1845     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1846     st->print("add  sp, sp, rscratch1\n\t");
 1847     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1848   }
 1849   if (VM_Version::use_rop_protection()) {
 1850     st->print("autia lr, rfp\n\t");
 1851     st->print("ldr zr, [lr]\n\t");
 1852   }
 1853 
 1854   if (do_polling() && C->is_method_compilation()) {
 1855     st->print("# test polling word\n\t");
 1856     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1857     st->print("cmp  sp, rscratch1\n\t");
 1858     st->print("bhi #slow_path");
 1859   }
 1860 }
 1861 #endif
 1862 
 1863 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1864   Compile* C = ra_->C;
 1865   C2_MacroAssembler _masm(&cbuf);
 1866   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1867 
 1868   __ remove_frame(framesize);
 1869 
 1870   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1871     __ reserved_stack_check();
 1872   }
 1873 
 1874   if (do_polling() && C->is_method_compilation()) {
 1875     Label dummy_label;
 1876     Label* code_stub = &dummy_label;
 1877     if (!C->output()->in_scratch_emit_size()) {
 1878       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1879     }
 1880     __ relocate(relocInfo::poll_return_type);
 1881     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1882   }
 1883 }
 1884 
 1885 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 1886   // Variable size. Determine dynamically.
 1887   return MachNode::size(ra_);
 1888 }
 1889 
 1890 int MachEpilogNode::reloc() const {
 1891   // Return number of relocatable values contained in this instruction.
 1892   return 1; // 1 for polling page.
 1893 }
 1894 
 1895 const Pipeline * MachEpilogNode::pipeline() const {
 1896   return MachNode::pipeline_class();
 1897 }
 1898 
 1899 //=============================================================================
 1900 
 1901 // Figure out which register class each belongs in: rc_int, rc_float or
 1902 // rc_stack.
 1903 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 1904 
 1905 static enum RC rc_class(OptoReg::Name reg) {
 1906 
 1907   if (reg == OptoReg::Bad) {
 1908     return rc_bad;
 1909   }
 1910 
 1911   // we have 32 int registers * 2 halves
 1912   int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
 1913 
 1914   if (reg < slots_of_int_registers) {
 1915     return rc_int;
 1916   }
 1917 
 1918   // we have 32 float register * 8 halves
 1919   int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
 1920   if (reg < slots_of_int_registers + slots_of_float_registers) {
 1921     return rc_float;
 1922   }
 1923 
 1924   int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
 1925   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 1926     return rc_predicate;
 1927   }
 1928 
 1929   // Between predicate regs & stack is the flags.
 1930   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 1931 
 1932   return rc_stack;
 1933 }
 1934 
 1935 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 1936   Compile* C = ra_->C;
 1937 
 1938   // Get registers to move.
 1939   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 1940   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 1941   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 1942   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 1943 
 1944   enum RC src_hi_rc = rc_class(src_hi);
 1945   enum RC src_lo_rc = rc_class(src_lo);
 1946   enum RC dst_hi_rc = rc_class(dst_hi);
 1947   enum RC dst_lo_rc = rc_class(dst_lo);
 1948 
 1949   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 1950 
 1951   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 1952     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 1953            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 1954            "expected aligned-adjacent pairs");
 1955   }
 1956 
 1957   if (src_lo == dst_lo && src_hi == dst_hi) {
 1958     return 0;            // Self copy, no move.
 1959   }
 1960 
 1961   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 1962               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 1963   int src_offset = ra_->reg2offset(src_lo);
 1964   int dst_offset = ra_->reg2offset(dst_lo);
 1965 
 1966   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 1967     uint ireg = ideal_reg();
 1968     if (ireg == Op_VecA && cbuf) {
 1969       C2_MacroAssembler _masm(cbuf);
 1970       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 1971       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1972         // stack->stack
 1973         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 1974                                                 sve_vector_reg_size_in_bytes);
 1975       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 1976         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 1977                             sve_vector_reg_size_in_bytes);
 1978       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 1979         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 1980                               sve_vector_reg_size_in_bytes);
 1981       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 1982         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1983                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 1984                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 1985       } else {
 1986         ShouldNotReachHere();
 1987       }
 1988     } else if (cbuf) {
 1989       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 1990       C2_MacroAssembler _masm(cbuf);
 1991       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 1992       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1993         // stack->stack
 1994         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 1995         if (ireg == Op_VecD) {
 1996           __ unspill(rscratch1, true, src_offset);
 1997           __ spill(rscratch1, true, dst_offset);
 1998         } else {
 1999           __ spill_copy128(src_offset, dst_offset);
 2000         }
 2001       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2002         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2003                ireg == Op_VecD ? __ T8B : __ T16B,
 2004                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2005       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2006         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2007                  ireg == Op_VecD ? __ D : __ Q,
 2008                  ra_->reg2offset(dst_lo));
 2009       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2010         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2011                    ireg == Op_VecD ? __ D : __ Q,
 2012                    ra_->reg2offset(src_lo));
 2013       } else {
 2014         ShouldNotReachHere();
 2015       }
 2016     }
 2017   } else if (cbuf) {
 2018     C2_MacroAssembler _masm(cbuf);
 2019     switch (src_lo_rc) {
 2020     case rc_int:
 2021       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2022         if (is64) {
 2023             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2024                    as_Register(Matcher::_regEncode[src_lo]));
 2025         } else {
 2026             C2_MacroAssembler _masm(cbuf);
 2027             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2028                     as_Register(Matcher::_regEncode[src_lo]));
 2029         }
 2030       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2031         if (is64) {
 2032             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2033                      as_Register(Matcher::_regEncode[src_lo]));
 2034         } else {
 2035             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2036                      as_Register(Matcher::_regEncode[src_lo]));
 2037         }
 2038       } else {                    // gpr --> stack spill
 2039         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2040         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2041       }
 2042       break;
 2043     case rc_float:
 2044       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2045         if (is64) {
 2046             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2047                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2048         } else {
 2049             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2050                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2051         }
 2052       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2053         if (is64) {
 2054             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2055                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2056         } else {
 2057             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2058                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2059         }
 2060       } else {                    // fpr --> stack spill
 2061         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2062         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2063                  is64 ? __ D : __ S, dst_offset);
 2064       }
 2065       break;
 2066     case rc_stack:
 2067       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2068         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2069       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2070         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2071                    is64 ? __ D : __ S, src_offset);
 2072       } else if (dst_lo_rc == rc_predicate) {
 2073         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2074                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2075       } else {                    // stack --> stack copy
 2076         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2077         if (ideal_reg() == Op_RegVectMask) {
 2078           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2079                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2080         } else {
 2081           __ unspill(rscratch1, is64, src_offset);
 2082           __ spill(rscratch1, is64, dst_offset);
 2083         }
 2084       }
 2085       break;
 2086     case rc_predicate:
 2087       if (dst_lo_rc == rc_predicate) {
 2088         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2089       } else if (dst_lo_rc == rc_stack) {
 2090         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2091                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2092       } else {
 2093         assert(false, "bad src and dst rc_class combination.");
 2094         ShouldNotReachHere();
 2095       }
 2096       break;
 2097     default:
 2098       assert(false, "bad rc_class for spill");
 2099       ShouldNotReachHere();
 2100     }
 2101   }
 2102 
 2103   if (st) {
 2104     st->print("spill ");
 2105     if (src_lo_rc == rc_stack) {
 2106       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2107     } else {
 2108       st->print("%s -> ", Matcher::regName[src_lo]);
 2109     }
 2110     if (dst_lo_rc == rc_stack) {
 2111       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2112     } else {
 2113       st->print("%s", Matcher::regName[dst_lo]);
 2114     }
 2115     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2116       int vsize = 0;
 2117       switch (ideal_reg()) {
 2118       case Op_VecD:
 2119         vsize = 64;
 2120         break;
 2121       case Op_VecX:
 2122         vsize = 128;
 2123         break;
 2124       case Op_VecA:
 2125         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2126         break;
 2127       default:
 2128         assert(false, "bad register type for spill");
 2129         ShouldNotReachHere();
 2130       }
 2131       st->print("\t# vector spill size = %d", vsize);
 2132     } else if (ideal_reg() == Op_RegVectMask) {
 2133       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2134       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2135       st->print("\t# predicate spill size = %d", vsize);
 2136     } else {
 2137       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2138     }
 2139   }
 2140 
 2141   return 0;
 2142 
 2143 }
 2144 
 2145 #ifndef PRODUCT
 2146 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2147   if (!ra_)
 2148     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2149   else
 2150     implementation(NULL, ra_, false, st);
 2151 }
 2152 #endif
 2153 
 2154 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2155   implementation(&cbuf, ra_, false, NULL);
 2156 }
 2157 
 2158 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2159   return MachNode::size(ra_);
 2160 }
 2161 
 2162 //=============================================================================
 2163 
 2164 #ifndef PRODUCT
 2165 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2166   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2167   int reg = ra_->get_reg_first(this);
 2168   st->print("add %s, rsp, #%d]\t# box lock",
 2169             Matcher::regName[reg], offset);
 2170 }
 2171 #endif
 2172 
 2173 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2174   C2_MacroAssembler _masm(&cbuf);
 2175 
 2176   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2177   int reg    = ra_->get_encode(this);
 2178 
 2179   // This add will handle any 24-bit signed offset. 24 bits allows an
 2180   // 8 megabyte stack frame.
 2181   __ add(as_Register(reg), sp, offset);
 2182 }
 2183 
 2184 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2185   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2186   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2187 
 2188   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2189     return NativeInstruction::instruction_size;
 2190   } else {
 2191     return 2 * NativeInstruction::instruction_size;
 2192   }
 2193 }
 2194 
 2195 //=============================================================================
 2196 
 2197 #ifndef PRODUCT
 2198 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2199 {
 2200   st->print_cr("# MachUEPNode");
 2201   if (UseCompressedClassPointers) {
 2202     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2203     if (CompressedKlassPointers::shift() != 0) {
 2204       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2205     }
 2206   } else {
 2207    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2208   }
 2209   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2210   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2211 }
 2212 #endif
 2213 
 2214 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2215 {
 2216   // This is the unverified entry point.
 2217   C2_MacroAssembler _masm(&cbuf);
 2218 
 2219   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2220   Label skip;
 2221   // TODO
 2222   // can we avoid this skip and still use a reloc?
 2223   __ br(Assembler::EQ, skip);
 2224   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2225   __ bind(skip);
 2226 }
 2227 
 2228 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2229 {
 2230   return MachNode::size(ra_);
 2231 }
 2232 
 2233 // REQUIRED EMIT CODE
 2234 
 2235 //=============================================================================
 2236 
 2237 // Emit exception handler code.
 2238 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2239 {
 2240   // mov rscratch1 #exception_blob_entry_point
 2241   // br rscratch1
 2242   // Note that the code buffer's insts_mark is always relative to insts.
 2243   // That's why we must use the macroassembler to generate a handler.
 2244   C2_MacroAssembler _masm(&cbuf);
 2245   address base = __ start_a_stub(size_exception_handler());
 2246   if (base == NULL) {
 2247     ciEnv::current()->record_failure("CodeCache is full");
 2248     return 0;  // CodeBuffer::expand failed
 2249   }
 2250   int offset = __ offset();
 2251   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2252   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2253   __ end_a_stub();
 2254   return offset;
 2255 }
 2256 
 2257 // Emit deopt handler code.
 2258 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2259 {
 2260   // Note that the code buffer's insts_mark is always relative to insts.
 2261   // That's why we must use the macroassembler to generate a handler.
 2262   C2_MacroAssembler _masm(&cbuf);
 2263   address base = __ start_a_stub(size_deopt_handler());
 2264   if (base == NULL) {
 2265     ciEnv::current()->record_failure("CodeCache is full");
 2266     return 0;  // CodeBuffer::expand failed
 2267   }
 2268   int offset = __ offset();
 2269 
 2270   __ adr(lr, __ pc());
 2271   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2272 
 2273   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2274   __ end_a_stub();
 2275   return offset;
 2276 }
 2277 
 2278 // REQUIRED MATCHER CODE
 2279 
 2280 //=============================================================================
 2281 
 2282 const bool Matcher::match_rule_supported(int opcode) {
 2283   if (!has_match_rule(opcode))
 2284     return false;
 2285 
 2286   bool ret_value = true;
 2287   switch (opcode) {
 2288     case Op_OnSpinWait:
 2289       return VM_Version::supports_on_spin_wait();
 2290     case Op_CacheWB:
 2291     case Op_CacheWBPreSync:
 2292     case Op_CacheWBPostSync:
 2293       if (!VM_Version::supports_data_cache_line_flush()) {
 2294         ret_value = false;
 2295       }
 2296       break;
 2297   }
 2298 
 2299   return ret_value; // Per default match rules are supported.
 2300 }
 2301 
 2302 const RegMask* Matcher::predicate_reg_mask(void) {
 2303   return &_PR_REG_mask;
 2304 }
 2305 
 2306 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2307   return new TypeVectMask(elemTy, length);
 2308 }
 2309 
 2310 // Vector calling convention not yet implemented.
 2311 const bool Matcher::supports_vector_calling_convention(void) {
 2312   return false;
 2313 }
 2314 
 2315 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2316   Unimplemented();
 2317   return OptoRegPair(0, 0);
 2318 }
 2319 
 2320 // Is this branch offset short enough that a short branch can be used?
 2321 //
 2322 // NOTE: If the platform does not provide any short branch variants, then
 2323 //       this method should return false for offset 0.
 2324 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2325   // The passed offset is relative to address of the branch.
 2326 
 2327   return (-32768 <= offset && offset < 32768);
 2328 }
 2329 
 2330 // Vector width in bytes.
 2331 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2332   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2333   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2334   // Minimum 2 values in vector
 2335   if (size < 2*type2aelembytes(bt)) size = 0;
 2336   // But never < 4
 2337   if (size < 4) size = 0;
 2338   return size;
 2339 }
 2340 
 2341 // Limits on vector size (number of elements) loaded into vector.
 2342 const int Matcher::max_vector_size(const BasicType bt) {
 2343   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2344 }
 2345 
 2346 const int Matcher::min_vector_size(const BasicType bt) {
 2347   int max_size = max_vector_size(bt);
 2348   // Limit the min vector size to 8 bytes.
 2349   int size = 8 / type2aelembytes(bt);
 2350   if (bt == T_BYTE) {
 2351     // To support vector api shuffle/rearrange.
 2352     size = 4;
 2353   } else if (bt == T_BOOLEAN) {
 2354     // To support vector api load/store mask.
 2355     size = 2;
 2356   }
 2357   if (size < 2) size = 2;
 2358   return MIN2(size, max_size);
 2359 }
 2360 
 2361 // Actual max scalable vector register length.
 2362 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2363   return Matcher::max_vector_size(bt);
 2364 }
 2365 
 2366 // Vector ideal reg.
 2367 const uint Matcher::vector_ideal_reg(int len) {
 2368   if (UseSVE > 0 && 16 < len && len <= 256) {
 2369     return Op_VecA;
 2370   }
 2371   switch(len) {
 2372     // For 16-bit/32-bit mask vector, reuse VecD.
 2373     case  2:
 2374     case  4:
 2375     case  8: return Op_VecD;
 2376     case 16: return Op_VecX;
 2377   }
 2378   ShouldNotReachHere();
 2379   return 0;
 2380 }
 2381 
 2382 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
 2383   assert(Matcher::is_generic_vector(generic_opnd), "not generic");
 2384   switch (ideal_reg) {
 2385     case Op_VecA: return new vecAOper();
 2386     case Op_VecD: return new vecDOper();
 2387     case Op_VecX: return new vecXOper();
 2388   }
 2389   ShouldNotReachHere();
 2390   return NULL;
 2391 }
 2392 
 2393 bool Matcher::is_reg2reg_move(MachNode* m) {
 2394   return false;
 2395 }
 2396 
 2397 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2398   return opnd->opcode() == VREG;
 2399 }
 2400 
 2401 // Return whether or not this register is ever used as an argument.
 2402 // This function is used on startup to build the trampoline stubs in
 2403 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2404 // call in the trampoline, and arguments in those registers not be
 2405 // available to the callee.
 2406 bool Matcher::can_be_java_arg(int reg)
 2407 {
 2408   return
 2409     reg ==  R0_num || reg == R0_H_num ||
 2410     reg ==  R1_num || reg == R1_H_num ||
 2411     reg ==  R2_num || reg == R2_H_num ||
 2412     reg ==  R3_num || reg == R3_H_num ||
 2413     reg ==  R4_num || reg == R4_H_num ||
 2414     reg ==  R5_num || reg == R5_H_num ||
 2415     reg ==  R6_num || reg == R6_H_num ||
 2416     reg ==  R7_num || reg == R7_H_num ||
 2417     reg ==  V0_num || reg == V0_H_num ||
 2418     reg ==  V1_num || reg == V1_H_num ||
 2419     reg ==  V2_num || reg == V2_H_num ||
 2420     reg ==  V3_num || reg == V3_H_num ||
 2421     reg ==  V4_num || reg == V4_H_num ||
 2422     reg ==  V5_num || reg == V5_H_num ||
 2423     reg ==  V6_num || reg == V6_H_num ||
 2424     reg ==  V7_num || reg == V7_H_num;
 2425 }
 2426 
 2427 bool Matcher::is_spillable_arg(int reg)
 2428 {
 2429   return can_be_java_arg(reg);
 2430 }
 2431 
 2432 uint Matcher::int_pressure_limit()
 2433 {
 2434   // JDK-8183543: When taking the number of available registers as int
 2435   // register pressure threshold, the jtreg test:
 2436   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2437   // failed due to C2 compilation failure with
 2438   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2439   //
 2440   // A derived pointer is live at CallNode and then is flagged by RA
 2441   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2442   // derived pointers and lastly fail to spill after reaching maximum
 2443   // number of iterations. Lowering the default pressure threshold to
 2444   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2445   // a high register pressure area of the code so that split_DEF can
 2446   // generate DefinitionSpillCopy for the derived pointer.
 2447   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2448   if (!PreserveFramePointer) {
 2449     // When PreserveFramePointer is off, frame pointer is allocatable,
 2450     // but different from other SOC registers, it is excluded from
 2451     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2452     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2453     // See check_pressure_at_fatproj().
 2454     default_int_pressure_threshold--;
 2455   }
 2456   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2457 }
 2458 
 2459 uint Matcher::float_pressure_limit()
 2460 {
 2461   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2462   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2463 }
 2464 
 2465 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2466   return false;
 2467 }
 2468 
 2469 RegMask Matcher::divI_proj_mask() {
 2470   ShouldNotReachHere();
 2471   return RegMask();
 2472 }
 2473 
 2474 // Register for MODI projection of divmodI.
 2475 RegMask Matcher::modI_proj_mask() {
 2476   ShouldNotReachHere();
 2477   return RegMask();
 2478 }
 2479 
 2480 // Register for DIVL projection of divmodL.
 2481 RegMask Matcher::divL_proj_mask() {
 2482   ShouldNotReachHere();
 2483   return RegMask();
 2484 }
 2485 
 2486 // Register for MODL projection of divmodL.
 2487 RegMask Matcher::modL_proj_mask() {
 2488   ShouldNotReachHere();
 2489   return RegMask();
 2490 }
 2491 
 2492 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2493   return FP_REG_mask();
 2494 }
 2495 
 2496 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2497   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2498     Node* u = addp->fast_out(i);
 2499     if (u->is_LoadStore()) {
 2500       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2501       // instructions) only take register indirect as an operand, so
 2502       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2503       // must fail.
 2504       return false;
 2505     }
 2506     if (u->is_Mem()) {
 2507       int opsize = u->as_Mem()->memory_size();
 2508       assert(opsize > 0, "unexpected memory operand size");
 2509       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2510         return false;
 2511       }
 2512     }
 2513   }
 2514   return true;
 2515 }
 2516 
 2517 // Binary src (Replicate con)
 2518 bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
 2519   if (n == NULL || m == NULL) {
 2520     return false;
 2521   }
 2522 
 2523   if (UseSVE == 0 || !VectorNode::is_invariant_vector(m)) {
 2524     return false;
 2525   }
 2526 
 2527   Node* imm_node = m->in(1);
 2528   if (!imm_node->is_Con()) {
 2529     return false;
 2530   }
 2531 
 2532   const Type* t = imm_node->bottom_type();
 2533   if (!(t->isa_int() || t->isa_long())) {
 2534     return false;
 2535   }
 2536 
 2537   switch (n->Opcode()) {
 2538   case Op_AndV:
 2539   case Op_OrV:
 2540   case Op_XorV: {
 2541     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
 2542     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2543     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2544   }
 2545   case Op_AddVB:
 2546     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2547   case Op_AddVS:
 2548   case Op_AddVI:
 2549     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2550   case Op_AddVL:
 2551     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2552   default:
 2553     return false;
 2554   }
 2555 }
 2556 
 2557 // (XorV src (Replicate m1))
 2558 // (XorVMask src (MaskAll m1))
 2559 bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
 2560   if (n != NULL && m != NULL) {
 2561     return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
 2562            VectorNode::is_all_ones_vector(m);
 2563   }
 2564   return false;
 2565 }
 2566 
 2567 // Should the matcher clone input 'm' of node 'n'?
 2568 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2569   if (is_vshift_con_pattern(n, m) ||
 2570       is_vector_bitwise_not_pattern(n, m) ||
 2571       is_valid_sve_arith_imm_pattern(n, m)) {
 2572     mstack.push(m, Visit);
 2573     return true;
 2574   }
 2575   return false;
 2576 }
 2577 
 2578 // Should the Matcher clone shifts on addressing modes, expecting them
 2579 // to be subsumed into complex addressing expressions or compute them
 2580 // into registers?
 2581 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2582   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2583     return true;
 2584   }
 2585 
 2586   Node *off = m->in(AddPNode::Offset);
 2587   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2588       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2589       // Are there other uses besides address expressions?
 2590       !is_visited(off)) {
 2591     address_visited.set(off->_idx); // Flag as address_visited
 2592     mstack.push(off->in(2), Visit);
 2593     Node *conv = off->in(1);
 2594     if (conv->Opcode() == Op_ConvI2L &&
 2595         // Are there other uses besides address expressions?
 2596         !is_visited(conv)) {
 2597       address_visited.set(conv->_idx); // Flag as address_visited
 2598       mstack.push(conv->in(1), Pre_Visit);
 2599     } else {
 2600       mstack.push(conv, Pre_Visit);
 2601     }
 2602     address_visited.test_set(m->_idx); // Flag as address_visited
 2603     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2604     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2605     return true;
 2606   } else if (off->Opcode() == Op_ConvI2L &&
 2607              // Are there other uses besides address expressions?
 2608              !is_visited(off)) {
 2609     address_visited.test_set(m->_idx); // Flag as address_visited
 2610     address_visited.set(off->_idx); // Flag as address_visited
 2611     mstack.push(off->in(1), Pre_Visit);
 2612     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2613     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2614     return true;
 2615   }
 2616   return false;
 2617 }
 2618 
 2619 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2620   C2_MacroAssembler _masm(&cbuf);                                       \
 2621   {                                                                     \
 2622     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2623     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2624     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2625     __ INSN(REG, as_Register(BASE));                                    \
 2626   }
 2627 
 2628 
 2629 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2630   {
 2631     Address::extend scale;
 2632 
 2633     // Hooboy, this is fugly.  We need a way to communicate to the
 2634     // encoder that the index needs to be sign extended, so we have to
 2635     // enumerate all the cases.
 2636     switch (opcode) {
 2637     case INDINDEXSCALEDI2L:
 2638     case INDINDEXSCALEDI2LN:
 2639     case INDINDEXI2L:
 2640     case INDINDEXI2LN:
 2641       scale = Address::sxtw(size);
 2642       break;
 2643     default:
 2644       scale = Address::lsl(size);
 2645     }
 2646 
 2647     if (index == -1) {
 2648       return Address(base, disp);
 2649     } else {
 2650       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2651       return Address(base, as_Register(index), scale);
 2652     }
 2653   }
 2654 
 2655 
 2656 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2657 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2658 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2659 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2660                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2661 
 2662   // Used for all non-volatile memory accesses.  The use of
 2663   // $mem->opcode() to discover whether this pattern uses sign-extended
 2664   // offsets is something of a kludge.
 2665   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2666                         Register reg, int opcode,
 2667                         Register base, int index, int scale, int disp,
 2668                         int size_in_memory)
 2669   {
 2670     Address addr = mem2address(opcode, base, index, scale, disp);
 2671     if (addr.getMode() == Address::base_plus_offset) {
 2672       /* If we get an out-of-range offset it is a bug in the compiler,
 2673          so we assert here. */
 2674       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2675              "c2 compiler bug");
 2676       /* Fix up any out-of-range offsets. */
 2677       assert_different_registers(rscratch1, base);
 2678       assert_different_registers(rscratch1, reg);
 2679       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2680     }
 2681     (masm.*insn)(reg, addr);
 2682   }
 2683 
 2684   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2685                         FloatRegister reg, int opcode,
 2686                         Register base, int index, int size, int disp,
 2687                         int size_in_memory)
 2688   {
 2689     Address::extend scale;
 2690 
 2691     switch (opcode) {
 2692     case INDINDEXSCALEDI2L:
 2693     case INDINDEXSCALEDI2LN:
 2694       scale = Address::sxtw(size);
 2695       break;
 2696     default:
 2697       scale = Address::lsl(size);
 2698     }
 2699 
 2700     if (index == -1) {
 2701       /* If we get an out-of-range offset it is a bug in the compiler,
 2702          so we assert here. */
 2703       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2704       /* Fix up any out-of-range offsets. */
 2705       assert_different_registers(rscratch1, base);
 2706       Address addr = Address(base, disp);
 2707       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2708       (masm.*insn)(reg, addr);
 2709     } else {
 2710       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2711       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2712     }
 2713   }
 2714 
 2715   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2716                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2717                         int opcode, Register base, int index, int size, int disp)
 2718   {
 2719     if (index == -1) {
 2720       (masm.*insn)(reg, T, Address(base, disp));
 2721     } else {
 2722       assert(disp == 0, "unsupported address mode");
 2723       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2724     }
 2725   }
 2726 
 2727 %}
 2728 
 2729 
 2730 
 2731 //----------ENCODING BLOCK-----------------------------------------------------
 2732 // This block specifies the encoding classes used by the compiler to
 2733 // output byte streams.  Encoding classes are parameterized macros
 2734 // used by Machine Instruction Nodes in order to generate the bit
 2735 // encoding of the instruction.  Operands specify their base encoding
 2736 // interface with the interface keyword.  There are currently
 2737 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2738 // COND_INTER.  REG_INTER causes an operand to generate a function
 2739 // which returns its register number when queried.  CONST_INTER causes
 2740 // an operand to generate a function which returns the value of the
 2741 // constant when queried.  MEMORY_INTER causes an operand to generate
 2742 // four functions which return the Base Register, the Index Register,
 2743 // the Scale Value, and the Offset Value of the operand when queried.
 2744 // COND_INTER causes an operand to generate six functions which return
 2745 // the encoding code (ie - encoding bits for the instruction)
 2746 // associated with each basic boolean condition for a conditional
 2747 // instruction.
 2748 //
 2749 // Instructions specify two basic values for encoding.  Again, a
 2750 // function is available to check if the constant displacement is an
 2751 // oop. They use the ins_encode keyword to specify their encoding
 2752 // classes (which must be a sequence of enc_class names, and their
 2753 // parameters, specified in the encoding block), and they use the
 2754 // opcode keyword to specify, in order, their primary, secondary, and
 2755 // tertiary opcode.  Only the opcode sections which a particular
 2756 // instruction needs for encoding need to be specified.
 2757 encode %{
 2758   // Build emit functions for each basic byte or larger field in the
 2759   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2760   // from C++ code in the enc_class source block.  Emit functions will
 2761   // live in the main source block for now.  In future, we can
 2762   // generalize this by adding a syntax that specifies the sizes of
 2763   // fields in an order, so that the adlc can build the emit functions
 2764   // automagically
 2765 
 2766   // catch all for unimplemented encodings
 2767   enc_class enc_unimplemented %{
 2768     C2_MacroAssembler _masm(&cbuf);
 2769     __ unimplemented("C2 catch all");
 2770   %}
 2771 
 2772   // BEGIN Non-volatile memory access
 2773 
 2774   // This encoding class is generated automatically from ad_encode.m4.
 2775   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2776   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2777     Register dst_reg = as_Register($dst$$reg);
 2778     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2779                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2780   %}
 2781 
 2782   // This encoding class is generated automatically from ad_encode.m4.
 2783   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2784   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2785     Register dst_reg = as_Register($dst$$reg);
 2786     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2787                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2788   %}
 2789 
 2790   // This encoding class is generated automatically from ad_encode.m4.
 2791   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2792   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2793     Register dst_reg = as_Register($dst$$reg);
 2794     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2795                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2796   %}
 2797 
 2798   // This encoding class is generated automatically from ad_encode.m4.
 2799   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2800   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2801     Register dst_reg = as_Register($dst$$reg);
 2802     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2803                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2804   %}
 2805 
 2806   // This encoding class is generated automatically from ad_encode.m4.
 2807   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2808   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2809     Register dst_reg = as_Register($dst$$reg);
 2810     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2811                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2812   %}
 2813 
 2814   // This encoding class is generated automatically from ad_encode.m4.
 2815   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2816   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2817     Register dst_reg = as_Register($dst$$reg);
 2818     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2819                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2820   %}
 2821 
 2822   // This encoding class is generated automatically from ad_encode.m4.
 2823   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2824   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2825     Register dst_reg = as_Register($dst$$reg);
 2826     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2827                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2828   %}
 2829 
 2830   // This encoding class is generated automatically from ad_encode.m4.
 2831   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2832   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2833     Register dst_reg = as_Register($dst$$reg);
 2834     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2835                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2836   %}
 2837 
 2838   // This encoding class is generated automatically from ad_encode.m4.
 2839   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2840   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2841     Register dst_reg = as_Register($dst$$reg);
 2842     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2843                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2844   %}
 2845 
 2846   // This encoding class is generated automatically from ad_encode.m4.
 2847   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2848   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2849     Register dst_reg = as_Register($dst$$reg);
 2850     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2851                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2852   %}
 2853 
 2854   // This encoding class is generated automatically from ad_encode.m4.
 2855   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2856   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2857     Register dst_reg = as_Register($dst$$reg);
 2858     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2859                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2860   %}
 2861 
 2862   // This encoding class is generated automatically from ad_encode.m4.
 2863   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2864   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2865     Register dst_reg = as_Register($dst$$reg);
 2866     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2867                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2868   %}
 2869 
 2870   // This encoding class is generated automatically from ad_encode.m4.
 2871   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2872   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2873     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2874     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2875                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2876   %}
 2877 
 2878   // This encoding class is generated automatically from ad_encode.m4.
 2879   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2880   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 2881     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2882     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2883                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2884   %}
 2885 
 2886   // This encoding class is generated automatically from ad_encode.m4.
 2887   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2888   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 2889     Register src_reg = as_Register($src$$reg);
 2890     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 2891                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2892   %}
 2893 
 2894   // This encoding class is generated automatically from ad_encode.m4.
 2895   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2896   enc_class aarch64_enc_strb0(memory1 mem) %{
 2897     C2_MacroAssembler _masm(&cbuf);
 2898     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2899                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2900   %}
 2901 
 2902   // This encoding class is generated automatically from ad_encode.m4.
 2903   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2904   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 2905     Register src_reg = as_Register($src$$reg);
 2906     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 2907                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2908   %}
 2909 
 2910   // This encoding class is generated automatically from ad_encode.m4.
 2911   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2912   enc_class aarch64_enc_strh0(memory2 mem) %{
 2913     C2_MacroAssembler _masm(&cbuf);
 2914     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 2915                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2916   %}
 2917 
 2918   // This encoding class is generated automatically from ad_encode.m4.
 2919   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2920   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 2921     Register src_reg = as_Register($src$$reg);
 2922     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 2923                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2924   %}
 2925 
 2926   // This encoding class is generated automatically from ad_encode.m4.
 2927   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2928   enc_class aarch64_enc_strw0(memory4 mem) %{
 2929     C2_MacroAssembler _masm(&cbuf);
 2930     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 2931                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2932   %}
 2933 
 2934   // This encoding class is generated automatically from ad_encode.m4.
 2935   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2936   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 2937     Register src_reg = as_Register($src$$reg);
 2938     // we sometimes get asked to store the stack pointer into the
 2939     // current thread -- we cannot do that directly on AArch64
 2940     if (src_reg == r31_sp) {
 2941       C2_MacroAssembler _masm(&cbuf);
 2942       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 2943       __ mov(rscratch2, sp);
 2944       src_reg = rscratch2;
 2945     }
 2946     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 2947                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2948   %}
 2949 
 2950   // This encoding class is generated automatically from ad_encode.m4.
 2951   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2952   enc_class aarch64_enc_str0(memory8 mem) %{
 2953     C2_MacroAssembler _masm(&cbuf);
 2954     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 2955                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2956   %}
 2957 
 2958   // This encoding class is generated automatically from ad_encode.m4.
 2959   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2960   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 2961     FloatRegister src_reg = as_FloatRegister($src$$reg);
 2962     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 2963                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2964   %}
 2965 
 2966   // This encoding class is generated automatically from ad_encode.m4.
 2967   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2968   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 2969     FloatRegister src_reg = as_FloatRegister($src$$reg);
 2970     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 2971                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2972   %}
 2973 
 2974   // This encoding class is generated automatically from ad_encode.m4.
 2975   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2976   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 2977       C2_MacroAssembler _masm(&cbuf);
 2978       __ membar(Assembler::StoreStore);
 2979       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2980                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2981   %}
 2982 
 2983   // END Non-volatile memory access
 2984 
 2985   // Vector loads and stores
 2986   enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
 2987     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2988     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 2989        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 2990   %}
 2991 
 2992   enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
 2993     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2994     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 2995        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 2996   %}
 2997 
 2998   enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
 2999     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3000     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3001        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3002   %}
 3003 
 3004   enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
 3005     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3006     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3007        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3008   %}
 3009 
 3010   enc_class aarch64_enc_strvH(vReg src, memory mem) %{
 3011     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3012     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3013        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3014   %}
 3015 
 3016   enc_class aarch64_enc_strvS(vReg src, memory mem) %{
 3017     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3018     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3019        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3020   %}
 3021 
 3022   enc_class aarch64_enc_strvD(vReg src, memory mem) %{
 3023     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3024     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3025        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3026   %}
 3027 
 3028   enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
 3029     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3030     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3031        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3032   %}
 3033 
 3034   // volatile loads and stores
 3035 
 3036   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3037     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3038                  rscratch1, stlrb);
 3039   %}
 3040 
 3041   enc_class aarch64_enc_stlrb0(memory mem) %{
 3042     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3043                  rscratch1, stlrb);
 3044   %}
 3045 
 3046   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3047     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3048                  rscratch1, stlrh);
 3049   %}
 3050 
 3051   enc_class aarch64_enc_stlrh0(memory mem) %{
 3052     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3053                  rscratch1, stlrh);
 3054   %}
 3055 
 3056   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3057     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3058                  rscratch1, stlrw);
 3059   %}
 3060 
 3061   enc_class aarch64_enc_stlrw0(memory mem) %{
 3062     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3063                  rscratch1, stlrw);
 3064   %}
 3065 
 3066   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3067     Register dst_reg = as_Register($dst$$reg);
 3068     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3069              rscratch1, ldarb);
 3070     __ sxtbw(dst_reg, dst_reg);
 3071   %}
 3072 
 3073   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3074     Register dst_reg = as_Register($dst$$reg);
 3075     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3076              rscratch1, ldarb);
 3077     __ sxtb(dst_reg, dst_reg);
 3078   %}
 3079 
 3080   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3081     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3082              rscratch1, ldarb);
 3083   %}
 3084 
 3085   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3086     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3087              rscratch1, ldarb);
 3088   %}
 3089 
 3090   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3091     Register dst_reg = as_Register($dst$$reg);
 3092     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3093              rscratch1, ldarh);
 3094     __ sxthw(dst_reg, dst_reg);
 3095   %}
 3096 
 3097   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3098     Register dst_reg = as_Register($dst$$reg);
 3099     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3100              rscratch1, ldarh);
 3101     __ sxth(dst_reg, dst_reg);
 3102   %}
 3103 
 3104   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3105     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3106              rscratch1, ldarh);
 3107   %}
 3108 
 3109   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3110     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3111              rscratch1, ldarh);
 3112   %}
 3113 
 3114   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3115     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3116              rscratch1, ldarw);
 3117   %}
 3118 
 3119   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3120     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3121              rscratch1, ldarw);
 3122   %}
 3123 
 3124   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3125     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3126              rscratch1, ldar);
 3127   %}
 3128 
 3129   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3130     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3131              rscratch1, ldarw);
 3132     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3133   %}
 3134 
 3135   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3136     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3137              rscratch1, ldar);
 3138     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3139   %}
 3140 
 3141   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3142     Register src_reg = as_Register($src$$reg);
 3143     // we sometimes get asked to store the stack pointer into the
 3144     // current thread -- we cannot do that directly on AArch64
 3145     if (src_reg == r31_sp) {
 3146       C2_MacroAssembler _masm(&cbuf);
 3147       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3148       __ mov(rscratch2, sp);
 3149       src_reg = rscratch2;
 3150     }
 3151     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3152                  rscratch1, stlr);
 3153   %}
 3154 
 3155   enc_class aarch64_enc_stlr0(memory mem) %{
 3156     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3157                  rscratch1, stlr);
 3158   %}
 3159 
 3160   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3161     {
 3162       C2_MacroAssembler _masm(&cbuf);
 3163       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3164       __ fmovs(rscratch2, src_reg);
 3165     }
 3166     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3167                  rscratch1, stlrw);
 3168   %}
 3169 
 3170   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3171     {
 3172       C2_MacroAssembler _masm(&cbuf);
 3173       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3174       __ fmovd(rscratch2, src_reg);
 3175     }
 3176     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3177                  rscratch1, stlr);
 3178   %}
 3179 
 3180   // synchronized read/update encodings
 3181 
 3182   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3183     C2_MacroAssembler _masm(&cbuf);
 3184     Register dst_reg = as_Register($dst$$reg);
 3185     Register base = as_Register($mem$$base);
 3186     int index = $mem$$index;
 3187     int scale = $mem$$scale;
 3188     int disp = $mem$$disp;
 3189     if (index == -1) {
 3190        if (disp != 0) {
 3191         __ lea(rscratch1, Address(base, disp));
 3192         __ ldaxr(dst_reg, rscratch1);
 3193       } else {
 3194         // TODO
 3195         // should we ever get anything other than this case?
 3196         __ ldaxr(dst_reg, base);
 3197       }
 3198     } else {
 3199       Register index_reg = as_Register(index);
 3200       if (disp == 0) {
 3201         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3202         __ ldaxr(dst_reg, rscratch1);
 3203       } else {
 3204         __ lea(rscratch1, Address(base, disp));
 3205         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3206         __ ldaxr(dst_reg, rscratch1);
 3207       }
 3208     }
 3209   %}
 3210 
 3211   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3212     C2_MacroAssembler _masm(&cbuf);
 3213     Register src_reg = as_Register($src$$reg);
 3214     Register base = as_Register($mem$$base);
 3215     int index = $mem$$index;
 3216     int scale = $mem$$scale;
 3217     int disp = $mem$$disp;
 3218     if (index == -1) {
 3219        if (disp != 0) {
 3220         __ lea(rscratch2, Address(base, disp));
 3221         __ stlxr(rscratch1, src_reg, rscratch2);
 3222       } else {
 3223         // TODO
 3224         // should we ever get anything other than this case?
 3225         __ stlxr(rscratch1, src_reg, base);
 3226       }
 3227     } else {
 3228       Register index_reg = as_Register(index);
 3229       if (disp == 0) {
 3230         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3231         __ stlxr(rscratch1, src_reg, rscratch2);
 3232       } else {
 3233         __ lea(rscratch2, Address(base, disp));
 3234         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3235         __ stlxr(rscratch1, src_reg, rscratch2);
 3236       }
 3237     }
 3238     __ cmpw(rscratch1, zr);
 3239   %}
 3240 
 3241   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3242     C2_MacroAssembler _masm(&cbuf);
 3243     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3244     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3245                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3246                /*weak*/ false, noreg);
 3247   %}
 3248 
 3249   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3250     C2_MacroAssembler _masm(&cbuf);
 3251     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3252     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3253                Assembler::word, /*acquire*/ false, /*release*/ true,
 3254                /*weak*/ false, noreg);
 3255   %}
 3256 
 3257   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3258     C2_MacroAssembler _masm(&cbuf);
 3259     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3260     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3261                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3262                /*weak*/ false, noreg);
 3263   %}
 3264 
 3265   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3266     C2_MacroAssembler _masm(&cbuf);
 3267     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3268     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3269                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3270                /*weak*/ false, noreg);
 3271   %}
 3272 
 3273 
 3274   // The only difference between aarch64_enc_cmpxchg and
 3275   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3276   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3277   // lock.
 3278   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3279     C2_MacroAssembler _masm(&cbuf);
 3280     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3281     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3282                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3283                /*weak*/ false, noreg);
 3284   %}
 3285 
 3286   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3287     C2_MacroAssembler _masm(&cbuf);
 3288     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3289     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3290                Assembler::word, /*acquire*/ true, /*release*/ true,
 3291                /*weak*/ false, noreg);
 3292   %}
 3293 
 3294   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3295     C2_MacroAssembler _masm(&cbuf);
 3296     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3297     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3298                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3299                /*weak*/ false, noreg);
 3300   %}
 3301 
 3302   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3303     C2_MacroAssembler _masm(&cbuf);
 3304     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3305     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3306                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3307                /*weak*/ false, noreg);
 3308   %}
 3309 
 3310   // auxiliary used for CompareAndSwapX to set result register
 3311   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3312     C2_MacroAssembler _masm(&cbuf);
 3313     Register res_reg = as_Register($res$$reg);
 3314     __ cset(res_reg, Assembler::EQ);
 3315   %}
 3316 
 3317   // prefetch encodings
 3318 
 3319   enc_class aarch64_enc_prefetchw(memory mem) %{
 3320     C2_MacroAssembler _masm(&cbuf);
 3321     Register base = as_Register($mem$$base);
 3322     int index = $mem$$index;
 3323     int scale = $mem$$scale;
 3324     int disp = $mem$$disp;
 3325     if (index == -1) {
 3326       __ prfm(Address(base, disp), PSTL1KEEP);
 3327     } else {
 3328       Register index_reg = as_Register(index);
 3329       if (disp == 0) {
 3330         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3331       } else {
 3332         __ lea(rscratch1, Address(base, disp));
 3333 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3334       }
 3335     }
 3336   %}
 3337 
 3338   /// mov envcodings
 3339 
 3340   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3341     C2_MacroAssembler _masm(&cbuf);
 3342     uint32_t con = (uint32_t)$src$$constant;
 3343     Register dst_reg = as_Register($dst$$reg);
 3344     if (con == 0) {
 3345       __ movw(dst_reg, zr);
 3346     } else {
 3347       __ movw(dst_reg, con);
 3348     }
 3349   %}
 3350 
 3351   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3352     C2_MacroAssembler _masm(&cbuf);
 3353     Register dst_reg = as_Register($dst$$reg);
 3354     uint64_t con = (uint64_t)$src$$constant;
 3355     if (con == 0) {
 3356       __ mov(dst_reg, zr);
 3357     } else {
 3358       __ mov(dst_reg, con);
 3359     }
 3360   %}
 3361 
 3362   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3363     C2_MacroAssembler _masm(&cbuf);
 3364     Register dst_reg = as_Register($dst$$reg);
 3365     address con = (address)$src$$constant;
 3366     if (con == NULL || con == (address)1) {
 3367       ShouldNotReachHere();
 3368     } else {
 3369       relocInfo::relocType rtype = $src->constant_reloc();
 3370       if (rtype == relocInfo::oop_type) {
 3371         __ movoop(dst_reg, (jobject)con);
 3372       } else if (rtype == relocInfo::metadata_type) {
 3373         __ mov_metadata(dst_reg, (Metadata*)con);
 3374       } else {
 3375         assert(rtype == relocInfo::none, "unexpected reloc type");
 3376         if (con < (address)(uintptr_t)os::vm_page_size()) {
 3377           __ mov(dst_reg, con);
 3378         } else {
 3379           uint64_t offset;
 3380           __ adrp(dst_reg, con, offset);
 3381           __ add(dst_reg, dst_reg, offset);
 3382         }
 3383       }
 3384     }
 3385   %}
 3386 
 3387   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3388     C2_MacroAssembler _masm(&cbuf);
 3389     Register dst_reg = as_Register($dst$$reg);
 3390     __ mov(dst_reg, zr);
 3391   %}
 3392 
 3393   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3394     C2_MacroAssembler _masm(&cbuf);
 3395     Register dst_reg = as_Register($dst$$reg);
 3396     __ mov(dst_reg, (uint64_t)1);
 3397   %}
 3398 
 3399   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3400     C2_MacroAssembler _masm(&cbuf);
 3401     __ load_byte_map_base($dst$$Register);
 3402   %}
 3403 
 3404   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3405     C2_MacroAssembler _masm(&cbuf);
 3406     Register dst_reg = as_Register($dst$$reg);
 3407     address con = (address)$src$$constant;
 3408     if (con == NULL) {
 3409       ShouldNotReachHere();
 3410     } else {
 3411       relocInfo::relocType rtype = $src->constant_reloc();
 3412       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3413       __ set_narrow_oop(dst_reg, (jobject)con);
 3414     }
 3415   %}
 3416 
 3417   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3418     C2_MacroAssembler _masm(&cbuf);
 3419     Register dst_reg = as_Register($dst$$reg);
 3420     __ mov(dst_reg, zr);
 3421   %}
 3422 
 3423   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3424     C2_MacroAssembler _masm(&cbuf);
 3425     Register dst_reg = as_Register($dst$$reg);
 3426     address con = (address)$src$$constant;
 3427     if (con == NULL) {
 3428       ShouldNotReachHere();
 3429     } else {
 3430       relocInfo::relocType rtype = $src->constant_reloc();
 3431       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3432       __ set_narrow_klass(dst_reg, (Klass *)con);
 3433     }
 3434   %}
 3435 
 3436   // arithmetic encodings
 3437 
 3438   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3439     C2_MacroAssembler _masm(&cbuf);
 3440     Register dst_reg = as_Register($dst$$reg);
 3441     Register src_reg = as_Register($src1$$reg);
 3442     int32_t con = (int32_t)$src2$$constant;
 3443     // add has primary == 0, subtract has primary == 1
 3444     if ($primary) { con = -con; }
 3445     if (con < 0) {
 3446       __ subw(dst_reg, src_reg, -con);
 3447     } else {
 3448       __ addw(dst_reg, src_reg, con);
 3449     }
 3450   %}
 3451 
 3452   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3453     C2_MacroAssembler _masm(&cbuf);
 3454     Register dst_reg = as_Register($dst$$reg);
 3455     Register src_reg = as_Register($src1$$reg);
 3456     int32_t con = (int32_t)$src2$$constant;
 3457     // add has primary == 0, subtract has primary == 1
 3458     if ($primary) { con = -con; }
 3459     if (con < 0) {
 3460       __ sub(dst_reg, src_reg, -con);
 3461     } else {
 3462       __ add(dst_reg, src_reg, con);
 3463     }
 3464   %}
 3465 
 3466   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3467     C2_MacroAssembler _masm(&cbuf);
 3468    Register dst_reg = as_Register($dst$$reg);
 3469    Register src1_reg = as_Register($src1$$reg);
 3470    Register src2_reg = as_Register($src2$$reg);
 3471     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3472   %}
 3473 
 3474   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3475     C2_MacroAssembler _masm(&cbuf);
 3476    Register dst_reg = as_Register($dst$$reg);
 3477    Register src1_reg = as_Register($src1$$reg);
 3478    Register src2_reg = as_Register($src2$$reg);
 3479     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3480   %}
 3481 
 3482   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3483     C2_MacroAssembler _masm(&cbuf);
 3484    Register dst_reg = as_Register($dst$$reg);
 3485    Register src1_reg = as_Register($src1$$reg);
 3486    Register src2_reg = as_Register($src2$$reg);
 3487     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3488   %}
 3489 
 3490   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3491     C2_MacroAssembler _masm(&cbuf);
 3492    Register dst_reg = as_Register($dst$$reg);
 3493    Register src1_reg = as_Register($src1$$reg);
 3494    Register src2_reg = as_Register($src2$$reg);
 3495     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3496   %}
 3497 
 3498   // compare instruction encodings
 3499 
 3500   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3501     C2_MacroAssembler _masm(&cbuf);
 3502     Register reg1 = as_Register($src1$$reg);
 3503     Register reg2 = as_Register($src2$$reg);
 3504     __ cmpw(reg1, reg2);
 3505   %}
 3506 
 3507   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3508     C2_MacroAssembler _masm(&cbuf);
 3509     Register reg = as_Register($src1$$reg);
 3510     int32_t val = $src2$$constant;
 3511     if (val >= 0) {
 3512       __ subsw(zr, reg, val);
 3513     } else {
 3514       __ addsw(zr, reg, -val);
 3515     }
 3516   %}
 3517 
 3518   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3519     C2_MacroAssembler _masm(&cbuf);
 3520     Register reg1 = as_Register($src1$$reg);
 3521     uint32_t val = (uint32_t)$src2$$constant;
 3522     __ movw(rscratch1, val);
 3523     __ cmpw(reg1, rscratch1);
 3524   %}
 3525 
 3526   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3527     C2_MacroAssembler _masm(&cbuf);
 3528     Register reg1 = as_Register($src1$$reg);
 3529     Register reg2 = as_Register($src2$$reg);
 3530     __ cmp(reg1, reg2);
 3531   %}
 3532 
 3533   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3534     C2_MacroAssembler _masm(&cbuf);
 3535     Register reg = as_Register($src1$$reg);
 3536     int64_t val = $src2$$constant;
 3537     if (val >= 0) {
 3538       __ subs(zr, reg, val);
 3539     } else if (val != -val) {
 3540       __ adds(zr, reg, -val);
 3541     } else {
 3542     // aargh, Long.MIN_VALUE is a special case
 3543       __ orr(rscratch1, zr, (uint64_t)val);
 3544       __ subs(zr, reg, rscratch1);
 3545     }
 3546   %}
 3547 
 3548   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3549     C2_MacroAssembler _masm(&cbuf);
 3550     Register reg1 = as_Register($src1$$reg);
 3551     uint64_t val = (uint64_t)$src2$$constant;
 3552     __ mov(rscratch1, val);
 3553     __ cmp(reg1, rscratch1);
 3554   %}
 3555 
 3556   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3557     C2_MacroAssembler _masm(&cbuf);
 3558     Register reg1 = as_Register($src1$$reg);
 3559     Register reg2 = as_Register($src2$$reg);
 3560     __ cmp(reg1, reg2);
 3561   %}
 3562 
 3563   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3564     C2_MacroAssembler _masm(&cbuf);
 3565     Register reg1 = as_Register($src1$$reg);
 3566     Register reg2 = as_Register($src2$$reg);
 3567     __ cmpw(reg1, reg2);
 3568   %}
 3569 
 3570   enc_class aarch64_enc_testp(iRegP src) %{
 3571     C2_MacroAssembler _masm(&cbuf);
 3572     Register reg = as_Register($src$$reg);
 3573     __ cmp(reg, zr);
 3574   %}
 3575 
 3576   enc_class aarch64_enc_testn(iRegN src) %{
 3577     C2_MacroAssembler _masm(&cbuf);
 3578     Register reg = as_Register($src$$reg);
 3579     __ cmpw(reg, zr);
 3580   %}
 3581 
 3582   enc_class aarch64_enc_b(label lbl) %{
 3583     C2_MacroAssembler _masm(&cbuf);
 3584     Label *L = $lbl$$label;
 3585     __ b(*L);
 3586   %}
 3587 
 3588   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3589     C2_MacroAssembler _masm(&cbuf);
 3590     Label *L = $lbl$$label;
 3591     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3592   %}
 3593 
 3594   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3595     C2_MacroAssembler _masm(&cbuf);
 3596     Label *L = $lbl$$label;
 3597     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3598   %}
 3599 
 3600   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3601   %{
 3602      Register sub_reg = as_Register($sub$$reg);
 3603      Register super_reg = as_Register($super$$reg);
 3604      Register temp_reg = as_Register($temp$$reg);
 3605      Register result_reg = as_Register($result$$reg);
 3606 
 3607      Label miss;
 3608      C2_MacroAssembler _masm(&cbuf);
 3609      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3610                                      NULL, &miss,
 3611                                      /*set_cond_codes:*/ true);
 3612      if ($primary) {
 3613        __ mov(result_reg, zr);
 3614      }
 3615      __ bind(miss);
 3616   %}
 3617 
 3618   enc_class aarch64_enc_java_static_call(method meth) %{
 3619     C2_MacroAssembler _masm(&cbuf);
 3620 
 3621     address addr = (address)$meth$$method;
 3622     address call;
 3623     if (!_method) {
 3624       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3625       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
 3626       if (call == NULL) {
 3627         ciEnv::current()->record_failure("CodeCache is full");
 3628         return;
 3629       }
 3630     } else {
 3631       int method_index = resolved_method_index(cbuf);
 3632       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3633                                                   : static_call_Relocation::spec(method_index);
 3634       call = __ trampoline_call(Address(addr, rspec));
 3635       if (call == NULL) {
 3636         ciEnv::current()->record_failure("CodeCache is full");
 3637         return;
 3638       }
 3639       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 3640         // Calls of the same statically bound method can share
 3641         // a stub to the interpreter.
 3642         cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
 3643       } else {
 3644         // Emit stub for static call
 3645         address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
 3646         if (stub == NULL) {
 3647           ciEnv::current()->record_failure("CodeCache is full");
 3648           return;
 3649         }
 3650       }
 3651     }
 3652 
 3653     __ post_call_nop();
 3654 
 3655     // Only non uncommon_trap calls need to reinitialize ptrue.
 3656     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3657       __ reinitialize_ptrue();
 3658     }
 3659   %}
 3660 
 3661   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3662     C2_MacroAssembler _masm(&cbuf);
 3663     int method_index = resolved_method_index(cbuf);
 3664     address call = __ ic_call((address)$meth$$method, method_index);
 3665     if (call == NULL) {
 3666       ciEnv::current()->record_failure("CodeCache is full");
 3667       return;
 3668     }
 3669     _masm.clear_inst_mark();
 3670     __ post_call_nop();
 3671     if (Compile::current()->max_vector_size() > 0) {
 3672       __ reinitialize_ptrue();
 3673     }
 3674   %}
 3675 
 3676   enc_class aarch64_enc_call_epilog() %{
 3677     C2_MacroAssembler _masm(&cbuf);
 3678     if (VerifyStackAtCalls) {
 3679       // Check that stack depth is unchanged: find majik cookie on stack
 3680       __ call_Unimplemented();
 3681     }
 3682   %}
 3683 
 3684   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3685     C2_MacroAssembler _masm(&cbuf);
 3686 
 3687     // some calls to generated routines (arraycopy code) are scheduled
 3688     // by C2 as runtime calls. if so we can call them using a br (they
 3689     // will be in a reachable segment) otherwise we have to use a blr
 3690     // which loads the absolute address into a register.
 3691     address entry = (address)$meth$$method;
 3692     CodeBlob *cb = CodeCache::find_blob(entry);
 3693     if (cb) {
 3694       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3695       if (call == NULL) {
 3696         ciEnv::current()->record_failure("CodeCache is full");
 3697         return;
 3698       }
 3699       __ post_call_nop();
 3700     } else {
 3701       Label retaddr;
 3702       __ adr(rscratch2, retaddr);
 3703       __ lea(rscratch1, RuntimeAddress(entry));
 3704       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3705       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3706       __ blr(rscratch1);
 3707       __ bind(retaddr);
 3708       __ post_call_nop();
 3709       __ add(sp, sp, 2 * wordSize);
 3710     }
 3711     if (Compile::current()->max_vector_size() > 0) {
 3712       __ reinitialize_ptrue();
 3713     }
 3714   %}
 3715 
 3716   enc_class aarch64_enc_rethrow() %{
 3717     C2_MacroAssembler _masm(&cbuf);
 3718     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3719   %}
 3720 
 3721   enc_class aarch64_enc_ret() %{
 3722     C2_MacroAssembler _masm(&cbuf);
 3723 #ifdef ASSERT
 3724     if (Compile::current()->max_vector_size() > 0) {
 3725       __ verify_ptrue();
 3726     }
 3727 #endif
 3728     __ ret(lr);
 3729   %}
 3730 
 3731   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3732     C2_MacroAssembler _masm(&cbuf);
 3733     Register target_reg = as_Register($jump_target$$reg);
 3734     __ br(target_reg);
 3735   %}
 3736 
 3737   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3738     C2_MacroAssembler _masm(&cbuf);
 3739     Register target_reg = as_Register($jump_target$$reg);
 3740     // exception oop should be in r0
 3741     // ret addr has been popped into lr
 3742     // callee expects it in r3
 3743     __ mov(r3, lr);
 3744     __ br(target_reg);
 3745   %}
 3746 
 3747   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3748     C2_MacroAssembler _masm(&cbuf);
 3749     Register oop = as_Register($object$$reg);
 3750     Register box = as_Register($box$$reg);
 3751     Register disp_hdr = as_Register($tmp$$reg);
 3752     Register tmp = as_Register($tmp2$$reg);
 3753     Label cont;
 3754     Label object_has_monitor;
 3755     Label no_count;
 3756 
 3757     assert_different_registers(oop, box, tmp, disp_hdr);
 3758 
 3759     // Load markWord from object into displaced_header.
 3760     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3761 
 3762     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3763       __ load_klass(tmp, oop);
 3764       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3765       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3766       __ br(Assembler::NE, cont);
 3767     }
 3768 
 3769     // Check for existing monitor
 3770     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3771 
 3772     if (!UseHeavyMonitors) {
 3773       Label slow;
 3774       __ fast_lock(oop, disp_hdr, box, tmp, rscratch1, slow);
 3775 
 3776       // Indicate success at cont.
 3777       __ cmp(oop, oop);
 3778       __ b(cont);
 3779       __ bind(slow);
 3780     }
 3781     __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3782     __ b(cont);
 3783 
 3784     // Handle existing monitor.
 3785     __ bind(object_has_monitor);
 3786 
 3787     // The object's monitor m is unlocked iff m->owner == NULL,
 3788     // otherwise m->owner may contain a thread or a stack address.
 3789     //
 3790     // Try to CAS m->owner from NULL to current thread.
 3791     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3792     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3793                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 3794 
 3795     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 3796 
 3797     __ cmp(rscratch1, rthread);
 3798     __ br(Assembler::NE, cont); // Check for recursive locking
 3799 
 3800     // Recursive lock case
 3801     __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
 3802     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 3803 
 3804     __ bind(cont);
 3805     // flag == EQ indicates success
 3806     // flag == NE indicates failure
 3807     __ br(Assembler::NE, no_count);
 3808 
 3809     __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
 3810 
 3811     __ bind(no_count);
 3812   %}
 3813 
 3814   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3815     C2_MacroAssembler _masm(&cbuf);
 3816     Register oop = as_Register($object$$reg);
 3817     Register box = as_Register($box$$reg);
 3818     Register disp_hdr = as_Register($tmp$$reg);
 3819     Register tmp = as_Register($tmp2$$reg);
 3820     Label cont;
 3821     Label object_has_monitor;
 3822     Label no_count;
 3823 
 3824     assert_different_registers(oop, box, tmp, disp_hdr);
 3825 
 3826     // Handle existing monitor.
 3827     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 3828 
 3829     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3830 
 3831     if (!UseHeavyMonitors) {
 3832       Label slow;
 3833       __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
 3834 
 3835       __ fast_unlock(oop, tmp, box, disp_hdr, slow);
 3836 
 3837       // Indicate success at cont.
 3838       __ cmp(oop, oop);
 3839       __ b(cont);
 3840 
 3841       __ bind(slow);
 3842     }
 3843     __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3844     __ b(cont);
 3845 
 3846     // Handle existing monitor.
 3847     __ bind(object_has_monitor);
 3848     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 3849     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 3850 
 3851     // If the owner is anonymous, we need to fix it -- in the slow-path.
 3852     {
 3853       Label L;
 3854       __ ldr(disp_hdr, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3855       __ cmp(disp_hdr, (unsigned char)(intptr_t) ANONYMOUS_OWNER);
 3856       __ br(Assembler::NE, L);
 3857       __ tst(oop, oop); // Indicate failure at cont -- dive into slow-path.
 3858       __ b(cont);
 3859       __ bind(L);
 3860     }
 3861 
 3862    __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3863 
 3864     Label notRecursive;
 3865     __ cbz(disp_hdr, notRecursive);
 3866 
 3867     // Recursive lock
 3868     __ sub(disp_hdr, disp_hdr, 1u);
 3869     __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3870     __ cmp(disp_hdr, disp_hdr); // Sets flags for result
 3871     __ b(cont);
 3872 
 3873     __ bind(notRecursive);
 3874     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 3875     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 3876     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 3877     __ cmp(rscratch1, zr); // Sets flags for result
 3878     __ cbnz(rscratch1, cont);
 3879     // need a release store here
 3880     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3881     __ stlr(zr, tmp); // set unowned
 3882 
 3883     __ bind(cont);
 3884     // flag == EQ indicates success
 3885     // flag == NE indicates failure
 3886     __ br(Assembler::NE, no_count);
 3887 
 3888     __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
 3889 
 3890     __ bind(no_count);
 3891   %}
 3892 
 3893 %}
 3894 
 3895 //----------FRAME--------------------------------------------------------------
 3896 // Definition of frame structure and management information.
 3897 //
 3898 //  S T A C K   L A Y O U T    Allocators stack-slot number
 3899 //                             |   (to get allocators register number
 3900 //  G  Owned by    |        |  v    add OptoReg::stack0())
 3901 //  r   CALLER     |        |
 3902 //  o     |        +--------+      pad to even-align allocators stack-slot
 3903 //  w     V        |  pad0  |        numbers; owned by CALLER
 3904 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 3905 //  h     ^        |   in   |  5
 3906 //        |        |  args  |  4   Holes in incoming args owned by SELF
 3907 //  |     |        |        |  3
 3908 //  |     |        +--------+
 3909 //  V     |        | old out|      Empty on Intel, window on Sparc
 3910 //        |    old |preserve|      Must be even aligned.
 3911 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 3912 //        |        |   in   |  3   area for Intel ret address
 3913 //     Owned by    |preserve|      Empty on Sparc.
 3914 //       SELF      +--------+
 3915 //        |        |  pad2  |  2   pad to align old SP
 3916 //        |        +--------+  1
 3917 //        |        | locks  |  0
 3918 //        |        +--------+----> OptoReg::stack0(), even aligned
 3919 //        |        |  pad1  | 11   pad to align new SP
 3920 //        |        +--------+
 3921 //        |        |        | 10
 3922 //        |        | spills |  9   spills
 3923 //        V        |        |  8   (pad0 slot for callee)
 3924 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 3925 //        ^        |  out   |  7
 3926 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 3927 //     Owned by    +--------+
 3928 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 3929 //        |    new |preserve|      Must be even-aligned.
 3930 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 3931 //        |        |        |
 3932 //
 3933 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 3934 //         known from SELF's arguments and the Java calling convention.
 3935 //         Region 6-7 is determined per call site.
 3936 // Note 2: If the calling convention leaves holes in the incoming argument
 3937 //         area, those holes are owned by SELF.  Holes in the outgoing area
 3938 //         are owned by the CALLEE.  Holes should not be necessary in the
 3939 //         incoming area, as the Java calling convention is completely under
 3940 //         the control of the AD file.  Doubles can be sorted and packed to
 3941 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 3942 //         varargs C calling conventions.
 3943 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 3944 //         even aligned with pad0 as needed.
 3945 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 3946 //           (the latter is true on Intel but is it false on AArch64?)
 3947 //         region 6-11 is even aligned; it may be padded out more so that
 3948 //         the region from SP to FP meets the minimum stack alignment.
 3949 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 3950 //         alignment.  Region 11, pad1, may be dynamically extended so that
 3951 //         SP meets the minimum alignment.
 3952 
 3953 frame %{
 3954   // These three registers define part of the calling convention
 3955   // between compiled code and the interpreter.
 3956 
 3957   // Inline Cache Register or Method for I2C.
 3958   inline_cache_reg(R12);
 3959 
 3960   // Number of stack slots consumed by locking an object
 3961   sync_stack_slots(2);
 3962 
 3963   // Compiled code's Frame Pointer
 3964   frame_pointer(R31);
 3965 
 3966   // Interpreter stores its frame pointer in a register which is
 3967   // stored to the stack by I2CAdaptors.
 3968   // I2CAdaptors convert from interpreted java to compiled java.
 3969   interpreter_frame_pointer(R29);
 3970 
 3971   // Stack alignment requirement
 3972   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 3973 
 3974   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 3975   // for calls to C.  Supports the var-args backing area for register parms.
 3976   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 3977 
 3978   // The after-PROLOG location of the return address.  Location of
 3979   // return address specifies a type (REG or STACK) and a number
 3980   // representing the register number (i.e. - use a register name) or
 3981   // stack slot.
 3982   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 3983   // Otherwise, it is above the locks and verification slot and alignment word
 3984   // TODO this may well be correct but need to check why that - 2 is there
 3985   // ppc port uses 0 but we definitely need to allow for fixed_slots
 3986   // which folds in the space used for monitors
 3987   return_addr(STACK - 2 +
 3988               align_up((Compile::current()->in_preserve_stack_slots() +
 3989                         Compile::current()->fixed_slots()),
 3990                        stack_alignment_in_slots()));
 3991 
 3992   // Location of compiled Java return values.  Same as C for now.
 3993   return_value
 3994   %{
 3995     // TODO do we allow ideal_reg == Op_RegN???
 3996     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 3997            "only return normal values");
 3998 
 3999     static const int lo[Op_RegL + 1] = { // enum name
 4000       0,                                 // Op_Node
 4001       0,                                 // Op_Set
 4002       R0_num,                            // Op_RegN
 4003       R0_num,                            // Op_RegI
 4004       R0_num,                            // Op_RegP
 4005       V0_num,                            // Op_RegF
 4006       V0_num,                            // Op_RegD
 4007       R0_num                             // Op_RegL
 4008     };
 4009 
 4010     static const int hi[Op_RegL + 1] = { // enum name
 4011       0,                                 // Op_Node
 4012       0,                                 // Op_Set
 4013       OptoReg::Bad,                      // Op_RegN
 4014       OptoReg::Bad,                      // Op_RegI
 4015       R0_H_num,                          // Op_RegP
 4016       OptoReg::Bad,                      // Op_RegF
 4017       V0_H_num,                          // Op_RegD
 4018       R0_H_num                           // Op_RegL
 4019     };
 4020 
 4021     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4022   %}
 4023 %}
 4024 
 4025 //----------ATTRIBUTES---------------------------------------------------------
 4026 //----------Operand Attributes-------------------------------------------------
 4027 op_attrib op_cost(1);        // Required cost attribute
 4028 
 4029 //----------Instruction Attributes---------------------------------------------
 4030 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4031 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4032 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4033                                 // a non-matching short branch variant
 4034                                 // of some long branch?
 4035 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4036                                 // be a power of 2) specifies the
 4037                                 // alignment that some part of the
 4038                                 // instruction (not necessarily the
 4039                                 // start) requires.  If > 1, a
 4040                                 // compute_padding() function must be
 4041                                 // provided for the instruction
 4042 
 4043 //----------OPERANDS-----------------------------------------------------------
 4044 // Operand definitions must precede instruction definitions for correct parsing
 4045 // in the ADLC because operands constitute user defined types which are used in
 4046 // instruction definitions.
 4047 
 4048 //----------Simple Operands----------------------------------------------------
 4049 
 4050 // Integer operands 32 bit
 4051 // 32 bit immediate
 4052 operand immI()
 4053 %{
 4054   match(ConI);
 4055 
 4056   op_cost(0);
 4057   format %{ %}
 4058   interface(CONST_INTER);
 4059 %}
 4060 
 4061 // 32 bit zero
 4062 operand immI0()
 4063 %{
 4064   predicate(n->get_int() == 0);
 4065   match(ConI);
 4066 
 4067   op_cost(0);
 4068   format %{ %}
 4069   interface(CONST_INTER);
 4070 %}
 4071 
 4072 // 32 bit unit increment
 4073 operand immI_1()
 4074 %{
 4075   predicate(n->get_int() == 1);
 4076   match(ConI);
 4077 
 4078   op_cost(0);
 4079   format %{ %}
 4080   interface(CONST_INTER);
 4081 %}
 4082 
 4083 // 32 bit unit decrement
 4084 operand immI_M1()
 4085 %{
 4086   predicate(n->get_int() == -1);
 4087   match(ConI);
 4088 
 4089   op_cost(0);
 4090   format %{ %}
 4091   interface(CONST_INTER);
 4092 %}
 4093 
 4094 // Shift values for add/sub extension shift
 4095 operand immIExt()
 4096 %{
 4097   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4098   match(ConI);
 4099 
 4100   op_cost(0);
 4101   format %{ %}
 4102   interface(CONST_INTER);
 4103 %}
 4104 
 4105 operand immI_gt_1()
 4106 %{
 4107   predicate(n->get_int() > 1);
 4108   match(ConI);
 4109 
 4110   op_cost(0);
 4111   format %{ %}
 4112   interface(CONST_INTER);
 4113 %}
 4114 
 4115 operand immI_le_4()
 4116 %{
 4117   predicate(n->get_int() <= 4);
 4118   match(ConI);
 4119 
 4120   op_cost(0);
 4121   format %{ %}
 4122   interface(CONST_INTER);
 4123 %}
 4124 
 4125 operand immI_16()
 4126 %{
 4127   predicate(n->get_int() == 16);
 4128   match(ConI);
 4129 
 4130   op_cost(0);
 4131   format %{ %}
 4132   interface(CONST_INTER);
 4133 %}
 4134 
 4135 operand immI_24()
 4136 %{
 4137   predicate(n->get_int() == 24);
 4138   match(ConI);
 4139 
 4140   op_cost(0);
 4141   format %{ %}
 4142   interface(CONST_INTER);
 4143 %}
 4144 
 4145 operand immI_32()
 4146 %{
 4147   predicate(n->get_int() == 32);
 4148   match(ConI);
 4149 
 4150   op_cost(0);
 4151   format %{ %}
 4152   interface(CONST_INTER);
 4153 %}
 4154 
 4155 operand immI_48()
 4156 %{
 4157   predicate(n->get_int() == 48);
 4158   match(ConI);
 4159 
 4160   op_cost(0);
 4161   format %{ %}
 4162   interface(CONST_INTER);
 4163 %}
 4164 
 4165 operand immI_56()
 4166 %{
 4167   predicate(n->get_int() == 56);
 4168   match(ConI);
 4169 
 4170   op_cost(0);
 4171   format %{ %}
 4172   interface(CONST_INTER);
 4173 %}
 4174 
 4175 operand immI_63()
 4176 %{
 4177   predicate(n->get_int() == 63);
 4178   match(ConI);
 4179 
 4180   op_cost(0);
 4181   format %{ %}
 4182   interface(CONST_INTER);
 4183 %}
 4184 
 4185 operand immI_64()
 4186 %{
 4187   predicate(n->get_int() == 64);
 4188   match(ConI);
 4189 
 4190   op_cost(0);
 4191   format %{ %}
 4192   interface(CONST_INTER);
 4193 %}
 4194 
 4195 operand immI_255()
 4196 %{
 4197   predicate(n->get_int() == 255);
 4198   match(ConI);
 4199 
 4200   op_cost(0);
 4201   format %{ %}
 4202   interface(CONST_INTER);
 4203 %}
 4204 
 4205 operand immI_65535()
 4206 %{
 4207   predicate(n->get_int() == 65535);
 4208   match(ConI);
 4209 
 4210   op_cost(0);
 4211   format %{ %}
 4212   interface(CONST_INTER);
 4213 %}
 4214 
 4215 operand immI_positive()
 4216 %{
 4217   predicate(n->get_int() > 0);
 4218   match(ConI);
 4219 
 4220   op_cost(0);
 4221   format %{ %}
 4222   interface(CONST_INTER);
 4223 %}
 4224 
 4225 operand immL_255()
 4226 %{
 4227   predicate(n->get_long() == 255L);
 4228   match(ConL);
 4229 
 4230   op_cost(0);
 4231   format %{ %}
 4232   interface(CONST_INTER);
 4233 %}
 4234 
 4235 operand immL_65535()
 4236 %{
 4237   predicate(n->get_long() == 65535L);
 4238   match(ConL);
 4239 
 4240   op_cost(0);
 4241   format %{ %}
 4242   interface(CONST_INTER);
 4243 %}
 4244 
 4245 operand immL_4294967295()
 4246 %{
 4247   predicate(n->get_long() == 4294967295L);
 4248   match(ConL);
 4249 
 4250   op_cost(0);
 4251   format %{ %}
 4252   interface(CONST_INTER);
 4253 %}
 4254 
 4255 operand immL_bitmask()
 4256 %{
 4257   predicate((n->get_long() != 0)
 4258             && ((n->get_long() & 0xc000000000000000l) == 0)
 4259             && is_power_of_2(n->get_long() + 1));
 4260   match(ConL);
 4261 
 4262   op_cost(0);
 4263   format %{ %}
 4264   interface(CONST_INTER);
 4265 %}
 4266 
 4267 operand immI_bitmask()
 4268 %{
 4269   predicate((n->get_int() != 0)
 4270             && ((n->get_int() & 0xc0000000) == 0)
 4271             && is_power_of_2(n->get_int() + 1));
 4272   match(ConI);
 4273 
 4274   op_cost(0);
 4275   format %{ %}
 4276   interface(CONST_INTER);
 4277 %}
 4278 
 4279 operand immL_positive_bitmaskI()
 4280 %{
 4281   predicate((n->get_long() != 0)
 4282             && ((julong)n->get_long() < 0x80000000ULL)
 4283             && is_power_of_2(n->get_long() + 1));
 4284   match(ConL);
 4285 
 4286   op_cost(0);
 4287   format %{ %}
 4288   interface(CONST_INTER);
 4289 %}
 4290 
 4291 // Scale values for scaled offset addressing modes (up to long but not quad)
 4292 operand immIScale()
 4293 %{
 4294   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4295   match(ConI);
 4296 
 4297   op_cost(0);
 4298   format %{ %}
 4299   interface(CONST_INTER);
 4300 %}
 4301 
 4302 // 26 bit signed offset -- for pc-relative branches
 4303 operand immI26()
 4304 %{
 4305   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4306   match(ConI);
 4307 
 4308   op_cost(0);
 4309   format %{ %}
 4310   interface(CONST_INTER);
 4311 %}
 4312 
 4313 // 19 bit signed offset -- for pc-relative loads
 4314 operand immI19()
 4315 %{
 4316   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4317   match(ConI);
 4318 
 4319   op_cost(0);
 4320   format %{ %}
 4321   interface(CONST_INTER);
 4322 %}
 4323 
 4324 // 12 bit unsigned offset -- for base plus immediate loads
 4325 operand immIU12()
 4326 %{
 4327   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4328   match(ConI);
 4329 
 4330   op_cost(0);
 4331   format %{ %}
 4332   interface(CONST_INTER);
 4333 %}
 4334 
 4335 operand immLU12()
 4336 %{
 4337   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4338   match(ConL);
 4339 
 4340   op_cost(0);
 4341   format %{ %}
 4342   interface(CONST_INTER);
 4343 %}
 4344 
 4345 // Offset for scaled or unscaled immediate loads and stores
 4346 operand immIOffset()
 4347 %{
 4348   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4349   match(ConI);
 4350 
 4351   op_cost(0);
 4352   format %{ %}
 4353   interface(CONST_INTER);
 4354 %}
 4355 
 4356 operand immIOffset1()
 4357 %{
 4358   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4359   match(ConI);
 4360 
 4361   op_cost(0);
 4362   format %{ %}
 4363   interface(CONST_INTER);
 4364 %}
 4365 
 4366 operand immIOffset2()
 4367 %{
 4368   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4369   match(ConI);
 4370 
 4371   op_cost(0);
 4372   format %{ %}
 4373   interface(CONST_INTER);
 4374 %}
 4375 
 4376 operand immIOffset4()
 4377 %{
 4378   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4379   match(ConI);
 4380 
 4381   op_cost(0);
 4382   format %{ %}
 4383   interface(CONST_INTER);
 4384 %}
 4385 
 4386 operand immIOffset8()
 4387 %{
 4388   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4389   match(ConI);
 4390 
 4391   op_cost(0);
 4392   format %{ %}
 4393   interface(CONST_INTER);
 4394 %}
 4395 
 4396 operand immIOffset16()
 4397 %{
 4398   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4399   match(ConI);
 4400 
 4401   op_cost(0);
 4402   format %{ %}
 4403   interface(CONST_INTER);
 4404 %}
 4405 
 4406 operand immLoffset()
 4407 %{
 4408   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4409   match(ConL);
 4410 
 4411   op_cost(0);
 4412   format %{ %}
 4413   interface(CONST_INTER);
 4414 %}
 4415 
 4416 operand immLoffset1()
 4417 %{
 4418   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4419   match(ConL);
 4420 
 4421   op_cost(0);
 4422   format %{ %}
 4423   interface(CONST_INTER);
 4424 %}
 4425 
 4426 operand immLoffset2()
 4427 %{
 4428   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4429   match(ConL);
 4430 
 4431   op_cost(0);
 4432   format %{ %}
 4433   interface(CONST_INTER);
 4434 %}
 4435 
 4436 operand immLoffset4()
 4437 %{
 4438   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4439   match(ConL);
 4440 
 4441   op_cost(0);
 4442   format %{ %}
 4443   interface(CONST_INTER);
 4444 %}
 4445 
 4446 operand immLoffset8()
 4447 %{
 4448   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4449   match(ConL);
 4450 
 4451   op_cost(0);
 4452   format %{ %}
 4453   interface(CONST_INTER);
 4454 %}
 4455 
 4456 operand immLoffset16()
 4457 %{
 4458   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4459   match(ConL);
 4460 
 4461   op_cost(0);
 4462   format %{ %}
 4463   interface(CONST_INTER);
 4464 %}
 4465 
 4466 // 8 bit signed value.
 4467 operand immI8()
 4468 %{
 4469   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4470   match(ConI);
 4471 
 4472   op_cost(0);
 4473   format %{ %}
 4474   interface(CONST_INTER);
 4475 %}
 4476 
 4477 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4478 operand immI8_shift8()
 4479 %{
 4480   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4481             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4482   match(ConI);
 4483 
 4484   op_cost(0);
 4485   format %{ %}
 4486   interface(CONST_INTER);
 4487 %}
 4488 
 4489 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4490 operand immL8_shift8()
 4491 %{
 4492   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4493             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4494   match(ConL);
 4495 
 4496   op_cost(0);
 4497   format %{ %}
 4498   interface(CONST_INTER);
 4499 %}
 4500 
 4501 // 8 bit integer valid for vector add sub immediate
 4502 operand immBAddSubV()
 4503 %{
 4504   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4505   match(ConI);
 4506 
 4507   op_cost(0);
 4508   format %{ %}
 4509   interface(CONST_INTER);
 4510 %}
 4511 
 4512 // 32 bit integer valid for add sub immediate
 4513 operand immIAddSub()
 4514 %{
 4515   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4516   match(ConI);
 4517   op_cost(0);
 4518   format %{ %}
 4519   interface(CONST_INTER);
 4520 %}
 4521 
 4522 // 32 bit integer valid for vector add sub immediate
 4523 operand immIAddSubV()
 4524 %{
 4525   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4526   match(ConI);
 4527 
 4528   op_cost(0);
 4529   format %{ %}
 4530   interface(CONST_INTER);
 4531 %}
 4532 
 4533 // 32 bit unsigned integer valid for logical immediate
 4534 
 4535 operand immBLog()
 4536 %{
 4537   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4538   match(ConI);
 4539 
 4540   op_cost(0);
 4541   format %{ %}
 4542   interface(CONST_INTER);
 4543 %}
 4544 
 4545 operand immSLog()
 4546 %{
 4547   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4548   match(ConI);
 4549 
 4550   op_cost(0);
 4551   format %{ %}
 4552   interface(CONST_INTER);
 4553 %}
 4554 
 4555 operand immILog()
 4556 %{
 4557   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4558   match(ConI);
 4559 
 4560   op_cost(0);
 4561   format %{ %}
 4562   interface(CONST_INTER);
 4563 %}
 4564 
 4565 // Integer operands 64 bit
 4566 // 64 bit immediate
 4567 operand immL()
 4568 %{
 4569   match(ConL);
 4570 
 4571   op_cost(0);
 4572   format %{ %}
 4573   interface(CONST_INTER);
 4574 %}
 4575 
 4576 // 64 bit zero
 4577 operand immL0()
 4578 %{
 4579   predicate(n->get_long() == 0);
 4580   match(ConL);
 4581 
 4582   op_cost(0);
 4583   format %{ %}
 4584   interface(CONST_INTER);
 4585 %}
 4586 
 4587 // 64 bit unit increment
 4588 operand immL_1()
 4589 %{
 4590   predicate(n->get_long() == 1);
 4591   match(ConL);
 4592 
 4593   op_cost(0);
 4594   format %{ %}
 4595   interface(CONST_INTER);
 4596 %}
 4597 
 4598 // 64 bit unit decrement
 4599 operand immL_M1()
 4600 %{
 4601   predicate(n->get_long() == -1);
 4602   match(ConL);
 4603 
 4604   op_cost(0);
 4605   format %{ %}
 4606   interface(CONST_INTER);
 4607 %}
 4608 
 4609 // 32 bit offset of pc in thread anchor
 4610 
 4611 operand immL_pc_off()
 4612 %{
 4613   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4614                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4615   match(ConL);
 4616 
 4617   op_cost(0);
 4618   format %{ %}
 4619   interface(CONST_INTER);
 4620 %}
 4621 
 4622 // 64 bit integer valid for add sub immediate
 4623 operand immLAddSub()
 4624 %{
 4625   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4626   match(ConL);
 4627   op_cost(0);
 4628   format %{ %}
 4629   interface(CONST_INTER);
 4630 %}
 4631 
 4632 // 64 bit integer valid for addv subv immediate
 4633 operand immLAddSubV()
 4634 %{
 4635   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4636   match(ConL);
 4637 
 4638   op_cost(0);
 4639   format %{ %}
 4640   interface(CONST_INTER);
 4641 %}
 4642 
 4643 // 64 bit integer valid for logical immediate
 4644 operand immLLog()
 4645 %{
 4646   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4647   match(ConL);
 4648   op_cost(0);
 4649   format %{ %}
 4650   interface(CONST_INTER);
 4651 %}
 4652 
 4653 // Long Immediate: low 32-bit mask
 4654 operand immL_32bits()
 4655 %{
 4656   predicate(n->get_long() == 0xFFFFFFFFL);
 4657   match(ConL);
 4658   op_cost(0);
 4659   format %{ %}
 4660   interface(CONST_INTER);
 4661 %}
 4662 
 4663 // Pointer operands
 4664 // Pointer Immediate
 4665 operand immP()
 4666 %{
 4667   match(ConP);
 4668 
 4669   op_cost(0);
 4670   format %{ %}
 4671   interface(CONST_INTER);
 4672 %}
 4673 
 4674 // NULL Pointer Immediate
 4675 operand immP0()
 4676 %{
 4677   predicate(n->get_ptr() == 0);
 4678   match(ConP);
 4679 
 4680   op_cost(0);
 4681   format %{ %}
 4682   interface(CONST_INTER);
 4683 %}
 4684 
 4685 // Pointer Immediate One
 4686 // this is used in object initialization (initial object header)
 4687 operand immP_1()
 4688 %{
 4689   predicate(n->get_ptr() == 1);
 4690   match(ConP);
 4691 
 4692   op_cost(0);
 4693   format %{ %}
 4694   interface(CONST_INTER);
 4695 %}
 4696 
 4697 // Card Table Byte Map Base
 4698 operand immByteMapBase()
 4699 %{
 4700   // Get base of card map
 4701   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4702             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4703   match(ConP);
 4704 
 4705   op_cost(0);
 4706   format %{ %}
 4707   interface(CONST_INTER);
 4708 %}
 4709 
 4710 // Pointer Immediate Minus One
 4711 // this is used when we want to write the current PC to the thread anchor
 4712 operand immP_M1()
 4713 %{
 4714   predicate(n->get_ptr() == -1);
 4715   match(ConP);
 4716 
 4717   op_cost(0);
 4718   format %{ %}
 4719   interface(CONST_INTER);
 4720 %}
 4721 
 4722 // Pointer Immediate Minus Two
 4723 // this is used when we want to write the current PC to the thread anchor
 4724 operand immP_M2()
 4725 %{
 4726   predicate(n->get_ptr() == -2);
 4727   match(ConP);
 4728 
 4729   op_cost(0);
 4730   format %{ %}
 4731   interface(CONST_INTER);
 4732 %}
 4733 
 4734 // Float and Double operands
 4735 // Double Immediate
 4736 operand immD()
 4737 %{
 4738   match(ConD);
 4739   op_cost(0);
 4740   format %{ %}
 4741   interface(CONST_INTER);
 4742 %}
 4743 
 4744 // Double Immediate: +0.0d
 4745 operand immD0()
 4746 %{
 4747   predicate(jlong_cast(n->getd()) == 0);
 4748   match(ConD);
 4749 
 4750   op_cost(0);
 4751   format %{ %}
 4752   interface(CONST_INTER);
 4753 %}
 4754 
 4755 // constant 'double +0.0'.
 4756 operand immDPacked()
 4757 %{
 4758   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4759   match(ConD);
 4760   op_cost(0);
 4761   format %{ %}
 4762   interface(CONST_INTER);
 4763 %}
 4764 
 4765 // Float Immediate
 4766 operand immF()
 4767 %{
 4768   match(ConF);
 4769   op_cost(0);
 4770   format %{ %}
 4771   interface(CONST_INTER);
 4772 %}
 4773 
 4774 // Float Immediate: +0.0f.
 4775 operand immF0()
 4776 %{
 4777   predicate(jint_cast(n->getf()) == 0);
 4778   match(ConF);
 4779 
 4780   op_cost(0);
 4781   format %{ %}
 4782   interface(CONST_INTER);
 4783 %}
 4784 
 4785 //
 4786 operand immFPacked()
 4787 %{
 4788   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4789   match(ConF);
 4790   op_cost(0);
 4791   format %{ %}
 4792   interface(CONST_INTER);
 4793 %}
 4794 
 4795 // Narrow pointer operands
 4796 // Narrow Pointer Immediate
 4797 operand immN()
 4798 %{
 4799   match(ConN);
 4800 
 4801   op_cost(0);
 4802   format %{ %}
 4803   interface(CONST_INTER);
 4804 %}
 4805 
 4806 // Narrow NULL Pointer Immediate
 4807 operand immN0()
 4808 %{
 4809   predicate(n->get_narrowcon() == 0);
 4810   match(ConN);
 4811 
 4812   op_cost(0);
 4813   format %{ %}
 4814   interface(CONST_INTER);
 4815 %}
 4816 
 4817 operand immNKlass()
 4818 %{
 4819   match(ConNKlass);
 4820 
 4821   op_cost(0);
 4822   format %{ %}
 4823   interface(CONST_INTER);
 4824 %}
 4825 
 4826 // Integer 32 bit Register Operands
 4827 // Integer 32 bitRegister (excludes SP)
 4828 operand iRegI()
 4829 %{
 4830   constraint(ALLOC_IN_RC(any_reg32));
 4831   match(RegI);
 4832   match(iRegINoSp);
 4833   op_cost(0);
 4834   format %{ %}
 4835   interface(REG_INTER);
 4836 %}
 4837 
 4838 // Integer 32 bit Register not Special
 4839 operand iRegINoSp()
 4840 %{
 4841   constraint(ALLOC_IN_RC(no_special_reg32));
 4842   match(RegI);
 4843   op_cost(0);
 4844   format %{ %}
 4845   interface(REG_INTER);
 4846 %}
 4847 
 4848 // Integer 64 bit Register Operands
 4849 // Integer 64 bit Register (includes SP)
 4850 operand iRegL()
 4851 %{
 4852   constraint(ALLOC_IN_RC(any_reg));
 4853   match(RegL);
 4854   match(iRegLNoSp);
 4855   op_cost(0);
 4856   format %{ %}
 4857   interface(REG_INTER);
 4858 %}
 4859 
 4860 // Integer 64 bit Register not Special
 4861 operand iRegLNoSp()
 4862 %{
 4863   constraint(ALLOC_IN_RC(no_special_reg));
 4864   match(RegL);
 4865   match(iRegL_R0);
 4866   format %{ %}
 4867   interface(REG_INTER);
 4868 %}
 4869 
 4870 // Pointer Register Operands
 4871 // Pointer Register
 4872 operand iRegP()
 4873 %{
 4874   constraint(ALLOC_IN_RC(ptr_reg));
 4875   match(RegP);
 4876   match(iRegPNoSp);
 4877   match(iRegP_R0);
 4878   //match(iRegP_R2);
 4879   //match(iRegP_R4);
 4880   //match(iRegP_R5);
 4881   match(thread_RegP);
 4882   op_cost(0);
 4883   format %{ %}
 4884   interface(REG_INTER);
 4885 %}
 4886 
 4887 // Pointer 64 bit Register not Special
 4888 operand iRegPNoSp()
 4889 %{
 4890   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4891   match(RegP);
 4892   // match(iRegP);
 4893   // match(iRegP_R0);
 4894   // match(iRegP_R2);
 4895   // match(iRegP_R4);
 4896   // match(iRegP_R5);
 4897   // match(thread_RegP);
 4898   op_cost(0);
 4899   format %{ %}
 4900   interface(REG_INTER);
 4901 %}
 4902 
 4903 // Pointer 64 bit Register R0 only
 4904 operand iRegP_R0()
 4905 %{
 4906   constraint(ALLOC_IN_RC(r0_reg));
 4907   match(RegP);
 4908   // match(iRegP);
 4909   match(iRegPNoSp);
 4910   op_cost(0);
 4911   format %{ %}
 4912   interface(REG_INTER);
 4913 %}
 4914 
 4915 // Pointer 64 bit Register R1 only
 4916 operand iRegP_R1()
 4917 %{
 4918   constraint(ALLOC_IN_RC(r1_reg));
 4919   match(RegP);
 4920   // match(iRegP);
 4921   match(iRegPNoSp);
 4922   op_cost(0);
 4923   format %{ %}
 4924   interface(REG_INTER);
 4925 %}
 4926 
 4927 // Pointer 64 bit Register R2 only
 4928 operand iRegP_R2()
 4929 %{
 4930   constraint(ALLOC_IN_RC(r2_reg));
 4931   match(RegP);
 4932   // match(iRegP);
 4933   match(iRegPNoSp);
 4934   op_cost(0);
 4935   format %{ %}
 4936   interface(REG_INTER);
 4937 %}
 4938 
 4939 // Pointer 64 bit Register R3 only
 4940 operand iRegP_R3()
 4941 %{
 4942   constraint(ALLOC_IN_RC(r3_reg));
 4943   match(RegP);
 4944   // match(iRegP);
 4945   match(iRegPNoSp);
 4946   op_cost(0);
 4947   format %{ %}
 4948   interface(REG_INTER);
 4949 %}
 4950 
 4951 // Pointer 64 bit Register R4 only
 4952 operand iRegP_R4()
 4953 %{
 4954   constraint(ALLOC_IN_RC(r4_reg));
 4955   match(RegP);
 4956   // match(iRegP);
 4957   match(iRegPNoSp);
 4958   op_cost(0);
 4959   format %{ %}
 4960   interface(REG_INTER);
 4961 %}
 4962 
 4963 // Pointer 64 bit Register R5 only
 4964 operand iRegP_R5()
 4965 %{
 4966   constraint(ALLOC_IN_RC(r5_reg));
 4967   match(RegP);
 4968   // match(iRegP);
 4969   match(iRegPNoSp);
 4970   op_cost(0);
 4971   format %{ %}
 4972   interface(REG_INTER);
 4973 %}
 4974 
 4975 // Pointer 64 bit Register R10 only
 4976 operand iRegP_R10()
 4977 %{
 4978   constraint(ALLOC_IN_RC(r10_reg));
 4979   match(RegP);
 4980   // match(iRegP);
 4981   match(iRegPNoSp);
 4982   op_cost(0);
 4983   format %{ %}
 4984   interface(REG_INTER);
 4985 %}
 4986 
 4987 // Long 64 bit Register R0 only
 4988 operand iRegL_R0()
 4989 %{
 4990   constraint(ALLOC_IN_RC(r0_reg));
 4991   match(RegL);
 4992   match(iRegLNoSp);
 4993   op_cost(0);
 4994   format %{ %}
 4995   interface(REG_INTER);
 4996 %}
 4997 
 4998 // Long 64 bit Register R2 only
 4999 operand iRegL_R2()
 5000 %{
 5001   constraint(ALLOC_IN_RC(r2_reg));
 5002   match(RegL);
 5003   match(iRegLNoSp);
 5004   op_cost(0);
 5005   format %{ %}
 5006   interface(REG_INTER);
 5007 %}
 5008 
 5009 // Long 64 bit Register R3 only
 5010 operand iRegL_R3()
 5011 %{
 5012   constraint(ALLOC_IN_RC(r3_reg));
 5013   match(RegL);
 5014   match(iRegLNoSp);
 5015   op_cost(0);
 5016   format %{ %}
 5017   interface(REG_INTER);
 5018 %}
 5019 
 5020 // Long 64 bit Register R11 only
 5021 operand iRegL_R11()
 5022 %{
 5023   constraint(ALLOC_IN_RC(r11_reg));
 5024   match(RegL);
 5025   match(iRegLNoSp);
 5026   op_cost(0);
 5027   format %{ %}
 5028   interface(REG_INTER);
 5029 %}
 5030 
 5031 // Pointer 64 bit Register FP only
 5032 operand iRegP_FP()
 5033 %{
 5034   constraint(ALLOC_IN_RC(fp_reg));
 5035   match(RegP);
 5036   // match(iRegP);
 5037   op_cost(0);
 5038   format %{ %}
 5039   interface(REG_INTER);
 5040 %}
 5041 
 5042 // Register R0 only
 5043 operand iRegI_R0()
 5044 %{
 5045   constraint(ALLOC_IN_RC(int_r0_reg));
 5046   match(RegI);
 5047   match(iRegINoSp);
 5048   op_cost(0);
 5049   format %{ %}
 5050   interface(REG_INTER);
 5051 %}
 5052 
 5053 // Register R2 only
 5054 operand iRegI_R2()
 5055 %{
 5056   constraint(ALLOC_IN_RC(int_r2_reg));
 5057   match(RegI);
 5058   match(iRegINoSp);
 5059   op_cost(0);
 5060   format %{ %}
 5061   interface(REG_INTER);
 5062 %}
 5063 
 5064 // Register R3 only
 5065 operand iRegI_R3()
 5066 %{
 5067   constraint(ALLOC_IN_RC(int_r3_reg));
 5068   match(RegI);
 5069   match(iRegINoSp);
 5070   op_cost(0);
 5071   format %{ %}
 5072   interface(REG_INTER);
 5073 %}
 5074 
 5075 
 5076 // Register R4 only
 5077 operand iRegI_R4()
 5078 %{
 5079   constraint(ALLOC_IN_RC(int_r4_reg));
 5080   match(RegI);
 5081   match(iRegINoSp);
 5082   op_cost(0);
 5083   format %{ %}
 5084   interface(REG_INTER);
 5085 %}
 5086 
 5087 
 5088 // Pointer Register Operands
 5089 // Narrow Pointer Register
 5090 operand iRegN()
 5091 %{
 5092   constraint(ALLOC_IN_RC(any_reg32));
 5093   match(RegN);
 5094   match(iRegNNoSp);
 5095   op_cost(0);
 5096   format %{ %}
 5097   interface(REG_INTER);
 5098 %}
 5099 
 5100 operand iRegN_R0()
 5101 %{
 5102   constraint(ALLOC_IN_RC(r0_reg));
 5103   match(iRegN);
 5104   op_cost(0);
 5105   format %{ %}
 5106   interface(REG_INTER);
 5107 %}
 5108 
 5109 operand iRegN_R2()
 5110 %{
 5111   constraint(ALLOC_IN_RC(r2_reg));
 5112   match(iRegN);
 5113   op_cost(0);
 5114   format %{ %}
 5115   interface(REG_INTER);
 5116 %}
 5117 
 5118 operand iRegN_R3()
 5119 %{
 5120   constraint(ALLOC_IN_RC(r3_reg));
 5121   match(iRegN);
 5122   op_cost(0);
 5123   format %{ %}
 5124   interface(REG_INTER);
 5125 %}
 5126 
 5127 // Integer 64 bit Register not Special
 5128 operand iRegNNoSp()
 5129 %{
 5130   constraint(ALLOC_IN_RC(no_special_reg32));
 5131   match(RegN);
 5132   op_cost(0);
 5133   format %{ %}
 5134   interface(REG_INTER);
 5135 %}
 5136 
 5137 // heap base register -- used for encoding immN0
 5138 
 5139 operand iRegIHeapbase()
 5140 %{
 5141   constraint(ALLOC_IN_RC(heapbase_reg));
 5142   match(RegI);
 5143   op_cost(0);
 5144   format %{ %}
 5145   interface(REG_INTER);
 5146 %}
 5147 
 5148 // Float Register
 5149 // Float register operands
 5150 operand vRegF()
 5151 %{
 5152   constraint(ALLOC_IN_RC(float_reg));
 5153   match(RegF);
 5154 
 5155   op_cost(0);
 5156   format %{ %}
 5157   interface(REG_INTER);
 5158 %}
 5159 
 5160 // Double Register
 5161 // Double register operands
 5162 operand vRegD()
 5163 %{
 5164   constraint(ALLOC_IN_RC(double_reg));
 5165   match(RegD);
 5166 
 5167   op_cost(0);
 5168   format %{ %}
 5169   interface(REG_INTER);
 5170 %}
 5171 
 5172 // Generic vector class. This will be used for
 5173 // all vector operands, including NEON and SVE.
 5174 operand vReg()
 5175 %{
 5176   constraint(ALLOC_IN_RC(dynamic));
 5177   match(VecA);
 5178   match(VecD);
 5179   match(VecX);
 5180 
 5181   op_cost(0);
 5182   format %{ %}
 5183   interface(REG_INTER);
 5184 %}
 5185 
 5186 operand vecA()
 5187 %{
 5188   constraint(ALLOC_IN_RC(vectora_reg));
 5189   match(VecA);
 5190 
 5191   op_cost(0);
 5192   format %{ %}
 5193   interface(REG_INTER);
 5194 %}
 5195 
 5196 operand vecD()
 5197 %{
 5198   constraint(ALLOC_IN_RC(vectord_reg));
 5199   match(VecD);
 5200 
 5201   op_cost(0);
 5202   format %{ %}
 5203   interface(REG_INTER);
 5204 %}
 5205 
 5206 operand vecX()
 5207 %{
 5208   constraint(ALLOC_IN_RC(vectorx_reg));
 5209   match(VecX);
 5210 
 5211   op_cost(0);
 5212   format %{ %}
 5213   interface(REG_INTER);
 5214 %}
 5215 
 5216 operand vRegD_V0()
 5217 %{
 5218   constraint(ALLOC_IN_RC(v0_reg));
 5219   match(RegD);
 5220   op_cost(0);
 5221   format %{ %}
 5222   interface(REG_INTER);
 5223 %}
 5224 
 5225 operand vRegD_V1()
 5226 %{
 5227   constraint(ALLOC_IN_RC(v1_reg));
 5228   match(RegD);
 5229   op_cost(0);
 5230   format %{ %}
 5231   interface(REG_INTER);
 5232 %}
 5233 
 5234 operand vRegD_V2()
 5235 %{
 5236   constraint(ALLOC_IN_RC(v2_reg));
 5237   match(RegD);
 5238   op_cost(0);
 5239   format %{ %}
 5240   interface(REG_INTER);
 5241 %}
 5242 
 5243 operand vRegD_V3()
 5244 %{
 5245   constraint(ALLOC_IN_RC(v3_reg));
 5246   match(RegD);
 5247   op_cost(0);
 5248   format %{ %}
 5249   interface(REG_INTER);
 5250 %}
 5251 
 5252 operand vRegD_V4()
 5253 %{
 5254   constraint(ALLOC_IN_RC(v4_reg));
 5255   match(RegD);
 5256   op_cost(0);
 5257   format %{ %}
 5258   interface(REG_INTER);
 5259 %}
 5260 
 5261 operand vRegD_V5()
 5262 %{
 5263   constraint(ALLOC_IN_RC(v5_reg));
 5264   match(RegD);
 5265   op_cost(0);
 5266   format %{ %}
 5267   interface(REG_INTER);
 5268 %}
 5269 
 5270 operand vRegD_V6()
 5271 %{
 5272   constraint(ALLOC_IN_RC(v6_reg));
 5273   match(RegD);
 5274   op_cost(0);
 5275   format %{ %}
 5276   interface(REG_INTER);
 5277 %}
 5278 
 5279 operand vRegD_V7()
 5280 %{
 5281   constraint(ALLOC_IN_RC(v7_reg));
 5282   match(RegD);
 5283   op_cost(0);
 5284   format %{ %}
 5285   interface(REG_INTER);
 5286 %}
 5287 
 5288 operand vRegD_V8()
 5289 %{
 5290   constraint(ALLOC_IN_RC(v8_reg));
 5291   match(RegD);
 5292   op_cost(0);
 5293   format %{ %}
 5294   interface(REG_INTER);
 5295 %}
 5296 
 5297 operand vRegD_V9()
 5298 %{
 5299   constraint(ALLOC_IN_RC(v9_reg));
 5300   match(RegD);
 5301   op_cost(0);
 5302   format %{ %}
 5303   interface(REG_INTER);
 5304 %}
 5305 
 5306 operand vRegD_V10()
 5307 %{
 5308   constraint(ALLOC_IN_RC(v10_reg));
 5309   match(RegD);
 5310   op_cost(0);
 5311   format %{ %}
 5312   interface(REG_INTER);
 5313 %}
 5314 
 5315 operand vRegD_V11()
 5316 %{
 5317   constraint(ALLOC_IN_RC(v11_reg));
 5318   match(RegD);
 5319   op_cost(0);
 5320   format %{ %}
 5321   interface(REG_INTER);
 5322 %}
 5323 
 5324 operand vRegD_V12()
 5325 %{
 5326   constraint(ALLOC_IN_RC(v12_reg));
 5327   match(RegD);
 5328   op_cost(0);
 5329   format %{ %}
 5330   interface(REG_INTER);
 5331 %}
 5332 
 5333 operand vRegD_V13()
 5334 %{
 5335   constraint(ALLOC_IN_RC(v13_reg));
 5336   match(RegD);
 5337   op_cost(0);
 5338   format %{ %}
 5339   interface(REG_INTER);
 5340 %}
 5341 
 5342 operand vRegD_V14()
 5343 %{
 5344   constraint(ALLOC_IN_RC(v14_reg));
 5345   match(RegD);
 5346   op_cost(0);
 5347   format %{ %}
 5348   interface(REG_INTER);
 5349 %}
 5350 
 5351 operand vRegD_V15()
 5352 %{
 5353   constraint(ALLOC_IN_RC(v15_reg));
 5354   match(RegD);
 5355   op_cost(0);
 5356   format %{ %}
 5357   interface(REG_INTER);
 5358 %}
 5359 
 5360 operand vRegD_V16()
 5361 %{
 5362   constraint(ALLOC_IN_RC(v16_reg));
 5363   match(RegD);
 5364   op_cost(0);
 5365   format %{ %}
 5366   interface(REG_INTER);
 5367 %}
 5368 
 5369 operand vRegD_V17()
 5370 %{
 5371   constraint(ALLOC_IN_RC(v17_reg));
 5372   match(RegD);
 5373   op_cost(0);
 5374   format %{ %}
 5375   interface(REG_INTER);
 5376 %}
 5377 
 5378 operand vRegD_V18()
 5379 %{
 5380   constraint(ALLOC_IN_RC(v18_reg));
 5381   match(RegD);
 5382   op_cost(0);
 5383   format %{ %}
 5384   interface(REG_INTER);
 5385 %}
 5386 
 5387 operand vRegD_V19()
 5388 %{
 5389   constraint(ALLOC_IN_RC(v19_reg));
 5390   match(RegD);
 5391   op_cost(0);
 5392   format %{ %}
 5393   interface(REG_INTER);
 5394 %}
 5395 
 5396 operand vRegD_V20()
 5397 %{
 5398   constraint(ALLOC_IN_RC(v20_reg));
 5399   match(RegD);
 5400   op_cost(0);
 5401   format %{ %}
 5402   interface(REG_INTER);
 5403 %}
 5404 
 5405 operand vRegD_V21()
 5406 %{
 5407   constraint(ALLOC_IN_RC(v21_reg));
 5408   match(RegD);
 5409   op_cost(0);
 5410   format %{ %}
 5411   interface(REG_INTER);
 5412 %}
 5413 
 5414 operand vRegD_V22()
 5415 %{
 5416   constraint(ALLOC_IN_RC(v22_reg));
 5417   match(RegD);
 5418   op_cost(0);
 5419   format %{ %}
 5420   interface(REG_INTER);
 5421 %}
 5422 
 5423 operand vRegD_V23()
 5424 %{
 5425   constraint(ALLOC_IN_RC(v23_reg));
 5426   match(RegD);
 5427   op_cost(0);
 5428   format %{ %}
 5429   interface(REG_INTER);
 5430 %}
 5431 
 5432 operand vRegD_V24()
 5433 %{
 5434   constraint(ALLOC_IN_RC(v24_reg));
 5435   match(RegD);
 5436   op_cost(0);
 5437   format %{ %}
 5438   interface(REG_INTER);
 5439 %}
 5440 
 5441 operand vRegD_V25()
 5442 %{
 5443   constraint(ALLOC_IN_RC(v25_reg));
 5444   match(RegD);
 5445   op_cost(0);
 5446   format %{ %}
 5447   interface(REG_INTER);
 5448 %}
 5449 
 5450 operand vRegD_V26()
 5451 %{
 5452   constraint(ALLOC_IN_RC(v26_reg));
 5453   match(RegD);
 5454   op_cost(0);
 5455   format %{ %}
 5456   interface(REG_INTER);
 5457 %}
 5458 
 5459 operand vRegD_V27()
 5460 %{
 5461   constraint(ALLOC_IN_RC(v27_reg));
 5462   match(RegD);
 5463   op_cost(0);
 5464   format %{ %}
 5465   interface(REG_INTER);
 5466 %}
 5467 
 5468 operand vRegD_V28()
 5469 %{
 5470   constraint(ALLOC_IN_RC(v28_reg));
 5471   match(RegD);
 5472   op_cost(0);
 5473   format %{ %}
 5474   interface(REG_INTER);
 5475 %}
 5476 
 5477 operand vRegD_V29()
 5478 %{
 5479   constraint(ALLOC_IN_RC(v29_reg));
 5480   match(RegD);
 5481   op_cost(0);
 5482   format %{ %}
 5483   interface(REG_INTER);
 5484 %}
 5485 
 5486 operand vRegD_V30()
 5487 %{
 5488   constraint(ALLOC_IN_RC(v30_reg));
 5489   match(RegD);
 5490   op_cost(0);
 5491   format %{ %}
 5492   interface(REG_INTER);
 5493 %}
 5494 
 5495 operand vRegD_V31()
 5496 %{
 5497   constraint(ALLOC_IN_RC(v31_reg));
 5498   match(RegD);
 5499   op_cost(0);
 5500   format %{ %}
 5501   interface(REG_INTER);
 5502 %}
 5503 
 5504 operand pReg()
 5505 %{
 5506   constraint(ALLOC_IN_RC(pr_reg));
 5507   match(RegVectMask);
 5508   match(pRegGov);
 5509   op_cost(0);
 5510   format %{ %}
 5511   interface(REG_INTER);
 5512 %}
 5513 
 5514 operand pRegGov()
 5515 %{
 5516   constraint(ALLOC_IN_RC(gov_pr));
 5517   match(RegVectMask);
 5518   op_cost(0);
 5519   format %{ %}
 5520   interface(REG_INTER);
 5521 %}
 5522 
 5523 operand pRegGov_P0()
 5524 %{
 5525   constraint(ALLOC_IN_RC(p0_reg));
 5526   match(RegVectMask);
 5527   op_cost(0);
 5528   format %{ %}
 5529   interface(REG_INTER);
 5530 %}
 5531 
 5532 operand pRegGov_P1()
 5533 %{
 5534   constraint(ALLOC_IN_RC(p1_reg));
 5535   match(RegVectMask);
 5536   op_cost(0);
 5537   format %{ %}
 5538   interface(REG_INTER);
 5539 %}
 5540 
 5541 // Flags register, used as output of signed compare instructions
 5542 
 5543 // note that on AArch64 we also use this register as the output for
 5544 // for floating point compare instructions (CmpF CmpD). this ensures
 5545 // that ordered inequality tests use GT, GE, LT or LE none of which
 5546 // pass through cases where the result is unordered i.e. one or both
 5547 // inputs to the compare is a NaN. this means that the ideal code can
 5548 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5549 // (where the comparison should always fail). EQ and NE tests are
 5550 // always generated in ideal code so that unordered folds into the NE
 5551 // case, matching the behaviour of AArch64 NE.
 5552 //
 5553 // This differs from x86 where the outputs of FP compares use a
 5554 // special FP flags registers and where compares based on this
 5555 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5556 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5557 // to explicitly handle the unordered case in branches. x86 also has
 5558 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5559 
 5560 operand rFlagsReg()
 5561 %{
 5562   constraint(ALLOC_IN_RC(int_flags));
 5563   match(RegFlags);
 5564 
 5565   op_cost(0);
 5566   format %{ "RFLAGS" %}
 5567   interface(REG_INTER);
 5568 %}
 5569 
 5570 // Flags register, used as output of unsigned compare instructions
 5571 operand rFlagsRegU()
 5572 %{
 5573   constraint(ALLOC_IN_RC(int_flags));
 5574   match(RegFlags);
 5575 
 5576   op_cost(0);
 5577   format %{ "RFLAGSU" %}
 5578   interface(REG_INTER);
 5579 %}
 5580 
 5581 // Special Registers
 5582 
 5583 // Method Register
 5584 operand inline_cache_RegP(iRegP reg)
 5585 %{
 5586   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5587   match(reg);
 5588   match(iRegPNoSp);
 5589   op_cost(0);
 5590   format %{ %}
 5591   interface(REG_INTER);
 5592 %}
 5593 
 5594 // Thread Register
 5595 operand thread_RegP(iRegP reg)
 5596 %{
 5597   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5598   match(reg);
 5599   op_cost(0);
 5600   format %{ %}
 5601   interface(REG_INTER);
 5602 %}
 5603 
 5604 operand lr_RegP(iRegP reg)
 5605 %{
 5606   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5607   match(reg);
 5608   op_cost(0);
 5609   format %{ %}
 5610   interface(REG_INTER);
 5611 %}
 5612 
 5613 //----------Memory Operands----------------------------------------------------
 5614 
 5615 operand indirect(iRegP reg)
 5616 %{
 5617   constraint(ALLOC_IN_RC(ptr_reg));
 5618   match(reg);
 5619   op_cost(0);
 5620   format %{ "[$reg]" %}
 5621   interface(MEMORY_INTER) %{
 5622     base($reg);
 5623     index(0xffffffff);
 5624     scale(0x0);
 5625     disp(0x0);
 5626   %}
 5627 %}
 5628 
 5629 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5630 %{
 5631   constraint(ALLOC_IN_RC(ptr_reg));
 5632   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5633   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5634   op_cost(0);
 5635   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5636   interface(MEMORY_INTER) %{
 5637     base($reg);
 5638     index($ireg);
 5639     scale($scale);
 5640     disp(0x0);
 5641   %}
 5642 %}
 5643 
 5644 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5645 %{
 5646   constraint(ALLOC_IN_RC(ptr_reg));
 5647   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5648   match(AddP reg (LShiftL lreg scale));
 5649   op_cost(0);
 5650   format %{ "$reg, $lreg lsl($scale)" %}
 5651   interface(MEMORY_INTER) %{
 5652     base($reg);
 5653     index($lreg);
 5654     scale($scale);
 5655     disp(0x0);
 5656   %}
 5657 %}
 5658 
 5659 operand indIndexI2L(iRegP reg, iRegI ireg)
 5660 %{
 5661   constraint(ALLOC_IN_RC(ptr_reg));
 5662   match(AddP reg (ConvI2L ireg));
 5663   op_cost(0);
 5664   format %{ "$reg, $ireg, 0, I2L" %}
 5665   interface(MEMORY_INTER) %{
 5666     base($reg);
 5667     index($ireg);
 5668     scale(0x0);
 5669     disp(0x0);
 5670   %}
 5671 %}
 5672 
 5673 operand indIndex(iRegP reg, iRegL lreg)
 5674 %{
 5675   constraint(ALLOC_IN_RC(ptr_reg));
 5676   match(AddP reg lreg);
 5677   op_cost(0);
 5678   format %{ "$reg, $lreg" %}
 5679   interface(MEMORY_INTER) %{
 5680     base($reg);
 5681     index($lreg);
 5682     scale(0x0);
 5683     disp(0x0);
 5684   %}
 5685 %}
 5686 
 5687 operand indOffI(iRegP reg, immIOffset off)
 5688 %{
 5689   constraint(ALLOC_IN_RC(ptr_reg));
 5690   match(AddP reg off);
 5691   op_cost(0);
 5692   format %{ "[$reg, $off]" %}
 5693   interface(MEMORY_INTER) %{
 5694     base($reg);
 5695     index(0xffffffff);
 5696     scale(0x0);
 5697     disp($off);
 5698   %}
 5699 %}
 5700 
 5701 operand indOffI1(iRegP reg, immIOffset1 off)
 5702 %{
 5703   constraint(ALLOC_IN_RC(ptr_reg));
 5704   match(AddP reg off);
 5705   op_cost(0);
 5706   format %{ "[$reg, $off]" %}
 5707   interface(MEMORY_INTER) %{
 5708     base($reg);
 5709     index(0xffffffff);
 5710     scale(0x0);
 5711     disp($off);
 5712   %}
 5713 %}
 5714 
 5715 operand indOffI2(iRegP reg, immIOffset2 off)
 5716 %{
 5717   constraint(ALLOC_IN_RC(ptr_reg));
 5718   match(AddP reg off);
 5719   op_cost(0);
 5720   format %{ "[$reg, $off]" %}
 5721   interface(MEMORY_INTER) %{
 5722     base($reg);
 5723     index(0xffffffff);
 5724     scale(0x0);
 5725     disp($off);
 5726   %}
 5727 %}
 5728 
 5729 operand indOffI4(iRegP reg, immIOffset4 off)
 5730 %{
 5731   constraint(ALLOC_IN_RC(ptr_reg));
 5732   match(AddP reg off);
 5733   op_cost(0);
 5734   format %{ "[$reg, $off]" %}
 5735   interface(MEMORY_INTER) %{
 5736     base($reg);
 5737     index(0xffffffff);
 5738     scale(0x0);
 5739     disp($off);
 5740   %}
 5741 %}
 5742 
 5743 operand indOffI8(iRegP reg, immIOffset8 off)
 5744 %{
 5745   constraint(ALLOC_IN_RC(ptr_reg));
 5746   match(AddP reg off);
 5747   op_cost(0);
 5748   format %{ "[$reg, $off]" %}
 5749   interface(MEMORY_INTER) %{
 5750     base($reg);
 5751     index(0xffffffff);
 5752     scale(0x0);
 5753     disp($off);
 5754   %}
 5755 %}
 5756 
 5757 operand indOffI16(iRegP reg, immIOffset16 off)
 5758 %{
 5759   constraint(ALLOC_IN_RC(ptr_reg));
 5760   match(AddP reg off);
 5761   op_cost(0);
 5762   format %{ "[$reg, $off]" %}
 5763   interface(MEMORY_INTER) %{
 5764     base($reg);
 5765     index(0xffffffff);
 5766     scale(0x0);
 5767     disp($off);
 5768   %}
 5769 %}
 5770 
 5771 operand indOffL(iRegP reg, immLoffset off)
 5772 %{
 5773   constraint(ALLOC_IN_RC(ptr_reg));
 5774   match(AddP reg off);
 5775   op_cost(0);
 5776   format %{ "[$reg, $off]" %}
 5777   interface(MEMORY_INTER) %{
 5778     base($reg);
 5779     index(0xffffffff);
 5780     scale(0x0);
 5781     disp($off);
 5782   %}
 5783 %}
 5784 
 5785 operand indOffL1(iRegP reg, immLoffset1 off)
 5786 %{
 5787   constraint(ALLOC_IN_RC(ptr_reg));
 5788   match(AddP reg off);
 5789   op_cost(0);
 5790   format %{ "[$reg, $off]" %}
 5791   interface(MEMORY_INTER) %{
 5792     base($reg);
 5793     index(0xffffffff);
 5794     scale(0x0);
 5795     disp($off);
 5796   %}
 5797 %}
 5798 
 5799 operand indOffL2(iRegP reg, immLoffset2 off)
 5800 %{
 5801   constraint(ALLOC_IN_RC(ptr_reg));
 5802   match(AddP reg off);
 5803   op_cost(0);
 5804   format %{ "[$reg, $off]" %}
 5805   interface(MEMORY_INTER) %{
 5806     base($reg);
 5807     index(0xffffffff);
 5808     scale(0x0);
 5809     disp($off);
 5810   %}
 5811 %}
 5812 
 5813 operand indOffL4(iRegP reg, immLoffset4 off)
 5814 %{
 5815   constraint(ALLOC_IN_RC(ptr_reg));
 5816   match(AddP reg off);
 5817   op_cost(0);
 5818   format %{ "[$reg, $off]" %}
 5819   interface(MEMORY_INTER) %{
 5820     base($reg);
 5821     index(0xffffffff);
 5822     scale(0x0);
 5823     disp($off);
 5824   %}
 5825 %}
 5826 
 5827 operand indOffL8(iRegP reg, immLoffset8 off)
 5828 %{
 5829   constraint(ALLOC_IN_RC(ptr_reg));
 5830   match(AddP reg off);
 5831   op_cost(0);
 5832   format %{ "[$reg, $off]" %}
 5833   interface(MEMORY_INTER) %{
 5834     base($reg);
 5835     index(0xffffffff);
 5836     scale(0x0);
 5837     disp($off);
 5838   %}
 5839 %}
 5840 
 5841 operand indOffL16(iRegP reg, immLoffset16 off)
 5842 %{
 5843   constraint(ALLOC_IN_RC(ptr_reg));
 5844   match(AddP reg off);
 5845   op_cost(0);
 5846   format %{ "[$reg, $off]" %}
 5847   interface(MEMORY_INTER) %{
 5848     base($reg);
 5849     index(0xffffffff);
 5850     scale(0x0);
 5851     disp($off);
 5852   %}
 5853 %}
 5854 
 5855 operand indirectN(iRegN reg)
 5856 %{
 5857   predicate(CompressedOops::shift() == 0);
 5858   constraint(ALLOC_IN_RC(ptr_reg));
 5859   match(DecodeN reg);
 5860   op_cost(0);
 5861   format %{ "[$reg]\t# narrow" %}
 5862   interface(MEMORY_INTER) %{
 5863     base($reg);
 5864     index(0xffffffff);
 5865     scale(0x0);
 5866     disp(0x0);
 5867   %}
 5868 %}
 5869 
 5870 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5871 %{
 5872   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5873   constraint(ALLOC_IN_RC(ptr_reg));
 5874   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5875   op_cost(0);
 5876   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5877   interface(MEMORY_INTER) %{
 5878     base($reg);
 5879     index($ireg);
 5880     scale($scale);
 5881     disp(0x0);
 5882   %}
 5883 %}
 5884 
 5885 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5886 %{
 5887   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5888   constraint(ALLOC_IN_RC(ptr_reg));
 5889   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5890   op_cost(0);
 5891   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5892   interface(MEMORY_INTER) %{
 5893     base($reg);
 5894     index($lreg);
 5895     scale($scale);
 5896     disp(0x0);
 5897   %}
 5898 %}
 5899 
 5900 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5901 %{
 5902   predicate(CompressedOops::shift() == 0);
 5903   constraint(ALLOC_IN_RC(ptr_reg));
 5904   match(AddP (DecodeN reg) (ConvI2L ireg));
 5905   op_cost(0);
 5906   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5907   interface(MEMORY_INTER) %{
 5908     base($reg);
 5909     index($ireg);
 5910     scale(0x0);
 5911     disp(0x0);
 5912   %}
 5913 %}
 5914 
 5915 operand indIndexN(iRegN reg, iRegL lreg)
 5916 %{
 5917   predicate(CompressedOops::shift() == 0);
 5918   constraint(ALLOC_IN_RC(ptr_reg));
 5919   match(AddP (DecodeN reg) lreg);
 5920   op_cost(0);
 5921   format %{ "$reg, $lreg\t# narrow" %}
 5922   interface(MEMORY_INTER) %{
 5923     base($reg);
 5924     index($lreg);
 5925     scale(0x0);
 5926     disp(0x0);
 5927   %}
 5928 %}
 5929 
 5930 operand indOffIN(iRegN reg, immIOffset off)
 5931 %{
 5932   predicate(CompressedOops::shift() == 0);
 5933   constraint(ALLOC_IN_RC(ptr_reg));
 5934   match(AddP (DecodeN reg) off);
 5935   op_cost(0);
 5936   format %{ "[$reg, $off]\t# narrow" %}
 5937   interface(MEMORY_INTER) %{
 5938     base($reg);
 5939     index(0xffffffff);
 5940     scale(0x0);
 5941     disp($off);
 5942   %}
 5943 %}
 5944 
 5945 operand indOffLN(iRegN reg, immLoffset off)
 5946 %{
 5947   predicate(CompressedOops::shift() == 0);
 5948   constraint(ALLOC_IN_RC(ptr_reg));
 5949   match(AddP (DecodeN reg) off);
 5950   op_cost(0);
 5951   format %{ "[$reg, $off]\t# narrow" %}
 5952   interface(MEMORY_INTER) %{
 5953     base($reg);
 5954     index(0xffffffff);
 5955     scale(0x0);
 5956     disp($off);
 5957   %}
 5958 %}
 5959 
 5960 
 5961 
 5962 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 5963 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 5964 %{
 5965   constraint(ALLOC_IN_RC(ptr_reg));
 5966   match(AddP reg off);
 5967   op_cost(0);
 5968   format %{ "[$reg, $off]" %}
 5969   interface(MEMORY_INTER) %{
 5970     base($reg);
 5971     index(0xffffffff);
 5972     scale(0x0);
 5973     disp($off);
 5974   %}
 5975 %}
 5976 
 5977 //----------Special Memory Operands--------------------------------------------
 5978 // Stack Slot Operand - This operand is used for loading and storing temporary
 5979 //                      values on the stack where a match requires a value to
 5980 //                      flow through memory.
 5981 operand stackSlotP(sRegP reg)
 5982 %{
 5983   constraint(ALLOC_IN_RC(stack_slots));
 5984   op_cost(100);
 5985   // No match rule because this operand is only generated in matching
 5986   // match(RegP);
 5987   format %{ "[$reg]" %}
 5988   interface(MEMORY_INTER) %{
 5989     base(0x1e);  // RSP
 5990     index(0x0);  // No Index
 5991     scale(0x0);  // No Scale
 5992     disp($reg);  // Stack Offset
 5993   %}
 5994 %}
 5995 
 5996 operand stackSlotI(sRegI reg)
 5997 %{
 5998   constraint(ALLOC_IN_RC(stack_slots));
 5999   // No match rule because this operand is only generated in matching
 6000   // match(RegI);
 6001   format %{ "[$reg]" %}
 6002   interface(MEMORY_INTER) %{
 6003     base(0x1e);  // RSP
 6004     index(0x0);  // No Index
 6005     scale(0x0);  // No Scale
 6006     disp($reg);  // Stack Offset
 6007   %}
 6008 %}
 6009 
 6010 operand stackSlotF(sRegF reg)
 6011 %{
 6012   constraint(ALLOC_IN_RC(stack_slots));
 6013   // No match rule because this operand is only generated in matching
 6014   // match(RegF);
 6015   format %{ "[$reg]" %}
 6016   interface(MEMORY_INTER) %{
 6017     base(0x1e);  // RSP
 6018     index(0x0);  // No Index
 6019     scale(0x0);  // No Scale
 6020     disp($reg);  // Stack Offset
 6021   %}
 6022 %}
 6023 
 6024 operand stackSlotD(sRegD reg)
 6025 %{
 6026   constraint(ALLOC_IN_RC(stack_slots));
 6027   // No match rule because this operand is only generated in matching
 6028   // match(RegD);
 6029   format %{ "[$reg]" %}
 6030   interface(MEMORY_INTER) %{
 6031     base(0x1e);  // RSP
 6032     index(0x0);  // No Index
 6033     scale(0x0);  // No Scale
 6034     disp($reg);  // Stack Offset
 6035   %}
 6036 %}
 6037 
 6038 operand stackSlotL(sRegL reg)
 6039 %{
 6040   constraint(ALLOC_IN_RC(stack_slots));
 6041   // No match rule because this operand is only generated in matching
 6042   // match(RegL);
 6043   format %{ "[$reg]" %}
 6044   interface(MEMORY_INTER) %{
 6045     base(0x1e);  // RSP
 6046     index(0x0);  // No Index
 6047     scale(0x0);  // No Scale
 6048     disp($reg);  // Stack Offset
 6049   %}
 6050 %}
 6051 
 6052 // Operands for expressing Control Flow
 6053 // NOTE: Label is a predefined operand which should not be redefined in
 6054 //       the AD file. It is generically handled within the ADLC.
 6055 
 6056 //----------Conditional Branch Operands----------------------------------------
 6057 // Comparison Op  - This is the operation of the comparison, and is limited to
 6058 //                  the following set of codes:
 6059 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6060 //
 6061 // Other attributes of the comparison, such as unsignedness, are specified
 6062 // by the comparison instruction that sets a condition code flags register.
 6063 // That result is represented by a flags operand whose subtype is appropriate
 6064 // to the unsignedness (etc.) of the comparison.
 6065 //
 6066 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6067 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6068 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6069 
 6070 // used for signed integral comparisons and fp comparisons
 6071 
 6072 operand cmpOp()
 6073 %{
 6074   match(Bool);
 6075 
 6076   format %{ "" %}
 6077   interface(COND_INTER) %{
 6078     equal(0x0, "eq");
 6079     not_equal(0x1, "ne");
 6080     less(0xb, "lt");
 6081     greater_equal(0xa, "ge");
 6082     less_equal(0xd, "le");
 6083     greater(0xc, "gt");
 6084     overflow(0x6, "vs");
 6085     no_overflow(0x7, "vc");
 6086   %}
 6087 %}
 6088 
 6089 // used for unsigned integral comparisons
 6090 
 6091 operand cmpOpU()
 6092 %{
 6093   match(Bool);
 6094 
 6095   format %{ "" %}
 6096   interface(COND_INTER) %{
 6097     equal(0x0, "eq");
 6098     not_equal(0x1, "ne");
 6099     less(0x3, "lo");
 6100     greater_equal(0x2, "hs");
 6101     less_equal(0x9, "ls");
 6102     greater(0x8, "hi");
 6103     overflow(0x6, "vs");
 6104     no_overflow(0x7, "vc");
 6105   %}
 6106 %}
 6107 
 6108 // used for certain integral comparisons which can be
 6109 // converted to cbxx or tbxx instructions
 6110 
 6111 operand cmpOpEqNe()
 6112 %{
 6113   match(Bool);
 6114   op_cost(0);
 6115   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6116             || n->as_Bool()->_test._test == BoolTest::eq);
 6117 
 6118   format %{ "" %}
 6119   interface(COND_INTER) %{
 6120     equal(0x0, "eq");
 6121     not_equal(0x1, "ne");
 6122     less(0xb, "lt");
 6123     greater_equal(0xa, "ge");
 6124     less_equal(0xd, "le");
 6125     greater(0xc, "gt");
 6126     overflow(0x6, "vs");
 6127     no_overflow(0x7, "vc");
 6128   %}
 6129 %}
 6130 
 6131 // used for certain integral comparisons which can be
 6132 // converted to cbxx or tbxx instructions
 6133 
 6134 operand cmpOpLtGe()
 6135 %{
 6136   match(Bool);
 6137   op_cost(0);
 6138 
 6139   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6140             || n->as_Bool()->_test._test == BoolTest::ge);
 6141 
 6142   format %{ "" %}
 6143   interface(COND_INTER) %{
 6144     equal(0x0, "eq");
 6145     not_equal(0x1, "ne");
 6146     less(0xb, "lt");
 6147     greater_equal(0xa, "ge");
 6148     less_equal(0xd, "le");
 6149     greater(0xc, "gt");
 6150     overflow(0x6, "vs");
 6151     no_overflow(0x7, "vc");
 6152   %}
 6153 %}
 6154 
 6155 // used for certain unsigned integral comparisons which can be
 6156 // converted to cbxx or tbxx instructions
 6157 
 6158 operand cmpOpUEqNeLtGe()
 6159 %{
 6160   match(Bool);
 6161   op_cost(0);
 6162 
 6163   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6164             || n->as_Bool()->_test._test == BoolTest::ne
 6165             || n->as_Bool()->_test._test == BoolTest::lt
 6166             || n->as_Bool()->_test._test == BoolTest::ge);
 6167 
 6168   format %{ "" %}
 6169   interface(COND_INTER) %{
 6170     equal(0x0, "eq");
 6171     not_equal(0x1, "ne");
 6172     less(0xb, "lt");
 6173     greater_equal(0xa, "ge");
 6174     less_equal(0xd, "le");
 6175     greater(0xc, "gt");
 6176     overflow(0x6, "vs");
 6177     no_overflow(0x7, "vc");
 6178   %}
 6179 %}
 6180 
 6181 // Special operand allowing long args to int ops to be truncated for free
 6182 
 6183 operand iRegL2I(iRegL reg) %{
 6184 
 6185   op_cost(0);
 6186 
 6187   match(ConvL2I reg);
 6188 
 6189   format %{ "l2i($reg)" %}
 6190 
 6191   interface(REG_INTER)
 6192 %}
 6193 
 6194 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6195 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6196 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6197 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6198 
 6199 //----------OPERAND CLASSES----------------------------------------------------
 6200 // Operand Classes are groups of operands that are used as to simplify
 6201 // instruction definitions by not requiring the AD writer to specify
 6202 // separate instructions for every form of operand when the
 6203 // instruction accepts multiple operand types with the same basic
 6204 // encoding and format. The classic case of this is memory operands.
 6205 
 6206 // memory is used to define read/write location for load/store
 6207 // instruction defs. we can turn a memory op into an Address
 6208 
 6209 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6210                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6211 
 6212 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6213                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6214 
 6215 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6216                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6217 
 6218 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6219                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6220 
 6221 // All of the memory operands. For the pipeline description.
 6222 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6223                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6224                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6225 
 6226 
 6227 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6228 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6229 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6230 // can be elided because the 32-bit instruction will just employ the
 6231 // lower 32 bits anyway.
 6232 //
 6233 // n.b. this does not elide all L2I conversions. if the truncated
 6234 // value is consumed by more than one operation then the ConvL2I
 6235 // cannot be bundled into the consuming nodes so an l2i gets planted
 6236 // (actually a movw $dst $src) and the downstream instructions consume
 6237 // the result of the l2i as an iRegI input. That's a shame since the
 6238 // movw is actually redundant but its not too costly.
 6239 
 6240 opclass iRegIorL2I(iRegI, iRegL2I);
 6241 
 6242 //----------PIPELINE-----------------------------------------------------------
 6243 // Rules which define the behavior of the target architectures pipeline.
 6244 
 6245 // For specific pipelines, eg A53, define the stages of that pipeline
 6246 //pipe_desc(ISS, EX1, EX2, WR);
 6247 #define ISS S0
 6248 #define EX1 S1
 6249 #define EX2 S2
 6250 #define WR  S3
 6251 
 6252 // Integer ALU reg operation
 6253 pipeline %{
 6254 
 6255 attributes %{
 6256   // ARM instructions are of fixed length
 6257   fixed_size_instructions;        // Fixed size instructions TODO does
 6258   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6259   // ARM instructions come in 32-bit word units
 6260   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6261   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6262   instruction_fetch_units = 1;       // of 64 bytes
 6263 
 6264   // List of nop instructions
 6265   nops( MachNop );
 6266 %}
 6267 
 6268 // We don't use an actual pipeline model so don't care about resources
 6269 // or description. we do use pipeline classes to introduce fixed
 6270 // latencies
 6271 
 6272 //----------RESOURCES----------------------------------------------------------
 6273 // Resources are the functional units available to the machine
 6274 
 6275 resources( INS0, INS1, INS01 = INS0 | INS1,
 6276            ALU0, ALU1, ALU = ALU0 | ALU1,
 6277            MAC,
 6278            DIV,
 6279            BRANCH,
 6280            LDST,
 6281            NEON_FP);
 6282 
 6283 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6284 // Pipeline Description specifies the stages in the machine's pipeline
 6285 
 6286 // Define the pipeline as a generic 6 stage pipeline
 6287 pipe_desc(S0, S1, S2, S3, S4, S5);
 6288 
 6289 //----------PIPELINE CLASSES---------------------------------------------------
 6290 // Pipeline Classes describe the stages in which input and output are
 6291 // referenced by the hardware pipeline.
 6292 
 6293 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6294 %{
 6295   single_instruction;
 6296   src1   : S1(read);
 6297   src2   : S2(read);
 6298   dst    : S5(write);
 6299   INS01  : ISS;
 6300   NEON_FP : S5;
 6301 %}
 6302 
 6303 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6304 %{
 6305   single_instruction;
 6306   src1   : S1(read);
 6307   src2   : S2(read);
 6308   dst    : S5(write);
 6309   INS01  : ISS;
 6310   NEON_FP : S5;
 6311 %}
 6312 
 6313 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6314 %{
 6315   single_instruction;
 6316   src    : S1(read);
 6317   dst    : S5(write);
 6318   INS01  : ISS;
 6319   NEON_FP : S5;
 6320 %}
 6321 
 6322 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6323 %{
 6324   single_instruction;
 6325   src    : S1(read);
 6326   dst    : S5(write);
 6327   INS01  : ISS;
 6328   NEON_FP : S5;
 6329 %}
 6330 
 6331 pipe_class fp_d2f(vRegF dst, vRegD src)
 6332 %{
 6333   single_instruction;
 6334   src    : S1(read);
 6335   dst    : S5(write);
 6336   INS01  : ISS;
 6337   NEON_FP : S5;
 6338 %}
 6339 
 6340 pipe_class fp_f2d(vRegD dst, vRegF src)
 6341 %{
 6342   single_instruction;
 6343   src    : S1(read);
 6344   dst    : S5(write);
 6345   INS01  : ISS;
 6346   NEON_FP : S5;
 6347 %}
 6348 
 6349 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6350 %{
 6351   single_instruction;
 6352   src    : S1(read);
 6353   dst    : S5(write);
 6354   INS01  : ISS;
 6355   NEON_FP : S5;
 6356 %}
 6357 
 6358 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6359 %{
 6360   single_instruction;
 6361   src    : S1(read);
 6362   dst    : S5(write);
 6363   INS01  : ISS;
 6364   NEON_FP : S5;
 6365 %}
 6366 
 6367 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6368 %{
 6369   single_instruction;
 6370   src    : S1(read);
 6371   dst    : S5(write);
 6372   INS01  : ISS;
 6373   NEON_FP : S5;
 6374 %}
 6375 
 6376 pipe_class fp_l2f(vRegF dst, iRegL src)
 6377 %{
 6378   single_instruction;
 6379   src    : S1(read);
 6380   dst    : S5(write);
 6381   INS01  : ISS;
 6382   NEON_FP : S5;
 6383 %}
 6384 
 6385 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6386 %{
 6387   single_instruction;
 6388   src    : S1(read);
 6389   dst    : S5(write);
 6390   INS01  : ISS;
 6391   NEON_FP : S5;
 6392 %}
 6393 
 6394 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6395 %{
 6396   single_instruction;
 6397   src    : S1(read);
 6398   dst    : S5(write);
 6399   INS01  : ISS;
 6400   NEON_FP : S5;
 6401 %}
 6402 
 6403 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6404 %{
 6405   single_instruction;
 6406   src    : S1(read);
 6407   dst    : S5(write);
 6408   INS01  : ISS;
 6409   NEON_FP : S5;
 6410 %}
 6411 
 6412 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6413 %{
 6414   single_instruction;
 6415   src    : S1(read);
 6416   dst    : S5(write);
 6417   INS01  : ISS;
 6418   NEON_FP : S5;
 6419 %}
 6420 
 6421 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6422 %{
 6423   single_instruction;
 6424   src1   : S1(read);
 6425   src2   : S2(read);
 6426   dst    : S5(write);
 6427   INS0   : ISS;
 6428   NEON_FP : S5;
 6429 %}
 6430 
 6431 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6432 %{
 6433   single_instruction;
 6434   src1   : S1(read);
 6435   src2   : S2(read);
 6436   dst    : S5(write);
 6437   INS0   : ISS;
 6438   NEON_FP : S5;
 6439 %}
 6440 
 6441 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6442 %{
 6443   single_instruction;
 6444   cr     : S1(read);
 6445   src1   : S1(read);
 6446   src2   : S1(read);
 6447   dst    : S3(write);
 6448   INS01  : ISS;
 6449   NEON_FP : S3;
 6450 %}
 6451 
 6452 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6453 %{
 6454   single_instruction;
 6455   cr     : S1(read);
 6456   src1   : S1(read);
 6457   src2   : S1(read);
 6458   dst    : S3(write);
 6459   INS01  : ISS;
 6460   NEON_FP : S3;
 6461 %}
 6462 
 6463 pipe_class fp_imm_s(vRegF dst)
 6464 %{
 6465   single_instruction;
 6466   dst    : S3(write);
 6467   INS01  : ISS;
 6468   NEON_FP : S3;
 6469 %}
 6470 
 6471 pipe_class fp_imm_d(vRegD dst)
 6472 %{
 6473   single_instruction;
 6474   dst    : S3(write);
 6475   INS01  : ISS;
 6476   NEON_FP : S3;
 6477 %}
 6478 
 6479 pipe_class fp_load_constant_s(vRegF dst)
 6480 %{
 6481   single_instruction;
 6482   dst    : S4(write);
 6483   INS01  : ISS;
 6484   NEON_FP : S4;
 6485 %}
 6486 
 6487 pipe_class fp_load_constant_d(vRegD dst)
 6488 %{
 6489   single_instruction;
 6490   dst    : S4(write);
 6491   INS01  : ISS;
 6492   NEON_FP : S4;
 6493 %}
 6494 
 6495 //------- Integer ALU operations --------------------------
 6496 
 6497 // Integer ALU reg-reg operation
 6498 // Operands needed in EX1, result generated in EX2
 6499 // Eg.  ADD     x0, x1, x2
 6500 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6501 %{
 6502   single_instruction;
 6503   dst    : EX2(write);
 6504   src1   : EX1(read);
 6505   src2   : EX1(read);
 6506   INS01  : ISS; // Dual issue as instruction 0 or 1
 6507   ALU    : EX2;
 6508 %}
 6509 
 6510 // Integer ALU reg-reg operation with constant shift
 6511 // Shifted register must be available in LATE_ISS instead of EX1
 6512 // Eg.  ADD     x0, x1, x2, LSL #2
 6513 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6514 %{
 6515   single_instruction;
 6516   dst    : EX2(write);
 6517   src1   : EX1(read);
 6518   src2   : ISS(read);
 6519   INS01  : ISS;
 6520   ALU    : EX2;
 6521 %}
 6522 
 6523 // Integer ALU reg operation with constant shift
 6524 // Eg.  LSL     x0, x1, #shift
 6525 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6526 %{
 6527   single_instruction;
 6528   dst    : EX2(write);
 6529   src1   : ISS(read);
 6530   INS01  : ISS;
 6531   ALU    : EX2;
 6532 %}
 6533 
 6534 // Integer ALU reg-reg operation with variable shift
 6535 // Both operands must be available in LATE_ISS instead of EX1
 6536 // Result is available in EX1 instead of EX2
 6537 // Eg.  LSLV    x0, x1, x2
 6538 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6539 %{
 6540   single_instruction;
 6541   dst    : EX1(write);
 6542   src1   : ISS(read);
 6543   src2   : ISS(read);
 6544   INS01  : ISS;
 6545   ALU    : EX1;
 6546 %}
 6547 
 6548 // Integer ALU reg-reg operation with extract
 6549 // As for _vshift above, but result generated in EX2
 6550 // Eg.  EXTR    x0, x1, x2, #N
 6551 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6552 %{
 6553   single_instruction;
 6554   dst    : EX2(write);
 6555   src1   : ISS(read);
 6556   src2   : ISS(read);
 6557   INS1   : ISS; // Can only dual issue as Instruction 1
 6558   ALU    : EX1;
 6559 %}
 6560 
 6561 // Integer ALU reg operation
 6562 // Eg.  NEG     x0, x1
 6563 pipe_class ialu_reg(iRegI dst, iRegI src)
 6564 %{
 6565   single_instruction;
 6566   dst    : EX2(write);
 6567   src    : EX1(read);
 6568   INS01  : ISS;
 6569   ALU    : EX2;
 6570 %}
 6571 
 6572 // Integer ALU reg mmediate operation
 6573 // Eg.  ADD     x0, x1, #N
 6574 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6575 %{
 6576   single_instruction;
 6577   dst    : EX2(write);
 6578   src1   : EX1(read);
 6579   INS01  : ISS;
 6580   ALU    : EX2;
 6581 %}
 6582 
 6583 // Integer ALU immediate operation (no source operands)
 6584 // Eg.  MOV     x0, #N
 6585 pipe_class ialu_imm(iRegI dst)
 6586 %{
 6587   single_instruction;
 6588   dst    : EX1(write);
 6589   INS01  : ISS;
 6590   ALU    : EX1;
 6591 %}
 6592 
 6593 //------- Compare operation -------------------------------
 6594 
 6595 // Compare reg-reg
 6596 // Eg.  CMP     x0, x1
 6597 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6598 %{
 6599   single_instruction;
 6600 //  fixed_latency(16);
 6601   cr     : EX2(write);
 6602   op1    : EX1(read);
 6603   op2    : EX1(read);
 6604   INS01  : ISS;
 6605   ALU    : EX2;
 6606 %}
 6607 
 6608 // Compare reg-reg
 6609 // Eg.  CMP     x0, #N
 6610 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6611 %{
 6612   single_instruction;
 6613 //  fixed_latency(16);
 6614   cr     : EX2(write);
 6615   op1    : EX1(read);
 6616   INS01  : ISS;
 6617   ALU    : EX2;
 6618 %}
 6619 
 6620 //------- Conditional instructions ------------------------
 6621 
 6622 // Conditional no operands
 6623 // Eg.  CSINC   x0, zr, zr, <cond>
 6624 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6625 %{
 6626   single_instruction;
 6627   cr     : EX1(read);
 6628   dst    : EX2(write);
 6629   INS01  : ISS;
 6630   ALU    : EX2;
 6631 %}
 6632 
 6633 // Conditional 2 operand
 6634 // EG.  CSEL    X0, X1, X2, <cond>
 6635 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6636 %{
 6637   single_instruction;
 6638   cr     : EX1(read);
 6639   src1   : EX1(read);
 6640   src2   : EX1(read);
 6641   dst    : EX2(write);
 6642   INS01  : ISS;
 6643   ALU    : EX2;
 6644 %}
 6645 
 6646 // Conditional 2 operand
 6647 // EG.  CSEL    X0, X1, X2, <cond>
 6648 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6649 %{
 6650   single_instruction;
 6651   cr     : EX1(read);
 6652   src    : EX1(read);
 6653   dst    : EX2(write);
 6654   INS01  : ISS;
 6655   ALU    : EX2;
 6656 %}
 6657 
 6658 //------- Multiply pipeline operations --------------------
 6659 
 6660 // Multiply reg-reg
 6661 // Eg.  MUL     w0, w1, w2
 6662 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6663 %{
 6664   single_instruction;
 6665   dst    : WR(write);
 6666   src1   : ISS(read);
 6667   src2   : ISS(read);
 6668   INS01  : ISS;
 6669   MAC    : WR;
 6670 %}
 6671 
 6672 // Multiply accumulate
 6673 // Eg.  MADD    w0, w1, w2, w3
 6674 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6675 %{
 6676   single_instruction;
 6677   dst    : WR(write);
 6678   src1   : ISS(read);
 6679   src2   : ISS(read);
 6680   src3   : ISS(read);
 6681   INS01  : ISS;
 6682   MAC    : WR;
 6683 %}
 6684 
 6685 // Eg.  MUL     w0, w1, w2
 6686 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6687 %{
 6688   single_instruction;
 6689   fixed_latency(3); // Maximum latency for 64 bit mul
 6690   dst    : WR(write);
 6691   src1   : ISS(read);
 6692   src2   : ISS(read);
 6693   INS01  : ISS;
 6694   MAC    : WR;
 6695 %}
 6696 
 6697 // Multiply accumulate
 6698 // Eg.  MADD    w0, w1, w2, w3
 6699 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6700 %{
 6701   single_instruction;
 6702   fixed_latency(3); // Maximum latency for 64 bit mul
 6703   dst    : WR(write);
 6704   src1   : ISS(read);
 6705   src2   : ISS(read);
 6706   src3   : ISS(read);
 6707   INS01  : ISS;
 6708   MAC    : WR;
 6709 %}
 6710 
 6711 //------- Divide pipeline operations --------------------
 6712 
 6713 // Eg.  SDIV    w0, w1, w2
 6714 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6715 %{
 6716   single_instruction;
 6717   fixed_latency(8); // Maximum latency for 32 bit divide
 6718   dst    : WR(write);
 6719   src1   : ISS(read);
 6720   src2   : ISS(read);
 6721   INS0   : ISS; // Can only dual issue as instruction 0
 6722   DIV    : WR;
 6723 %}
 6724 
 6725 // Eg.  SDIV    x0, x1, x2
 6726 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6727 %{
 6728   single_instruction;
 6729   fixed_latency(16); // Maximum latency for 64 bit divide
 6730   dst    : WR(write);
 6731   src1   : ISS(read);
 6732   src2   : ISS(read);
 6733   INS0   : ISS; // Can only dual issue as instruction 0
 6734   DIV    : WR;
 6735 %}
 6736 
 6737 //------- Load pipeline operations ------------------------
 6738 
 6739 // Load - prefetch
 6740 // Eg.  PFRM    <mem>
 6741 pipe_class iload_prefetch(memory mem)
 6742 %{
 6743   single_instruction;
 6744   mem    : ISS(read);
 6745   INS01  : ISS;
 6746   LDST   : WR;
 6747 %}
 6748 
 6749 // Load - reg, mem
 6750 // Eg.  LDR     x0, <mem>
 6751 pipe_class iload_reg_mem(iRegI dst, memory mem)
 6752 %{
 6753   single_instruction;
 6754   dst    : WR(write);
 6755   mem    : ISS(read);
 6756   INS01  : ISS;
 6757   LDST   : WR;
 6758 %}
 6759 
 6760 // Load - reg, reg
 6761 // Eg.  LDR     x0, [sp, x1]
 6762 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 6763 %{
 6764   single_instruction;
 6765   dst    : WR(write);
 6766   src    : ISS(read);
 6767   INS01  : ISS;
 6768   LDST   : WR;
 6769 %}
 6770 
 6771 //------- Store pipeline operations -----------------------
 6772 
 6773 // Store - zr, mem
 6774 // Eg.  STR     zr, <mem>
 6775 pipe_class istore_mem(memory mem)
 6776 %{
 6777   single_instruction;
 6778   mem    : ISS(read);
 6779   INS01  : ISS;
 6780   LDST   : WR;
 6781 %}
 6782 
 6783 // Store - reg, mem
 6784 // Eg.  STR     x0, <mem>
 6785 pipe_class istore_reg_mem(iRegI src, memory mem)
 6786 %{
 6787   single_instruction;
 6788   mem    : ISS(read);
 6789   src    : EX2(read);
 6790   INS01  : ISS;
 6791   LDST   : WR;
 6792 %}
 6793 
 6794 // Store - reg, reg
 6795 // Eg. STR      x0, [sp, x1]
 6796 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 6797 %{
 6798   single_instruction;
 6799   dst    : ISS(read);
 6800   src    : EX2(read);
 6801   INS01  : ISS;
 6802   LDST   : WR;
 6803 %}
 6804 
 6805 //------- Store pipeline operations -----------------------
 6806 
 6807 // Branch
 6808 pipe_class pipe_branch()
 6809 %{
 6810   single_instruction;
 6811   INS01  : ISS;
 6812   BRANCH : EX1;
 6813 %}
 6814 
 6815 // Conditional branch
 6816 pipe_class pipe_branch_cond(rFlagsReg cr)
 6817 %{
 6818   single_instruction;
 6819   cr     : EX1(read);
 6820   INS01  : ISS;
 6821   BRANCH : EX1;
 6822 %}
 6823 
 6824 // Compare & Branch
 6825 // EG.  CBZ/CBNZ
 6826 pipe_class pipe_cmp_branch(iRegI op1)
 6827 %{
 6828   single_instruction;
 6829   op1    : EX1(read);
 6830   INS01  : ISS;
 6831   BRANCH : EX1;
 6832 %}
 6833 
 6834 //------- Synchronisation operations ----------------------
 6835 
 6836 // Any operation requiring serialization.
 6837 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 6838 pipe_class pipe_serial()
 6839 %{
 6840   single_instruction;
 6841   force_serialization;
 6842   fixed_latency(16);
 6843   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6844   LDST   : WR;
 6845 %}
 6846 
 6847 // Generic big/slow expanded idiom - also serialized
 6848 pipe_class pipe_slow()
 6849 %{
 6850   instruction_count(10);
 6851   multiple_bundles;
 6852   force_serialization;
 6853   fixed_latency(16);
 6854   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6855   LDST   : WR;
 6856 %}
 6857 
 6858 // Empty pipeline class
 6859 pipe_class pipe_class_empty()
 6860 %{
 6861   single_instruction;
 6862   fixed_latency(0);
 6863 %}
 6864 
 6865 // Default pipeline class.
 6866 pipe_class pipe_class_default()
 6867 %{
 6868   single_instruction;
 6869   fixed_latency(2);
 6870 %}
 6871 
 6872 // Pipeline class for compares.
 6873 pipe_class pipe_class_compare()
 6874 %{
 6875   single_instruction;
 6876   fixed_latency(16);
 6877 %}
 6878 
 6879 // Pipeline class for memory operations.
 6880 pipe_class pipe_class_memory()
 6881 %{
 6882   single_instruction;
 6883   fixed_latency(16);
 6884 %}
 6885 
 6886 // Pipeline class for call.
 6887 pipe_class pipe_class_call()
 6888 %{
 6889   single_instruction;
 6890   fixed_latency(100);
 6891 %}
 6892 
 6893 // Define the class for the Nop node.
 6894 define %{
 6895    MachNop = pipe_class_empty;
 6896 %}
 6897 
 6898 %}
 6899 //----------INSTRUCTIONS-------------------------------------------------------
 6900 //
 6901 // match      -- States which machine-independent subtree may be replaced
 6902 //               by this instruction.
 6903 // ins_cost   -- The estimated cost of this instruction is used by instruction
 6904 //               selection to identify a minimum cost tree of machine
 6905 //               instructions that matches a tree of machine-independent
 6906 //               instructions.
 6907 // format     -- A string providing the disassembly for this instruction.
 6908 //               The value of an instruction's operand may be inserted
 6909 //               by referring to it with a '$' prefix.
 6910 // opcode     -- Three instruction opcodes may be provided.  These are referred
 6911 //               to within an encode class as $primary, $secondary, and $tertiary
 6912 //               rrspectively.  The primary opcode is commonly used to
 6913 //               indicate the type of machine instruction, while secondary
 6914 //               and tertiary are often used for prefix options or addressing
 6915 //               modes.
 6916 // ins_encode -- A list of encode classes with parameters. The encode class
 6917 //               name must have been defined in an 'enc_class' specification
 6918 //               in the encode section of the architecture description.
 6919 
 6920 // ============================================================================
 6921 // Memory (Load/Store) Instructions
 6922 
 6923 // Load Instructions
 6924 
 6925 // Load Byte (8 bit signed)
 6926 instruct loadB(iRegINoSp dst, memory1 mem)
 6927 %{
 6928   match(Set dst (LoadB mem));
 6929   predicate(!needs_acquiring_load(n));
 6930 
 6931   ins_cost(4 * INSN_COST);
 6932   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 6933 
 6934   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 6935 
 6936   ins_pipe(iload_reg_mem);
 6937 %}
 6938 
 6939 // Load Byte (8 bit signed) into long
 6940 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 6941 %{
 6942   match(Set dst (ConvI2L (LoadB mem)));
 6943   predicate(!needs_acquiring_load(n->in(1)));
 6944 
 6945   ins_cost(4 * INSN_COST);
 6946   format %{ "ldrsb  $dst, $mem\t# byte" %}
 6947 
 6948   ins_encode(aarch64_enc_ldrsb(dst, mem));
 6949 
 6950   ins_pipe(iload_reg_mem);
 6951 %}
 6952 
 6953 // Load Byte (8 bit unsigned)
 6954 instruct loadUB(iRegINoSp dst, memory1 mem)
 6955 %{
 6956   match(Set dst (LoadUB mem));
 6957   predicate(!needs_acquiring_load(n));
 6958 
 6959   ins_cost(4 * INSN_COST);
 6960   format %{ "ldrbw  $dst, $mem\t# byte" %}
 6961 
 6962   ins_encode(aarch64_enc_ldrb(dst, mem));
 6963 
 6964   ins_pipe(iload_reg_mem);
 6965 %}
 6966 
 6967 // Load Byte (8 bit unsigned) into long
 6968 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 6969 %{
 6970   match(Set dst (ConvI2L (LoadUB mem)));
 6971   predicate(!needs_acquiring_load(n->in(1)));
 6972 
 6973   ins_cost(4 * INSN_COST);
 6974   format %{ "ldrb  $dst, $mem\t# byte" %}
 6975 
 6976   ins_encode(aarch64_enc_ldrb(dst, mem));
 6977 
 6978   ins_pipe(iload_reg_mem);
 6979 %}
 6980 
 6981 // Load Short (16 bit signed)
 6982 instruct loadS(iRegINoSp dst, memory2 mem)
 6983 %{
 6984   match(Set dst (LoadS mem));
 6985   predicate(!needs_acquiring_load(n));
 6986 
 6987   ins_cost(4 * INSN_COST);
 6988   format %{ "ldrshw  $dst, $mem\t# short" %}
 6989 
 6990   ins_encode(aarch64_enc_ldrshw(dst, mem));
 6991 
 6992   ins_pipe(iload_reg_mem);
 6993 %}
 6994 
 6995 // Load Short (16 bit signed) into long
 6996 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 6997 %{
 6998   match(Set dst (ConvI2L (LoadS mem)));
 6999   predicate(!needs_acquiring_load(n->in(1)));
 7000 
 7001   ins_cost(4 * INSN_COST);
 7002   format %{ "ldrsh  $dst, $mem\t# short" %}
 7003 
 7004   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7005 
 7006   ins_pipe(iload_reg_mem);
 7007 %}
 7008 
 7009 // Load Char (16 bit unsigned)
 7010 instruct loadUS(iRegINoSp dst, memory2 mem)
 7011 %{
 7012   match(Set dst (LoadUS mem));
 7013   predicate(!needs_acquiring_load(n));
 7014 
 7015   ins_cost(4 * INSN_COST);
 7016   format %{ "ldrh  $dst, $mem\t# short" %}
 7017 
 7018   ins_encode(aarch64_enc_ldrh(dst, mem));
 7019 
 7020   ins_pipe(iload_reg_mem);
 7021 %}
 7022 
 7023 // Load Short/Char (16 bit unsigned) into long
 7024 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7025 %{
 7026   match(Set dst (ConvI2L (LoadUS mem)));
 7027   predicate(!needs_acquiring_load(n->in(1)));
 7028 
 7029   ins_cost(4 * INSN_COST);
 7030   format %{ "ldrh  $dst, $mem\t# short" %}
 7031 
 7032   ins_encode(aarch64_enc_ldrh(dst, mem));
 7033 
 7034   ins_pipe(iload_reg_mem);
 7035 %}
 7036 
 7037 // Load Integer (32 bit signed)
 7038 instruct loadI(iRegINoSp dst, memory4 mem)
 7039 %{
 7040   match(Set dst (LoadI mem));
 7041   predicate(!needs_acquiring_load(n));
 7042 
 7043   ins_cost(4 * INSN_COST);
 7044   format %{ "ldrw  $dst, $mem\t# int" %}
 7045 
 7046   ins_encode(aarch64_enc_ldrw(dst, mem));
 7047 
 7048   ins_pipe(iload_reg_mem);
 7049 %}
 7050 
 7051 // Load Integer (32 bit signed) into long
 7052 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7053 %{
 7054   match(Set dst (ConvI2L (LoadI mem)));
 7055   predicate(!needs_acquiring_load(n->in(1)));
 7056 
 7057   ins_cost(4 * INSN_COST);
 7058   format %{ "ldrsw  $dst, $mem\t# int" %}
 7059 
 7060   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7061 
 7062   ins_pipe(iload_reg_mem);
 7063 %}
 7064 
 7065 // Load Integer (32 bit unsigned) into long
 7066 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7067 %{
 7068   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7069   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7070 
 7071   ins_cost(4 * INSN_COST);
 7072   format %{ "ldrw  $dst, $mem\t# int" %}
 7073 
 7074   ins_encode(aarch64_enc_ldrw(dst, mem));
 7075 
 7076   ins_pipe(iload_reg_mem);
 7077 %}
 7078 
 7079 // Load Long (64 bit signed)
 7080 instruct loadL(iRegLNoSp dst, memory8 mem)
 7081 %{
 7082   match(Set dst (LoadL mem));
 7083   predicate(!needs_acquiring_load(n));
 7084 
 7085   ins_cost(4 * INSN_COST);
 7086   format %{ "ldr  $dst, $mem\t# int" %}
 7087 
 7088   ins_encode(aarch64_enc_ldr(dst, mem));
 7089 
 7090   ins_pipe(iload_reg_mem);
 7091 %}
 7092 
 7093 // Load Range
 7094 instruct loadRange(iRegINoSp dst, memory4 mem)
 7095 %{
 7096   match(Set dst (LoadRange mem));
 7097 
 7098   ins_cost(4 * INSN_COST);
 7099   format %{ "ldrw  $dst, $mem\t# range" %}
 7100 
 7101   ins_encode(aarch64_enc_ldrw(dst, mem));
 7102 
 7103   ins_pipe(iload_reg_mem);
 7104 %}
 7105 
 7106 // Load Pointer
 7107 instruct loadP(iRegPNoSp dst, memory8 mem)
 7108 %{
 7109   match(Set dst (LoadP mem));
 7110   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7111 
 7112   ins_cost(4 * INSN_COST);
 7113   format %{ "ldr  $dst, $mem\t# ptr" %}
 7114 
 7115   ins_encode(aarch64_enc_ldr(dst, mem));
 7116 
 7117   ins_pipe(iload_reg_mem);
 7118 %}
 7119 
 7120 // Load Compressed Pointer
 7121 instruct loadN(iRegNNoSp dst, memory4 mem)
 7122 %{
 7123   match(Set dst (LoadN mem));
 7124   predicate(!needs_acquiring_load(n));
 7125 
 7126   ins_cost(4 * INSN_COST);
 7127   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7128 
 7129   ins_encode(aarch64_enc_ldrw(dst, mem));
 7130 
 7131   ins_pipe(iload_reg_mem);
 7132 %}
 7133 
 7134 // Load Klass Pointer
 7135 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7136 %{
 7137   match(Set dst (LoadKlass mem));
 7138   predicate(!needs_acquiring_load(n));
 7139 
 7140   ins_cost(4 * INSN_COST);
 7141   format %{ "ldr  $dst, $mem\t# class" %}
 7142 
 7143   ins_encode(aarch64_enc_ldr(dst, mem));
 7144 
 7145   ins_pipe(iload_reg_mem);
 7146 %}
 7147 
 7148 // Load Narrow Klass Pointer
 7149 instruct loadNKlass(iRegNNoSp dst, memory4 mem, rFlagsReg cr)
 7150 %{
 7151   match(Set dst (LoadNKlass mem));
 7152   effect(TEMP_DEF dst, KILL cr);
 7153   predicate(!needs_acquiring_load(n));
 7154 
 7155   ins_cost(4 * INSN_COST);
 7156   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7157   ins_encode %{
 7158     assert($mem$$disp == oopDesc::klass_offset_in_bytes(), "expect correct offset");
 7159     assert($mem$$index$$Register == noreg, "expect no index");
 7160     __ load_nklass($dst$$Register, $mem$$base$$Register);
 7161   %}
 7162   ins_pipe(pipe_slow);
 7163 %}
 7164 
 7165 // Load Float
 7166 instruct loadF(vRegF dst, memory4 mem)
 7167 %{
 7168   match(Set dst (LoadF mem));
 7169   predicate(!needs_acquiring_load(n));
 7170 
 7171   ins_cost(4 * INSN_COST);
 7172   format %{ "ldrs  $dst, $mem\t# float" %}
 7173 
 7174   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7175 
 7176   ins_pipe(pipe_class_memory);
 7177 %}
 7178 
 7179 // Load Double
 7180 instruct loadD(vRegD dst, memory8 mem)
 7181 %{
 7182   match(Set dst (LoadD mem));
 7183   predicate(!needs_acquiring_load(n));
 7184 
 7185   ins_cost(4 * INSN_COST);
 7186   format %{ "ldrd  $dst, $mem\t# double" %}
 7187 
 7188   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7189 
 7190   ins_pipe(pipe_class_memory);
 7191 %}
 7192 
 7193 
 7194 // Load Int Constant
 7195 instruct loadConI(iRegINoSp dst, immI src)
 7196 %{
 7197   match(Set dst src);
 7198 
 7199   ins_cost(INSN_COST);
 7200   format %{ "mov $dst, $src\t# int" %}
 7201 
 7202   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7203 
 7204   ins_pipe(ialu_imm);
 7205 %}
 7206 
 7207 // Load Long Constant
 7208 instruct loadConL(iRegLNoSp dst, immL src)
 7209 %{
 7210   match(Set dst src);
 7211 
 7212   ins_cost(INSN_COST);
 7213   format %{ "mov $dst, $src\t# long" %}
 7214 
 7215   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7216 
 7217   ins_pipe(ialu_imm);
 7218 %}
 7219 
 7220 // Load Pointer Constant
 7221 
 7222 instruct loadConP(iRegPNoSp dst, immP con)
 7223 %{
 7224   match(Set dst con);
 7225 
 7226   ins_cost(INSN_COST * 4);
 7227   format %{
 7228     "mov  $dst, $con\t# ptr\n\t"
 7229   %}
 7230 
 7231   ins_encode(aarch64_enc_mov_p(dst, con));
 7232 
 7233   ins_pipe(ialu_imm);
 7234 %}
 7235 
 7236 // Load Null Pointer Constant
 7237 
 7238 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7239 %{
 7240   match(Set dst con);
 7241 
 7242   ins_cost(INSN_COST);
 7243   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7244 
 7245   ins_encode(aarch64_enc_mov_p0(dst, con));
 7246 
 7247   ins_pipe(ialu_imm);
 7248 %}
 7249 
 7250 // Load Pointer Constant One
 7251 
 7252 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7253 %{
 7254   match(Set dst con);
 7255 
 7256   ins_cost(INSN_COST);
 7257   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7258 
 7259   ins_encode(aarch64_enc_mov_p1(dst, con));
 7260 
 7261   ins_pipe(ialu_imm);
 7262 %}
 7263 
 7264 // Load Byte Map Base Constant
 7265 
 7266 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7267 %{
 7268   match(Set dst con);
 7269 
 7270   ins_cost(INSN_COST);
 7271   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7272 
 7273   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7274 
 7275   ins_pipe(ialu_imm);
 7276 %}
 7277 
 7278 // Load Narrow Pointer Constant
 7279 
 7280 instruct loadConN(iRegNNoSp dst, immN con)
 7281 %{
 7282   match(Set dst con);
 7283 
 7284   ins_cost(INSN_COST * 4);
 7285   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7286 
 7287   ins_encode(aarch64_enc_mov_n(dst, con));
 7288 
 7289   ins_pipe(ialu_imm);
 7290 %}
 7291 
 7292 // Load Narrow Null Pointer Constant
 7293 
 7294 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7295 %{
 7296   match(Set dst con);
 7297 
 7298   ins_cost(INSN_COST);
 7299   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7300 
 7301   ins_encode(aarch64_enc_mov_n0(dst, con));
 7302 
 7303   ins_pipe(ialu_imm);
 7304 %}
 7305 
 7306 // Load Narrow Klass Constant
 7307 
 7308 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7309 %{
 7310   match(Set dst con);
 7311 
 7312   ins_cost(INSN_COST);
 7313   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7314 
 7315   ins_encode(aarch64_enc_mov_nk(dst, con));
 7316 
 7317   ins_pipe(ialu_imm);
 7318 %}
 7319 
 7320 // Load Packed Float Constant
 7321 
 7322 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7323   match(Set dst con);
 7324   ins_cost(INSN_COST * 4);
 7325   format %{ "fmovs  $dst, $con"%}
 7326   ins_encode %{
 7327     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7328   %}
 7329 
 7330   ins_pipe(fp_imm_s);
 7331 %}
 7332 
 7333 // Load Float Constant
 7334 
 7335 instruct loadConF(vRegF dst, immF con) %{
 7336   match(Set dst con);
 7337 
 7338   ins_cost(INSN_COST * 4);
 7339 
 7340   format %{
 7341     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7342   %}
 7343 
 7344   ins_encode %{
 7345     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7346   %}
 7347 
 7348   ins_pipe(fp_load_constant_s);
 7349 %}
 7350 
 7351 // Load Packed Double Constant
 7352 
 7353 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7354   match(Set dst con);
 7355   ins_cost(INSN_COST);
 7356   format %{ "fmovd  $dst, $con"%}
 7357   ins_encode %{
 7358     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7359   %}
 7360 
 7361   ins_pipe(fp_imm_d);
 7362 %}
 7363 
 7364 // Load Double Constant
 7365 
 7366 instruct loadConD(vRegD dst, immD con) %{
 7367   match(Set dst con);
 7368 
 7369   ins_cost(INSN_COST * 5);
 7370   format %{
 7371     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7372   %}
 7373 
 7374   ins_encode %{
 7375     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7376   %}
 7377 
 7378   ins_pipe(fp_load_constant_d);
 7379 %}
 7380 
 7381 // Store Instructions
 7382 
 7383 // Store CMS card-mark Immediate
 7384 instruct storeimmCM0(immI0 zero, memory1 mem)
 7385 %{
 7386   match(Set mem (StoreCM mem zero));
 7387 
 7388   ins_cost(INSN_COST);
 7389   format %{ "storestore (elided)\n\t"
 7390             "strb zr, $mem\t# byte" %}
 7391 
 7392   ins_encode(aarch64_enc_strb0(mem));
 7393 
 7394   ins_pipe(istore_mem);
 7395 %}
 7396 
 7397 // Store CMS card-mark Immediate with intervening StoreStore
 7398 // needed when using CMS with no conditional card marking
 7399 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7400 %{
 7401   match(Set mem (StoreCM mem zero));
 7402 
 7403   ins_cost(INSN_COST * 2);
 7404   format %{ "storestore\n\t"
 7405             "dmb ishst"
 7406             "\n\tstrb zr, $mem\t# byte" %}
 7407 
 7408   ins_encode(aarch64_enc_strb0_ordered(mem));
 7409 
 7410   ins_pipe(istore_mem);
 7411 %}
 7412 
 7413 // Store Byte
 7414 instruct storeB(iRegIorL2I src, memory1 mem)
 7415 %{
 7416   match(Set mem (StoreB mem src));
 7417   predicate(!needs_releasing_store(n));
 7418 
 7419   ins_cost(INSN_COST);
 7420   format %{ "strb  $src, $mem\t# byte" %}
 7421 
 7422   ins_encode(aarch64_enc_strb(src, mem));
 7423 
 7424   ins_pipe(istore_reg_mem);
 7425 %}
 7426 
 7427 
 7428 instruct storeimmB0(immI0 zero, memory1 mem)
 7429 %{
 7430   match(Set mem (StoreB mem zero));
 7431   predicate(!needs_releasing_store(n));
 7432 
 7433   ins_cost(INSN_COST);
 7434   format %{ "strb rscractch2, $mem\t# byte" %}
 7435 
 7436   ins_encode(aarch64_enc_strb0(mem));
 7437 
 7438   ins_pipe(istore_mem);
 7439 %}
 7440 
 7441 // Store Char/Short
 7442 instruct storeC(iRegIorL2I src, memory2 mem)
 7443 %{
 7444   match(Set mem (StoreC mem src));
 7445   predicate(!needs_releasing_store(n));
 7446 
 7447   ins_cost(INSN_COST);
 7448   format %{ "strh  $src, $mem\t# short" %}
 7449 
 7450   ins_encode(aarch64_enc_strh(src, mem));
 7451 
 7452   ins_pipe(istore_reg_mem);
 7453 %}
 7454 
 7455 instruct storeimmC0(immI0 zero, memory2 mem)
 7456 %{
 7457   match(Set mem (StoreC mem zero));
 7458   predicate(!needs_releasing_store(n));
 7459 
 7460   ins_cost(INSN_COST);
 7461   format %{ "strh  zr, $mem\t# short" %}
 7462 
 7463   ins_encode(aarch64_enc_strh0(mem));
 7464 
 7465   ins_pipe(istore_mem);
 7466 %}
 7467 
 7468 // Store Integer
 7469 
 7470 instruct storeI(iRegIorL2I src, memory4 mem)
 7471 %{
 7472   match(Set mem(StoreI mem src));
 7473   predicate(!needs_releasing_store(n));
 7474 
 7475   ins_cost(INSN_COST);
 7476   format %{ "strw  $src, $mem\t# int" %}
 7477 
 7478   ins_encode(aarch64_enc_strw(src, mem));
 7479 
 7480   ins_pipe(istore_reg_mem);
 7481 %}
 7482 
 7483 instruct storeimmI0(immI0 zero, memory4 mem)
 7484 %{
 7485   match(Set mem(StoreI mem zero));
 7486   predicate(!needs_releasing_store(n));
 7487 
 7488   ins_cost(INSN_COST);
 7489   format %{ "strw  zr, $mem\t# int" %}
 7490 
 7491   ins_encode(aarch64_enc_strw0(mem));
 7492 
 7493   ins_pipe(istore_mem);
 7494 %}
 7495 
 7496 // Store Long (64 bit signed)
 7497 instruct storeL(iRegL src, memory8 mem)
 7498 %{
 7499   match(Set mem (StoreL mem src));
 7500   predicate(!needs_releasing_store(n));
 7501 
 7502   ins_cost(INSN_COST);
 7503   format %{ "str  $src, $mem\t# int" %}
 7504 
 7505   ins_encode(aarch64_enc_str(src, mem));
 7506 
 7507   ins_pipe(istore_reg_mem);
 7508 %}
 7509 
 7510 // Store Long (64 bit signed)
 7511 instruct storeimmL0(immL0 zero, memory8 mem)
 7512 %{
 7513   match(Set mem (StoreL mem zero));
 7514   predicate(!needs_releasing_store(n));
 7515 
 7516   ins_cost(INSN_COST);
 7517   format %{ "str  zr, $mem\t# int" %}
 7518 
 7519   ins_encode(aarch64_enc_str0(mem));
 7520 
 7521   ins_pipe(istore_mem);
 7522 %}
 7523 
 7524 // Store Pointer
 7525 instruct storeP(iRegP src, memory8 mem)
 7526 %{
 7527   match(Set mem (StoreP mem src));
 7528   predicate(!needs_releasing_store(n));
 7529 
 7530   ins_cost(INSN_COST);
 7531   format %{ "str  $src, $mem\t# ptr" %}
 7532 
 7533   ins_encode(aarch64_enc_str(src, mem));
 7534 
 7535   ins_pipe(istore_reg_mem);
 7536 %}
 7537 
 7538 // Store Pointer
 7539 instruct storeimmP0(immP0 zero, memory8 mem)
 7540 %{
 7541   match(Set mem (StoreP mem zero));
 7542   predicate(!needs_releasing_store(n));
 7543 
 7544   ins_cost(INSN_COST);
 7545   format %{ "str zr, $mem\t# ptr" %}
 7546 
 7547   ins_encode(aarch64_enc_str0(mem));
 7548 
 7549   ins_pipe(istore_mem);
 7550 %}
 7551 
 7552 // Store Compressed Pointer
 7553 instruct storeN(iRegN src, memory4 mem)
 7554 %{
 7555   match(Set mem (StoreN mem src));
 7556   predicate(!needs_releasing_store(n));
 7557 
 7558   ins_cost(INSN_COST);
 7559   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7560 
 7561   ins_encode(aarch64_enc_strw(src, mem));
 7562 
 7563   ins_pipe(istore_reg_mem);
 7564 %}
 7565 
 7566 instruct storeImmN0(immN0 zero, memory4 mem)
 7567 %{
 7568   match(Set mem (StoreN mem zero));
 7569   predicate(!needs_releasing_store(n));
 7570 
 7571   ins_cost(INSN_COST);
 7572   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7573 
 7574   ins_encode(aarch64_enc_strw0(mem));
 7575 
 7576   ins_pipe(istore_mem);
 7577 %}
 7578 
 7579 // Store Float
 7580 instruct storeF(vRegF src, memory4 mem)
 7581 %{
 7582   match(Set mem (StoreF mem src));
 7583   predicate(!needs_releasing_store(n));
 7584 
 7585   ins_cost(INSN_COST);
 7586   format %{ "strs  $src, $mem\t# float" %}
 7587 
 7588   ins_encode( aarch64_enc_strs(src, mem) );
 7589 
 7590   ins_pipe(pipe_class_memory);
 7591 %}
 7592 
 7593 // TODO
 7594 // implement storeImmF0 and storeFImmPacked
 7595 
 7596 // Store Double
 7597 instruct storeD(vRegD src, memory8 mem)
 7598 %{
 7599   match(Set mem (StoreD mem src));
 7600   predicate(!needs_releasing_store(n));
 7601 
 7602   ins_cost(INSN_COST);
 7603   format %{ "strd  $src, $mem\t# double" %}
 7604 
 7605   ins_encode( aarch64_enc_strd(src, mem) );
 7606 
 7607   ins_pipe(pipe_class_memory);
 7608 %}
 7609 
 7610 // Store Compressed Klass Pointer
 7611 instruct storeNKlass(iRegN src, memory4 mem)
 7612 %{
 7613   predicate(!needs_releasing_store(n));
 7614   match(Set mem (StoreNKlass mem src));
 7615 
 7616   ins_cost(INSN_COST);
 7617   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7618 
 7619   ins_encode(aarch64_enc_strw(src, mem));
 7620 
 7621   ins_pipe(istore_reg_mem);
 7622 %}
 7623 
 7624 // TODO
 7625 // implement storeImmD0 and storeDImmPacked
 7626 
 7627 // prefetch instructions
 7628 // Must be safe to execute with invalid address (cannot fault).
 7629 
 7630 instruct prefetchalloc( memory8 mem ) %{
 7631   match(PrefetchAllocation mem);
 7632 
 7633   ins_cost(INSN_COST);
 7634   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7635 
 7636   ins_encode( aarch64_enc_prefetchw(mem) );
 7637 
 7638   ins_pipe(iload_prefetch);
 7639 %}
 7640 
 7641 //  ---------------- volatile loads and stores ----------------
 7642 
 7643 // Load Byte (8 bit signed)
 7644 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7645 %{
 7646   match(Set dst (LoadB mem));
 7647 
 7648   ins_cost(VOLATILE_REF_COST);
 7649   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7650 
 7651   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7652 
 7653   ins_pipe(pipe_serial);
 7654 %}
 7655 
 7656 // Load Byte (8 bit signed) into long
 7657 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7658 %{
 7659   match(Set dst (ConvI2L (LoadB mem)));
 7660 
 7661   ins_cost(VOLATILE_REF_COST);
 7662   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7663 
 7664   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7665 
 7666   ins_pipe(pipe_serial);
 7667 %}
 7668 
 7669 // Load Byte (8 bit unsigned)
 7670 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7671 %{
 7672   match(Set dst (LoadUB mem));
 7673 
 7674   ins_cost(VOLATILE_REF_COST);
 7675   format %{ "ldarb  $dst, $mem\t# byte" %}
 7676 
 7677   ins_encode(aarch64_enc_ldarb(dst, mem));
 7678 
 7679   ins_pipe(pipe_serial);
 7680 %}
 7681 
 7682 // Load Byte (8 bit unsigned) into long
 7683 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7684 %{
 7685   match(Set dst (ConvI2L (LoadUB mem)));
 7686 
 7687   ins_cost(VOLATILE_REF_COST);
 7688   format %{ "ldarb  $dst, $mem\t# byte" %}
 7689 
 7690   ins_encode(aarch64_enc_ldarb(dst, mem));
 7691 
 7692   ins_pipe(pipe_serial);
 7693 %}
 7694 
 7695 // Load Short (16 bit signed)
 7696 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7697 %{
 7698   match(Set dst (LoadS mem));
 7699 
 7700   ins_cost(VOLATILE_REF_COST);
 7701   format %{ "ldarshw  $dst, $mem\t# short" %}
 7702 
 7703   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7704 
 7705   ins_pipe(pipe_serial);
 7706 %}
 7707 
 7708 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7709 %{
 7710   match(Set dst (LoadUS mem));
 7711 
 7712   ins_cost(VOLATILE_REF_COST);
 7713   format %{ "ldarhw  $dst, $mem\t# short" %}
 7714 
 7715   ins_encode(aarch64_enc_ldarhw(dst, mem));
 7716 
 7717   ins_pipe(pipe_serial);
 7718 %}
 7719 
 7720 // Load Short/Char (16 bit unsigned) into long
 7721 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7722 %{
 7723   match(Set dst (ConvI2L (LoadUS mem)));
 7724 
 7725   ins_cost(VOLATILE_REF_COST);
 7726   format %{ "ldarh  $dst, $mem\t# short" %}
 7727 
 7728   ins_encode(aarch64_enc_ldarh(dst, mem));
 7729 
 7730   ins_pipe(pipe_serial);
 7731 %}
 7732 
 7733 // Load Short/Char (16 bit signed) into long
 7734 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7735 %{
 7736   match(Set dst (ConvI2L (LoadS mem)));
 7737 
 7738   ins_cost(VOLATILE_REF_COST);
 7739   format %{ "ldarh  $dst, $mem\t# short" %}
 7740 
 7741   ins_encode(aarch64_enc_ldarsh(dst, mem));
 7742 
 7743   ins_pipe(pipe_serial);
 7744 %}
 7745 
 7746 // Load Integer (32 bit signed)
 7747 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7748 %{
 7749   match(Set dst (LoadI mem));
 7750 
 7751   ins_cost(VOLATILE_REF_COST);
 7752   format %{ "ldarw  $dst, $mem\t# int" %}
 7753 
 7754   ins_encode(aarch64_enc_ldarw(dst, mem));
 7755 
 7756   ins_pipe(pipe_serial);
 7757 %}
 7758 
 7759 // Load Integer (32 bit unsigned) into long
 7760 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 7761 %{
 7762   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7763 
 7764   ins_cost(VOLATILE_REF_COST);
 7765   format %{ "ldarw  $dst, $mem\t# int" %}
 7766 
 7767   ins_encode(aarch64_enc_ldarw(dst, mem));
 7768 
 7769   ins_pipe(pipe_serial);
 7770 %}
 7771 
 7772 // Load Long (64 bit signed)
 7773 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7774 %{
 7775   match(Set dst (LoadL mem));
 7776 
 7777   ins_cost(VOLATILE_REF_COST);
 7778   format %{ "ldar  $dst, $mem\t# int" %}
 7779 
 7780   ins_encode(aarch64_enc_ldar(dst, mem));
 7781 
 7782   ins_pipe(pipe_serial);
 7783 %}
 7784 
 7785 // Load Pointer
 7786 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 7787 %{
 7788   match(Set dst (LoadP mem));
 7789   predicate(n->as_Load()->barrier_data() == 0);
 7790 
 7791   ins_cost(VOLATILE_REF_COST);
 7792   format %{ "ldar  $dst, $mem\t# ptr" %}
 7793 
 7794   ins_encode(aarch64_enc_ldar(dst, mem));
 7795 
 7796   ins_pipe(pipe_serial);
 7797 %}
 7798 
 7799 // Load Compressed Pointer
 7800 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 7801 %{
 7802   match(Set dst (LoadN mem));
 7803 
 7804   ins_cost(VOLATILE_REF_COST);
 7805   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 7806 
 7807   ins_encode(aarch64_enc_ldarw(dst, mem));
 7808 
 7809   ins_pipe(pipe_serial);
 7810 %}
 7811 
 7812 // Load Float
 7813 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 7814 %{
 7815   match(Set dst (LoadF mem));
 7816 
 7817   ins_cost(VOLATILE_REF_COST);
 7818   format %{ "ldars  $dst, $mem\t# float" %}
 7819 
 7820   ins_encode( aarch64_enc_fldars(dst, mem) );
 7821 
 7822   ins_pipe(pipe_serial);
 7823 %}
 7824 
 7825 // Load Double
 7826 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 7827 %{
 7828   match(Set dst (LoadD mem));
 7829 
 7830   ins_cost(VOLATILE_REF_COST);
 7831   format %{ "ldard  $dst, $mem\t# double" %}
 7832 
 7833   ins_encode( aarch64_enc_fldard(dst, mem) );
 7834 
 7835   ins_pipe(pipe_serial);
 7836 %}
 7837 
 7838 // Store Byte
 7839 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7840 %{
 7841   match(Set mem (StoreB mem src));
 7842 
 7843   ins_cost(VOLATILE_REF_COST);
 7844   format %{ "stlrb  $src, $mem\t# byte" %}
 7845 
 7846   ins_encode(aarch64_enc_stlrb(src, mem));
 7847 
 7848   ins_pipe(pipe_class_memory);
 7849 %}
 7850 
 7851 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7852 %{
 7853   match(Set mem (StoreB mem zero));
 7854 
 7855   ins_cost(VOLATILE_REF_COST);
 7856   format %{ "stlrb  zr, $mem\t# byte" %}
 7857 
 7858   ins_encode(aarch64_enc_stlrb0(mem));
 7859 
 7860   ins_pipe(pipe_class_memory);
 7861 %}
 7862 
 7863 // Store Char/Short
 7864 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7865 %{
 7866   match(Set mem (StoreC mem src));
 7867 
 7868   ins_cost(VOLATILE_REF_COST);
 7869   format %{ "stlrh  $src, $mem\t# short" %}
 7870 
 7871   ins_encode(aarch64_enc_stlrh(src, mem));
 7872 
 7873   ins_pipe(pipe_class_memory);
 7874 %}
 7875 
 7876 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7877 %{
 7878   match(Set mem (StoreC mem zero));
 7879 
 7880   ins_cost(VOLATILE_REF_COST);
 7881   format %{ "stlrh  zr, $mem\t# short" %}
 7882 
 7883   ins_encode(aarch64_enc_stlrh0(mem));
 7884 
 7885   ins_pipe(pipe_class_memory);
 7886 %}
 7887 
 7888 // Store Integer
 7889 
 7890 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7891 %{
 7892   match(Set mem(StoreI mem src));
 7893 
 7894   ins_cost(VOLATILE_REF_COST);
 7895   format %{ "stlrw  $src, $mem\t# int" %}
 7896 
 7897   ins_encode(aarch64_enc_stlrw(src, mem));
 7898 
 7899   ins_pipe(pipe_class_memory);
 7900 %}
 7901 
 7902 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7903 %{
 7904   match(Set mem(StoreI mem zero));
 7905 
 7906   ins_cost(VOLATILE_REF_COST);
 7907   format %{ "stlrw  zr, $mem\t# int" %}
 7908 
 7909   ins_encode(aarch64_enc_stlrw0(mem));
 7910 
 7911   ins_pipe(pipe_class_memory);
 7912 %}
 7913 
 7914 // Store Long (64 bit signed)
 7915 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 7916 %{
 7917   match(Set mem (StoreL mem src));
 7918 
 7919   ins_cost(VOLATILE_REF_COST);
 7920   format %{ "stlr  $src, $mem\t# int" %}
 7921 
 7922   ins_encode(aarch64_enc_stlr(src, mem));
 7923 
 7924   ins_pipe(pipe_class_memory);
 7925 %}
 7926 
 7927 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 7928 %{
 7929   match(Set mem (StoreL mem zero));
 7930 
 7931   ins_cost(VOLATILE_REF_COST);
 7932   format %{ "stlr  zr, $mem\t# int" %}
 7933 
 7934   ins_encode(aarch64_enc_stlr0(mem));
 7935 
 7936   ins_pipe(pipe_class_memory);
 7937 %}
 7938 
 7939 // Store Pointer
 7940 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 7941 %{
 7942   match(Set mem (StoreP mem src));
 7943 
 7944   ins_cost(VOLATILE_REF_COST);
 7945   format %{ "stlr  $src, $mem\t# ptr" %}
 7946 
 7947   ins_encode(aarch64_enc_stlr(src, mem));
 7948 
 7949   ins_pipe(pipe_class_memory);
 7950 %}
 7951 
 7952 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 7953 %{
 7954   match(Set mem (StoreP mem zero));
 7955 
 7956   ins_cost(VOLATILE_REF_COST);
 7957   format %{ "stlr  zr, $mem\t# ptr" %}
 7958 
 7959   ins_encode(aarch64_enc_stlr0(mem));
 7960 
 7961   ins_pipe(pipe_class_memory);
 7962 %}
 7963 
 7964 // Store Compressed Pointer
 7965 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 7966 %{
 7967   match(Set mem (StoreN mem src));
 7968 
 7969   ins_cost(VOLATILE_REF_COST);
 7970   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 7971 
 7972   ins_encode(aarch64_enc_stlrw(src, mem));
 7973 
 7974   ins_pipe(pipe_class_memory);
 7975 %}
 7976 
 7977 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 7978 %{
 7979   match(Set mem (StoreN mem zero));
 7980 
 7981   ins_cost(VOLATILE_REF_COST);
 7982   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 7983 
 7984   ins_encode(aarch64_enc_stlrw0(mem));
 7985 
 7986   ins_pipe(pipe_class_memory);
 7987 %}
 7988 
 7989 // Store Float
 7990 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 7991 %{
 7992   match(Set mem (StoreF mem src));
 7993 
 7994   ins_cost(VOLATILE_REF_COST);
 7995   format %{ "stlrs  $src, $mem\t# float" %}
 7996 
 7997   ins_encode( aarch64_enc_fstlrs(src, mem) );
 7998 
 7999   ins_pipe(pipe_class_memory);
 8000 %}
 8001 
 8002 // TODO
 8003 // implement storeImmF0 and storeFImmPacked
 8004 
 8005 // Store Double
 8006 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8007 %{
 8008   match(Set mem (StoreD mem src));
 8009 
 8010   ins_cost(VOLATILE_REF_COST);
 8011   format %{ "stlrd  $src, $mem\t# double" %}
 8012 
 8013   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8014 
 8015   ins_pipe(pipe_class_memory);
 8016 %}
 8017 
 8018 //  ---------------- end of volatile loads and stores ----------------
 8019 
 8020 instruct cacheWB(indirect addr)
 8021 %{
 8022   predicate(VM_Version::supports_data_cache_line_flush());
 8023   match(CacheWB addr);
 8024 
 8025   ins_cost(100);
 8026   format %{"cache wb $addr" %}
 8027   ins_encode %{
 8028     assert($addr->index_position() < 0, "should be");
 8029     assert($addr$$disp == 0, "should be");
 8030     __ cache_wb(Address($addr$$base$$Register, 0));
 8031   %}
 8032   ins_pipe(pipe_slow); // XXX
 8033 %}
 8034 
 8035 instruct cacheWBPreSync()
 8036 %{
 8037   predicate(VM_Version::supports_data_cache_line_flush());
 8038   match(CacheWBPreSync);
 8039 
 8040   ins_cost(100);
 8041   format %{"cache wb presync" %}
 8042   ins_encode %{
 8043     __ cache_wbsync(true);
 8044   %}
 8045   ins_pipe(pipe_slow); // XXX
 8046 %}
 8047 
 8048 instruct cacheWBPostSync()
 8049 %{
 8050   predicate(VM_Version::supports_data_cache_line_flush());
 8051   match(CacheWBPostSync);
 8052 
 8053   ins_cost(100);
 8054   format %{"cache wb postsync" %}
 8055   ins_encode %{
 8056     __ cache_wbsync(false);
 8057   %}
 8058   ins_pipe(pipe_slow); // XXX
 8059 %}
 8060 
 8061 // ============================================================================
 8062 // BSWAP Instructions
 8063 
 8064 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8065   match(Set dst (ReverseBytesI src));
 8066 
 8067   ins_cost(INSN_COST);
 8068   format %{ "revw  $dst, $src" %}
 8069 
 8070   ins_encode %{
 8071     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8072   %}
 8073 
 8074   ins_pipe(ialu_reg);
 8075 %}
 8076 
 8077 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8078   match(Set dst (ReverseBytesL src));
 8079 
 8080   ins_cost(INSN_COST);
 8081   format %{ "rev  $dst, $src" %}
 8082 
 8083   ins_encode %{
 8084     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8085   %}
 8086 
 8087   ins_pipe(ialu_reg);
 8088 %}
 8089 
 8090 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8091   match(Set dst (ReverseBytesUS src));
 8092 
 8093   ins_cost(INSN_COST);
 8094   format %{ "rev16w  $dst, $src" %}
 8095 
 8096   ins_encode %{
 8097     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8098   %}
 8099 
 8100   ins_pipe(ialu_reg);
 8101 %}
 8102 
 8103 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8104   match(Set dst (ReverseBytesS src));
 8105 
 8106   ins_cost(INSN_COST);
 8107   format %{ "rev16w  $dst, $src\n\t"
 8108             "sbfmw $dst, $dst, #0, #15" %}
 8109 
 8110   ins_encode %{
 8111     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8112     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8113   %}
 8114 
 8115   ins_pipe(ialu_reg);
 8116 %}
 8117 
 8118 // ============================================================================
 8119 // Zero Count Instructions
 8120 
 8121 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8122   match(Set dst (CountLeadingZerosI src));
 8123 
 8124   ins_cost(INSN_COST);
 8125   format %{ "clzw  $dst, $src" %}
 8126   ins_encode %{
 8127     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8128   %}
 8129 
 8130   ins_pipe(ialu_reg);
 8131 %}
 8132 
 8133 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8134   match(Set dst (CountLeadingZerosL src));
 8135 
 8136   ins_cost(INSN_COST);
 8137   format %{ "clz   $dst, $src" %}
 8138   ins_encode %{
 8139     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8140   %}
 8141 
 8142   ins_pipe(ialu_reg);
 8143 %}
 8144 
 8145 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8146   match(Set dst (CountTrailingZerosI src));
 8147 
 8148   ins_cost(INSN_COST * 2);
 8149   format %{ "rbitw  $dst, $src\n\t"
 8150             "clzw   $dst, $dst" %}
 8151   ins_encode %{
 8152     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8153     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8154   %}
 8155 
 8156   ins_pipe(ialu_reg);
 8157 %}
 8158 
 8159 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8160   match(Set dst (CountTrailingZerosL src));
 8161 
 8162   ins_cost(INSN_COST * 2);
 8163   format %{ "rbit   $dst, $src\n\t"
 8164             "clz    $dst, $dst" %}
 8165   ins_encode %{
 8166     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8167     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8168   %}
 8169 
 8170   ins_pipe(ialu_reg);
 8171 %}
 8172 
 8173 //---------- Population Count Instructions -------------------------------------
 8174 //
 8175 
 8176 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8177   match(Set dst (PopCountI src));
 8178   effect(TEMP tmp);
 8179   ins_cost(INSN_COST * 13);
 8180 
 8181   format %{ "movw   $src, $src\n\t"
 8182             "mov    $tmp, $src\t# vector (1D)\n\t"
 8183             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8184             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8185             "mov    $dst, $tmp\t# vector (1D)" %}
 8186   ins_encode %{
 8187     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8188     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8189     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8190     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8191     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8192   %}
 8193 
 8194   ins_pipe(pipe_class_default);
 8195 %}
 8196 
 8197 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8198   match(Set dst (PopCountI (LoadI mem)));
 8199   effect(TEMP tmp);
 8200   ins_cost(INSN_COST * 13);
 8201 
 8202   format %{ "ldrs   $tmp, $mem\n\t"
 8203             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8204             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8205             "mov    $dst, $tmp\t# vector (1D)" %}
 8206   ins_encode %{
 8207     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8208     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8209               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8210     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8211     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8212     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8213   %}
 8214 
 8215   ins_pipe(pipe_class_default);
 8216 %}
 8217 
 8218 // Note: Long.bitCount(long) returns an int.
 8219 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8220   match(Set dst (PopCountL src));
 8221   effect(TEMP tmp);
 8222   ins_cost(INSN_COST * 13);
 8223 
 8224   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8225             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8226             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8227             "mov    $dst, $tmp\t# vector (1D)" %}
 8228   ins_encode %{
 8229     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8230     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8231     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8232     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8233   %}
 8234 
 8235   ins_pipe(pipe_class_default);
 8236 %}
 8237 
 8238 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8239   match(Set dst (PopCountL (LoadL mem)));
 8240   effect(TEMP tmp);
 8241   ins_cost(INSN_COST * 13);
 8242 
 8243   format %{ "ldrd   $tmp, $mem\n\t"
 8244             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8245             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8246             "mov    $dst, $tmp\t# vector (1D)" %}
 8247   ins_encode %{
 8248     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8249     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8250               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8251     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8252     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8253     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8254   %}
 8255 
 8256   ins_pipe(pipe_class_default);
 8257 %}
 8258 
 8259 // ============================================================================
 8260 // MemBar Instruction
 8261 
 8262 instruct load_fence() %{
 8263   match(LoadFence);
 8264   ins_cost(VOLATILE_REF_COST);
 8265 
 8266   format %{ "load_fence" %}
 8267 
 8268   ins_encode %{
 8269     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8270   %}
 8271   ins_pipe(pipe_serial);
 8272 %}
 8273 
 8274 instruct unnecessary_membar_acquire() %{
 8275   predicate(unnecessary_acquire(n));
 8276   match(MemBarAcquire);
 8277   ins_cost(0);
 8278 
 8279   format %{ "membar_acquire (elided)" %}
 8280 
 8281   ins_encode %{
 8282     __ block_comment("membar_acquire (elided)");
 8283   %}
 8284 
 8285   ins_pipe(pipe_class_empty);
 8286 %}
 8287 
 8288 instruct membar_acquire() %{
 8289   match(MemBarAcquire);
 8290   ins_cost(VOLATILE_REF_COST);
 8291 
 8292   format %{ "membar_acquire\n\t"
 8293             "dmb ish" %}
 8294 
 8295   ins_encode %{
 8296     __ block_comment("membar_acquire");
 8297     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8298   %}
 8299 
 8300   ins_pipe(pipe_serial);
 8301 %}
 8302 
 8303 
 8304 instruct membar_acquire_lock() %{
 8305   match(MemBarAcquireLock);
 8306   ins_cost(VOLATILE_REF_COST);
 8307 
 8308   format %{ "membar_acquire_lock (elided)" %}
 8309 
 8310   ins_encode %{
 8311     __ block_comment("membar_acquire_lock (elided)");
 8312   %}
 8313 
 8314   ins_pipe(pipe_serial);
 8315 %}
 8316 
 8317 instruct store_fence() %{
 8318   match(StoreFence);
 8319   ins_cost(VOLATILE_REF_COST);
 8320 
 8321   format %{ "store_fence" %}
 8322 
 8323   ins_encode %{
 8324     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8325   %}
 8326   ins_pipe(pipe_serial);
 8327 %}
 8328 
 8329 instruct unnecessary_membar_release() %{
 8330   predicate(unnecessary_release(n));
 8331   match(MemBarRelease);
 8332   ins_cost(0);
 8333 
 8334   format %{ "membar_release (elided)" %}
 8335 
 8336   ins_encode %{
 8337     __ block_comment("membar_release (elided)");
 8338   %}
 8339   ins_pipe(pipe_serial);
 8340 %}
 8341 
 8342 instruct membar_release() %{
 8343   match(MemBarRelease);
 8344   ins_cost(VOLATILE_REF_COST);
 8345 
 8346   format %{ "membar_release\n\t"
 8347             "dmb ish" %}
 8348 
 8349   ins_encode %{
 8350     __ block_comment("membar_release");
 8351     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8352   %}
 8353   ins_pipe(pipe_serial);
 8354 %}
 8355 
 8356 instruct membar_storestore() %{
 8357   match(MemBarStoreStore);
 8358   match(StoreStoreFence);
 8359   ins_cost(VOLATILE_REF_COST);
 8360 
 8361   format %{ "MEMBAR-store-store" %}
 8362 
 8363   ins_encode %{
 8364     __ membar(Assembler::StoreStore);
 8365   %}
 8366   ins_pipe(pipe_serial);
 8367 %}
 8368 
 8369 instruct membar_release_lock() %{
 8370   match(MemBarReleaseLock);
 8371   ins_cost(VOLATILE_REF_COST);
 8372 
 8373   format %{ "membar_release_lock (elided)" %}
 8374 
 8375   ins_encode %{
 8376     __ block_comment("membar_release_lock (elided)");
 8377   %}
 8378 
 8379   ins_pipe(pipe_serial);
 8380 %}
 8381 
 8382 instruct unnecessary_membar_volatile() %{
 8383   predicate(unnecessary_volatile(n));
 8384   match(MemBarVolatile);
 8385   ins_cost(0);
 8386 
 8387   format %{ "membar_volatile (elided)" %}
 8388 
 8389   ins_encode %{
 8390     __ block_comment("membar_volatile (elided)");
 8391   %}
 8392 
 8393   ins_pipe(pipe_serial);
 8394 %}
 8395 
 8396 instruct membar_volatile() %{
 8397   match(MemBarVolatile);
 8398   ins_cost(VOLATILE_REF_COST*100);
 8399 
 8400   format %{ "membar_volatile\n\t"
 8401              "dmb ish"%}
 8402 
 8403   ins_encode %{
 8404     __ block_comment("membar_volatile");
 8405     __ membar(Assembler::StoreLoad);
 8406   %}
 8407 
 8408   ins_pipe(pipe_serial);
 8409 %}
 8410 
 8411 // ============================================================================
 8412 // Cast/Convert Instructions
 8413 
 8414 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8415   match(Set dst (CastX2P src));
 8416 
 8417   ins_cost(INSN_COST);
 8418   format %{ "mov $dst, $src\t# long -> ptr" %}
 8419 
 8420   ins_encode %{
 8421     if ($dst$$reg != $src$$reg) {
 8422       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8423     }
 8424   %}
 8425 
 8426   ins_pipe(ialu_reg);
 8427 %}
 8428 
 8429 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8430   match(Set dst (CastP2X src));
 8431 
 8432   ins_cost(INSN_COST);
 8433   format %{ "mov $dst, $src\t# ptr -> long" %}
 8434 
 8435   ins_encode %{
 8436     if ($dst$$reg != $src$$reg) {
 8437       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8438     }
 8439   %}
 8440 
 8441   ins_pipe(ialu_reg);
 8442 %}
 8443 
 8444 // Convert oop into int for vectors alignment masking
 8445 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8446   match(Set dst (ConvL2I (CastP2X src)));
 8447 
 8448   ins_cost(INSN_COST);
 8449   format %{ "movw $dst, $src\t# ptr -> int" %}
 8450   ins_encode %{
 8451     __ movw($dst$$Register, $src$$Register);
 8452   %}
 8453 
 8454   ins_pipe(ialu_reg);
 8455 %}
 8456 
 8457 // Convert compressed oop into int for vectors alignment masking
 8458 // in case of 32bit oops (heap < 4Gb).
 8459 instruct convN2I(iRegINoSp dst, iRegN src)
 8460 %{
 8461   predicate(CompressedOops::shift() == 0);
 8462   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8463 
 8464   ins_cost(INSN_COST);
 8465   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8466   ins_encode %{
 8467     __ movw($dst$$Register, $src$$Register);
 8468   %}
 8469 
 8470   ins_pipe(ialu_reg);
 8471 %}
 8472 
 8473 
 8474 // Convert oop pointer into compressed form
 8475 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8476   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8477   match(Set dst (EncodeP src));
 8478   effect(KILL cr);
 8479   ins_cost(INSN_COST * 3);
 8480   format %{ "encode_heap_oop $dst, $src" %}
 8481   ins_encode %{
 8482     Register s = $src$$Register;
 8483     Register d = $dst$$Register;
 8484     __ encode_heap_oop(d, s);
 8485   %}
 8486   ins_pipe(ialu_reg);
 8487 %}
 8488 
 8489 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8490   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8491   match(Set dst (EncodeP src));
 8492   ins_cost(INSN_COST * 3);
 8493   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8494   ins_encode %{
 8495     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8496   %}
 8497   ins_pipe(ialu_reg);
 8498 %}
 8499 
 8500 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8501   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8502             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8503   match(Set dst (DecodeN src));
 8504   ins_cost(INSN_COST * 3);
 8505   format %{ "decode_heap_oop $dst, $src" %}
 8506   ins_encode %{
 8507     Register s = $src$$Register;
 8508     Register d = $dst$$Register;
 8509     __ decode_heap_oop(d, s);
 8510   %}
 8511   ins_pipe(ialu_reg);
 8512 %}
 8513 
 8514 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8515   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8516             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8517   match(Set dst (DecodeN src));
 8518   ins_cost(INSN_COST * 3);
 8519   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8520   ins_encode %{
 8521     Register s = $src$$Register;
 8522     Register d = $dst$$Register;
 8523     __ decode_heap_oop_not_null(d, s);
 8524   %}
 8525   ins_pipe(ialu_reg);
 8526 %}
 8527 
 8528 // n.b. AArch64 implementations of encode_klass_not_null and
 8529 // decode_klass_not_null do not modify the flags register so, unlike
 8530 // Intel, we don't kill CR as a side effect here
 8531 
 8532 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8533   match(Set dst (EncodePKlass src));
 8534 
 8535   ins_cost(INSN_COST * 3);
 8536   format %{ "encode_klass_not_null $dst,$src" %}
 8537 
 8538   ins_encode %{
 8539     Register src_reg = as_Register($src$$reg);
 8540     Register dst_reg = as_Register($dst$$reg);
 8541     __ encode_klass_not_null(dst_reg, src_reg);
 8542   %}
 8543 
 8544    ins_pipe(ialu_reg);
 8545 %}
 8546 
 8547 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8548   match(Set dst (DecodeNKlass src));
 8549 
 8550   ins_cost(INSN_COST * 3);
 8551   format %{ "decode_klass_not_null $dst,$src" %}
 8552 
 8553   ins_encode %{
 8554     Register src_reg = as_Register($src$$reg);
 8555     Register dst_reg = as_Register($dst$$reg);
 8556     if (dst_reg != src_reg) {
 8557       __ decode_klass_not_null(dst_reg, src_reg);
 8558     } else {
 8559       __ decode_klass_not_null(dst_reg);
 8560     }
 8561   %}
 8562 
 8563    ins_pipe(ialu_reg);
 8564 %}
 8565 
 8566 instruct checkCastPP(iRegPNoSp dst)
 8567 %{
 8568   match(Set dst (CheckCastPP dst));
 8569 
 8570   size(0);
 8571   format %{ "# checkcastPP of $dst" %}
 8572   ins_encode(/* empty encoding */);
 8573   ins_pipe(pipe_class_empty);
 8574 %}
 8575 
 8576 instruct castPP(iRegPNoSp dst)
 8577 %{
 8578   match(Set dst (CastPP dst));
 8579 
 8580   size(0);
 8581   format %{ "# castPP of $dst" %}
 8582   ins_encode(/* empty encoding */);
 8583   ins_pipe(pipe_class_empty);
 8584 %}
 8585 
 8586 instruct castII(iRegI dst)
 8587 %{
 8588   match(Set dst (CastII dst));
 8589 
 8590   size(0);
 8591   format %{ "# castII of $dst" %}
 8592   ins_encode(/* empty encoding */);
 8593   ins_cost(0);
 8594   ins_pipe(pipe_class_empty);
 8595 %}
 8596 
 8597 instruct castLL(iRegL dst)
 8598 %{
 8599   match(Set dst (CastLL dst));
 8600 
 8601   size(0);
 8602   format %{ "# castLL of $dst" %}
 8603   ins_encode(/* empty encoding */);
 8604   ins_cost(0);
 8605   ins_pipe(pipe_class_empty);
 8606 %}
 8607 
 8608 instruct castFF(vRegF dst)
 8609 %{
 8610   match(Set dst (CastFF dst));
 8611 
 8612   size(0);
 8613   format %{ "# castFF of $dst" %}
 8614   ins_encode(/* empty encoding */);
 8615   ins_cost(0);
 8616   ins_pipe(pipe_class_empty);
 8617 %}
 8618 
 8619 instruct castDD(vRegD dst)
 8620 %{
 8621   match(Set dst (CastDD dst));
 8622 
 8623   size(0);
 8624   format %{ "# castDD of $dst" %}
 8625   ins_encode(/* empty encoding */);
 8626   ins_cost(0);
 8627   ins_pipe(pipe_class_empty);
 8628 %}
 8629 
 8630 instruct castVV(vReg dst)
 8631 %{
 8632   match(Set dst (CastVV dst));
 8633 
 8634   size(0);
 8635   format %{ "# castVV of $dst" %}
 8636   ins_encode(/* empty encoding */);
 8637   ins_cost(0);
 8638   ins_pipe(pipe_class_empty);
 8639 %}
 8640 
 8641 instruct castVVMask(pRegGov dst)
 8642 %{
 8643   match(Set dst (CastVV dst));
 8644 
 8645   size(0);
 8646   format %{ "# castVV of $dst" %}
 8647   ins_encode(/* empty encoding */);
 8648   ins_cost(0);
 8649   ins_pipe(pipe_class_empty);
 8650 %}
 8651 
 8652 // ============================================================================
 8653 // Atomic operation instructions
 8654 //
 8655 
 8656 // standard CompareAndSwapX when we are using barriers
 8657 // these have higher priority than the rules selected by a predicate
 8658 
 8659 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 8660 // can't match them
 8661 
 8662 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8663 
 8664   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8665   ins_cost(2 * VOLATILE_REF_COST);
 8666 
 8667   effect(KILL cr);
 8668 
 8669   format %{
 8670     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8671     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8672   %}
 8673 
 8674   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 8675             aarch64_enc_cset_eq(res));
 8676 
 8677   ins_pipe(pipe_slow);
 8678 %}
 8679 
 8680 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8681 
 8682   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8683   ins_cost(2 * VOLATILE_REF_COST);
 8684 
 8685   effect(KILL cr);
 8686 
 8687   format %{
 8688     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8689     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8690   %}
 8691 
 8692   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 8693             aarch64_enc_cset_eq(res));
 8694 
 8695   ins_pipe(pipe_slow);
 8696 %}
 8697 
 8698 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8699 
 8700   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8701   ins_cost(2 * VOLATILE_REF_COST);
 8702 
 8703   effect(KILL cr);
 8704 
 8705  format %{
 8706     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8707     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8708  %}
 8709 
 8710  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8711             aarch64_enc_cset_eq(res));
 8712 
 8713   ins_pipe(pipe_slow);
 8714 %}
 8715 
 8716 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8717 
 8718   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8719   ins_cost(2 * VOLATILE_REF_COST);
 8720 
 8721   effect(KILL cr);
 8722 
 8723  format %{
 8724     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8725     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8726  %}
 8727 
 8728  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8729             aarch64_enc_cset_eq(res));
 8730 
 8731   ins_pipe(pipe_slow);
 8732 %}
 8733 
 8734 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8735 
 8736   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8737   predicate(n->as_LoadStore()->barrier_data() == 0);
 8738   ins_cost(2 * VOLATILE_REF_COST);
 8739 
 8740   effect(KILL cr);
 8741 
 8742  format %{
 8743     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8744     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8745  %}
 8746 
 8747  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8748             aarch64_enc_cset_eq(res));
 8749 
 8750   ins_pipe(pipe_slow);
 8751 %}
 8752 
 8753 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8754 
 8755   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8756   ins_cost(2 * VOLATILE_REF_COST);
 8757 
 8758   effect(KILL cr);
 8759 
 8760  format %{
 8761     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8762     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8763  %}
 8764 
 8765  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8766             aarch64_enc_cset_eq(res));
 8767 
 8768   ins_pipe(pipe_slow);
 8769 %}
 8770 
 8771 // alternative CompareAndSwapX when we are eliding barriers
 8772 
 8773 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8774 
 8775   predicate(needs_acquiring_load_exclusive(n));
 8776   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8777   ins_cost(VOLATILE_REF_COST);
 8778 
 8779   effect(KILL cr);
 8780 
 8781   format %{
 8782     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8783     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8784   %}
 8785 
 8786   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 8787             aarch64_enc_cset_eq(res));
 8788 
 8789   ins_pipe(pipe_slow);
 8790 %}
 8791 
 8792 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8793 
 8794   predicate(needs_acquiring_load_exclusive(n));
 8795   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8796   ins_cost(VOLATILE_REF_COST);
 8797 
 8798   effect(KILL cr);
 8799 
 8800   format %{
 8801     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8802     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8803   %}
 8804 
 8805   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 8806             aarch64_enc_cset_eq(res));
 8807 
 8808   ins_pipe(pipe_slow);
 8809 %}
 8810 
 8811 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8812 
 8813   predicate(needs_acquiring_load_exclusive(n));
 8814   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8815   ins_cost(VOLATILE_REF_COST);
 8816 
 8817   effect(KILL cr);
 8818 
 8819  format %{
 8820     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8821     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8822  %}
 8823 
 8824  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8825             aarch64_enc_cset_eq(res));
 8826 
 8827   ins_pipe(pipe_slow);
 8828 %}
 8829 
 8830 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8831 
 8832   predicate(needs_acquiring_load_exclusive(n));
 8833   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8834   ins_cost(VOLATILE_REF_COST);
 8835 
 8836   effect(KILL cr);
 8837 
 8838  format %{
 8839     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8840     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8841  %}
 8842 
 8843  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8844             aarch64_enc_cset_eq(res));
 8845 
 8846   ins_pipe(pipe_slow);
 8847 %}
 8848 
 8849 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8850 
 8851   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8852   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8853   ins_cost(VOLATILE_REF_COST);
 8854 
 8855   effect(KILL cr);
 8856 
 8857  format %{
 8858     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8859     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8860  %}
 8861 
 8862  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8863             aarch64_enc_cset_eq(res));
 8864 
 8865   ins_pipe(pipe_slow);
 8866 %}
 8867 
 8868 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8869 
 8870   predicate(needs_acquiring_load_exclusive(n));
 8871   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8872   ins_cost(VOLATILE_REF_COST);
 8873 
 8874   effect(KILL cr);
 8875 
 8876  format %{
 8877     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8878     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8879  %}
 8880 
 8881  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8882             aarch64_enc_cset_eq(res));
 8883 
 8884   ins_pipe(pipe_slow);
 8885 %}
 8886 
 8887 
 8888 // ---------------------------------------------------------------------
 8889 
 8890 // BEGIN This section of the file is automatically generated. Do not edit --------------
 8891 
 8892 // Sundry CAS operations.  Note that release is always true,
 8893 // regardless of the memory ordering of the CAS.  This is because we
 8894 // need the volatile case to be sequentially consistent but there is
 8895 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 8896 // can't check the type of memory ordering here, so we always emit a
 8897 // STLXR.
 8898 
 8899 // This section is generated from aarch64_ad_cas.m4
 8900 
 8901 
 8902 
 8903 // This pattern is generated automatically from cas.m4.
 8904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8905 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8906 
 8907   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8908   ins_cost(2 * VOLATILE_REF_COST);
 8909   effect(TEMP_DEF res, KILL cr);
 8910   format %{
 8911     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8912   %}
 8913   ins_encode %{
 8914     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8915                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8916                /*weak*/ false, $res$$Register);
 8917     __ sxtbw($res$$Register, $res$$Register);
 8918   %}
 8919   ins_pipe(pipe_slow);
 8920 %}
 8921 
 8922 // This pattern is generated automatically from cas.m4.
 8923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8924 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8925 
 8926   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 8927   ins_cost(2 * VOLATILE_REF_COST);
 8928   effect(TEMP_DEF res, KILL cr);
 8929   format %{
 8930     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8931   %}
 8932   ins_encode %{
 8933     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8934                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 8935                /*weak*/ false, $res$$Register);
 8936     __ sxthw($res$$Register, $res$$Register);
 8937   %}
 8938   ins_pipe(pipe_slow);
 8939 %}
 8940 
 8941 // This pattern is generated automatically from cas.m4.
 8942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8943 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8944 
 8945   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 8946   ins_cost(2 * VOLATILE_REF_COST);
 8947   effect(TEMP_DEF res, KILL cr);
 8948   format %{
 8949     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8950   %}
 8951   ins_encode %{
 8952     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8953                Assembler::word, /*acquire*/ false, /*release*/ true,
 8954                /*weak*/ false, $res$$Register);
 8955   %}
 8956   ins_pipe(pipe_slow);
 8957 %}
 8958 
 8959 // This pattern is generated automatically from cas.m4.
 8960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8961 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8962 
 8963   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 8964   ins_cost(2 * VOLATILE_REF_COST);
 8965   effect(TEMP_DEF res, KILL cr);
 8966   format %{
 8967     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8968   %}
 8969   ins_encode %{
 8970     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8971                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8972                /*weak*/ false, $res$$Register);
 8973   %}
 8974   ins_pipe(pipe_slow);
 8975 %}
 8976 
 8977 // This pattern is generated automatically from cas.m4.
 8978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8979 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8980 
 8981   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 8982   ins_cost(2 * VOLATILE_REF_COST);
 8983   effect(TEMP_DEF res, KILL cr);
 8984   format %{
 8985     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8986   %}
 8987   ins_encode %{
 8988     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8989                Assembler::word, /*acquire*/ false, /*release*/ true,
 8990                /*weak*/ false, $res$$Register);
 8991   %}
 8992   ins_pipe(pipe_slow);
 8993 %}
 8994 
 8995 // This pattern is generated automatically from cas.m4.
 8996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8997 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8998   predicate(n->as_LoadStore()->barrier_data() == 0);
 8999   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9000   ins_cost(2 * VOLATILE_REF_COST);
 9001   effect(TEMP_DEF res, KILL cr);
 9002   format %{
 9003     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9004   %}
 9005   ins_encode %{
 9006     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9007                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9008                /*weak*/ false, $res$$Register);
 9009   %}
 9010   ins_pipe(pipe_slow);
 9011 %}
 9012 
 9013 // This pattern is generated automatically from cas.m4.
 9014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9015 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9016   predicate(needs_acquiring_load_exclusive(n));
 9017   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9018   ins_cost(VOLATILE_REF_COST);
 9019   effect(TEMP_DEF res, KILL cr);
 9020   format %{
 9021     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9022   %}
 9023   ins_encode %{
 9024     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9025                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9026                /*weak*/ false, $res$$Register);
 9027     __ sxtbw($res$$Register, $res$$Register);
 9028   %}
 9029   ins_pipe(pipe_slow);
 9030 %}
 9031 
 9032 // This pattern is generated automatically from cas.m4.
 9033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9034 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9035   predicate(needs_acquiring_load_exclusive(n));
 9036   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9037   ins_cost(VOLATILE_REF_COST);
 9038   effect(TEMP_DEF res, KILL cr);
 9039   format %{
 9040     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9041   %}
 9042   ins_encode %{
 9043     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9044                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9045                /*weak*/ false, $res$$Register);
 9046     __ sxthw($res$$Register, $res$$Register);
 9047   %}
 9048   ins_pipe(pipe_slow);
 9049 %}
 9050 
 9051 // This pattern is generated automatically from cas.m4.
 9052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9053 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9054   predicate(needs_acquiring_load_exclusive(n));
 9055   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9056   ins_cost(VOLATILE_REF_COST);
 9057   effect(TEMP_DEF res, KILL cr);
 9058   format %{
 9059     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9060   %}
 9061   ins_encode %{
 9062     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9063                Assembler::word, /*acquire*/ true, /*release*/ true,
 9064                /*weak*/ false, $res$$Register);
 9065   %}
 9066   ins_pipe(pipe_slow);
 9067 %}
 9068 
 9069 // This pattern is generated automatically from cas.m4.
 9070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9071 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9072   predicate(needs_acquiring_load_exclusive(n));
 9073   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9074   ins_cost(VOLATILE_REF_COST);
 9075   effect(TEMP_DEF res, KILL cr);
 9076   format %{
 9077     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9078   %}
 9079   ins_encode %{
 9080     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9081                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9082                /*weak*/ false, $res$$Register);
 9083   %}
 9084   ins_pipe(pipe_slow);
 9085 %}
 9086 
 9087 // This pattern is generated automatically from cas.m4.
 9088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9089 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9090   predicate(needs_acquiring_load_exclusive(n));
 9091   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9092   ins_cost(VOLATILE_REF_COST);
 9093   effect(TEMP_DEF res, KILL cr);
 9094   format %{
 9095     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9096   %}
 9097   ins_encode %{
 9098     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9099                Assembler::word, /*acquire*/ true, /*release*/ true,
 9100                /*weak*/ false, $res$$Register);
 9101   %}
 9102   ins_pipe(pipe_slow);
 9103 %}
 9104 
 9105 // This pattern is generated automatically from cas.m4.
 9106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9107 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9108   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9109   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9110   ins_cost(VOLATILE_REF_COST);
 9111   effect(TEMP_DEF res, KILL cr);
 9112   format %{
 9113     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9114   %}
 9115   ins_encode %{
 9116     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9117                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9118                /*weak*/ false, $res$$Register);
 9119   %}
 9120   ins_pipe(pipe_slow);
 9121 %}
 9122 
 9123 // This pattern is generated automatically from cas.m4.
 9124 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9125 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9126 
 9127   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9128   ins_cost(2 * VOLATILE_REF_COST);
 9129   effect(KILL cr);
 9130   format %{
 9131     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9132     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9133   %}
 9134   ins_encode %{
 9135     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9136                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9137                /*weak*/ true, noreg);
 9138     __ csetw($res$$Register, Assembler::EQ);
 9139   %}
 9140   ins_pipe(pipe_slow);
 9141 %}
 9142 
 9143 // This pattern is generated automatically from cas.m4.
 9144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9145 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9146 
 9147   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9148   ins_cost(2 * VOLATILE_REF_COST);
 9149   effect(KILL cr);
 9150   format %{
 9151     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9152     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9153   %}
 9154   ins_encode %{
 9155     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9156                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9157                /*weak*/ true, noreg);
 9158     __ csetw($res$$Register, Assembler::EQ);
 9159   %}
 9160   ins_pipe(pipe_slow);
 9161 %}
 9162 
 9163 // This pattern is generated automatically from cas.m4.
 9164 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9165 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9166 
 9167   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9168   ins_cost(2 * VOLATILE_REF_COST);
 9169   effect(KILL cr);
 9170   format %{
 9171     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9172     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9173   %}
 9174   ins_encode %{
 9175     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9176                Assembler::word, /*acquire*/ false, /*release*/ true,
 9177                /*weak*/ true, noreg);
 9178     __ csetw($res$$Register, Assembler::EQ);
 9179   %}
 9180   ins_pipe(pipe_slow);
 9181 %}
 9182 
 9183 // This pattern is generated automatically from cas.m4.
 9184 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9185 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9186 
 9187   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9188   ins_cost(2 * VOLATILE_REF_COST);
 9189   effect(KILL cr);
 9190   format %{
 9191     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9192     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9193   %}
 9194   ins_encode %{
 9195     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9196                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9197                /*weak*/ true, noreg);
 9198     __ csetw($res$$Register, Assembler::EQ);
 9199   %}
 9200   ins_pipe(pipe_slow);
 9201 %}
 9202 
 9203 // This pattern is generated automatically from cas.m4.
 9204 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9205 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9206 
 9207   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9208   ins_cost(2 * VOLATILE_REF_COST);
 9209   effect(KILL cr);
 9210   format %{
 9211     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9212     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9213   %}
 9214   ins_encode %{
 9215     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9216                Assembler::word, /*acquire*/ false, /*release*/ true,
 9217                /*weak*/ true, noreg);
 9218     __ csetw($res$$Register, Assembler::EQ);
 9219   %}
 9220   ins_pipe(pipe_slow);
 9221 %}
 9222 
 9223 // This pattern is generated automatically from cas.m4.
 9224 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9225 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9226   predicate(n->as_LoadStore()->barrier_data() == 0);
 9227   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9228   ins_cost(2 * VOLATILE_REF_COST);
 9229   effect(KILL cr);
 9230   format %{
 9231     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9232     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9233   %}
 9234   ins_encode %{
 9235     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9236                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9237                /*weak*/ true, noreg);
 9238     __ csetw($res$$Register, Assembler::EQ);
 9239   %}
 9240   ins_pipe(pipe_slow);
 9241 %}
 9242 
 9243 // This pattern is generated automatically from cas.m4.
 9244 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9245 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9246   predicate(needs_acquiring_load_exclusive(n));
 9247   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9248   ins_cost(VOLATILE_REF_COST);
 9249   effect(KILL cr);
 9250   format %{
 9251     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9252     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9253   %}
 9254   ins_encode %{
 9255     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9256                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9257                /*weak*/ true, noreg);
 9258     __ csetw($res$$Register, Assembler::EQ);
 9259   %}
 9260   ins_pipe(pipe_slow);
 9261 %}
 9262 
 9263 // This pattern is generated automatically from cas.m4.
 9264 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9265 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9266   predicate(needs_acquiring_load_exclusive(n));
 9267   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9268   ins_cost(VOLATILE_REF_COST);
 9269   effect(KILL cr);
 9270   format %{
 9271     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9272     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9273   %}
 9274   ins_encode %{
 9275     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9276                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9277                /*weak*/ true, noreg);
 9278     __ csetw($res$$Register, Assembler::EQ);
 9279   %}
 9280   ins_pipe(pipe_slow);
 9281 %}
 9282 
 9283 // This pattern is generated automatically from cas.m4.
 9284 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9285 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9286   predicate(needs_acquiring_load_exclusive(n));
 9287   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9288   ins_cost(VOLATILE_REF_COST);
 9289   effect(KILL cr);
 9290   format %{
 9291     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9292     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9293   %}
 9294   ins_encode %{
 9295     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9296                Assembler::word, /*acquire*/ true, /*release*/ true,
 9297                /*weak*/ true, noreg);
 9298     __ csetw($res$$Register, Assembler::EQ);
 9299   %}
 9300   ins_pipe(pipe_slow);
 9301 %}
 9302 
 9303 // This pattern is generated automatically from cas.m4.
 9304 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9305 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9306   predicate(needs_acquiring_load_exclusive(n));
 9307   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9308   ins_cost(VOLATILE_REF_COST);
 9309   effect(KILL cr);
 9310   format %{
 9311     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9312     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9313   %}
 9314   ins_encode %{
 9315     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9316                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9317                /*weak*/ true, noreg);
 9318     __ csetw($res$$Register, Assembler::EQ);
 9319   %}
 9320   ins_pipe(pipe_slow);
 9321 %}
 9322 
 9323 // This pattern is generated automatically from cas.m4.
 9324 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9325 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9326   predicate(needs_acquiring_load_exclusive(n));
 9327   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9328   ins_cost(VOLATILE_REF_COST);
 9329   effect(KILL cr);
 9330   format %{
 9331     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9332     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9333   %}
 9334   ins_encode %{
 9335     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9336                Assembler::word, /*acquire*/ true, /*release*/ true,
 9337                /*weak*/ true, noreg);
 9338     __ csetw($res$$Register, Assembler::EQ);
 9339   %}
 9340   ins_pipe(pipe_slow);
 9341 %}
 9342 
 9343 // This pattern is generated automatically from cas.m4.
 9344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9345 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9346   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9347   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9348   ins_cost(VOLATILE_REF_COST);
 9349   effect(KILL cr);
 9350   format %{
 9351     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9352     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9353   %}
 9354   ins_encode %{
 9355     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9356                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9357                /*weak*/ true, noreg);
 9358     __ csetw($res$$Register, Assembler::EQ);
 9359   %}
 9360   ins_pipe(pipe_slow);
 9361 %}
 9362 
 9363 // END This section of the file is automatically generated. Do not edit --------------
 9364 // ---------------------------------------------------------------------
 9365 
 9366 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9367   match(Set prev (GetAndSetI mem newv));
 9368   ins_cost(2 * VOLATILE_REF_COST);
 9369   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9370   ins_encode %{
 9371     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9372   %}
 9373   ins_pipe(pipe_serial);
 9374 %}
 9375 
 9376 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9377   match(Set prev (GetAndSetL mem newv));
 9378   ins_cost(2 * VOLATILE_REF_COST);
 9379   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9380   ins_encode %{
 9381     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9382   %}
 9383   ins_pipe(pipe_serial);
 9384 %}
 9385 
 9386 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9387   match(Set prev (GetAndSetN mem newv));
 9388   ins_cost(2 * VOLATILE_REF_COST);
 9389   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9390   ins_encode %{
 9391     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9392   %}
 9393   ins_pipe(pipe_serial);
 9394 %}
 9395 
 9396 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9397   predicate(n->as_LoadStore()->barrier_data() == 0);
 9398   match(Set prev (GetAndSetP mem newv));
 9399   ins_cost(2 * VOLATILE_REF_COST);
 9400   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9401   ins_encode %{
 9402     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9403   %}
 9404   ins_pipe(pipe_serial);
 9405 %}
 9406 
 9407 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9408   predicate(needs_acquiring_load_exclusive(n));
 9409   match(Set prev (GetAndSetI mem newv));
 9410   ins_cost(VOLATILE_REF_COST);
 9411   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9412   ins_encode %{
 9413     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9414   %}
 9415   ins_pipe(pipe_serial);
 9416 %}
 9417 
 9418 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9419   predicate(needs_acquiring_load_exclusive(n));
 9420   match(Set prev (GetAndSetL mem newv));
 9421   ins_cost(VOLATILE_REF_COST);
 9422   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9423   ins_encode %{
 9424     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9425   %}
 9426   ins_pipe(pipe_serial);
 9427 %}
 9428 
 9429 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9430   predicate(needs_acquiring_load_exclusive(n));
 9431   match(Set prev (GetAndSetN mem newv));
 9432   ins_cost(VOLATILE_REF_COST);
 9433   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9434   ins_encode %{
 9435     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9436   %}
 9437   ins_pipe(pipe_serial);
 9438 %}
 9439 
 9440 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9441   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9442   match(Set prev (GetAndSetP mem newv));
 9443   ins_cost(VOLATILE_REF_COST);
 9444   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9445   ins_encode %{
 9446     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9447   %}
 9448   ins_pipe(pipe_serial);
 9449 %}
 9450 
 9451 
 9452 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9453   match(Set newval (GetAndAddL mem incr));
 9454   ins_cost(2 * VOLATILE_REF_COST + 1);
 9455   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9456   ins_encode %{
 9457     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9458   %}
 9459   ins_pipe(pipe_serial);
 9460 %}
 9461 
 9462 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9463   predicate(n->as_LoadStore()->result_not_used());
 9464   match(Set dummy (GetAndAddL mem incr));
 9465   ins_cost(2 * VOLATILE_REF_COST);
 9466   format %{ "get_and_addL [$mem], $incr" %}
 9467   ins_encode %{
 9468     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9469   %}
 9470   ins_pipe(pipe_serial);
 9471 %}
 9472 
 9473 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9474   match(Set newval (GetAndAddL mem incr));
 9475   ins_cost(2 * VOLATILE_REF_COST + 1);
 9476   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9477   ins_encode %{
 9478     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9479   %}
 9480   ins_pipe(pipe_serial);
 9481 %}
 9482 
 9483 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9484   predicate(n->as_LoadStore()->result_not_used());
 9485   match(Set dummy (GetAndAddL mem incr));
 9486   ins_cost(2 * VOLATILE_REF_COST);
 9487   format %{ "get_and_addL [$mem], $incr" %}
 9488   ins_encode %{
 9489     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9490   %}
 9491   ins_pipe(pipe_serial);
 9492 %}
 9493 
 9494 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9495   match(Set newval (GetAndAddI mem incr));
 9496   ins_cost(2 * VOLATILE_REF_COST + 1);
 9497   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9498   ins_encode %{
 9499     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9500   %}
 9501   ins_pipe(pipe_serial);
 9502 %}
 9503 
 9504 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9505   predicate(n->as_LoadStore()->result_not_used());
 9506   match(Set dummy (GetAndAddI mem incr));
 9507   ins_cost(2 * VOLATILE_REF_COST);
 9508   format %{ "get_and_addI [$mem], $incr" %}
 9509   ins_encode %{
 9510     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9511   %}
 9512   ins_pipe(pipe_serial);
 9513 %}
 9514 
 9515 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9516   match(Set newval (GetAndAddI mem incr));
 9517   ins_cost(2 * VOLATILE_REF_COST + 1);
 9518   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9519   ins_encode %{
 9520     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9521   %}
 9522   ins_pipe(pipe_serial);
 9523 %}
 9524 
 9525 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9526   predicate(n->as_LoadStore()->result_not_used());
 9527   match(Set dummy (GetAndAddI mem incr));
 9528   ins_cost(2 * VOLATILE_REF_COST);
 9529   format %{ "get_and_addI [$mem], $incr" %}
 9530   ins_encode %{
 9531     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9532   %}
 9533   ins_pipe(pipe_serial);
 9534 %}
 9535 
 9536 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9537   predicate(needs_acquiring_load_exclusive(n));
 9538   match(Set newval (GetAndAddL mem incr));
 9539   ins_cost(VOLATILE_REF_COST + 1);
 9540   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9541   ins_encode %{
 9542     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9543   %}
 9544   ins_pipe(pipe_serial);
 9545 %}
 9546 
 9547 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9548   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9549   match(Set dummy (GetAndAddL mem incr));
 9550   ins_cost(VOLATILE_REF_COST);
 9551   format %{ "get_and_addL_acq [$mem], $incr" %}
 9552   ins_encode %{
 9553     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9554   %}
 9555   ins_pipe(pipe_serial);
 9556 %}
 9557 
 9558 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9559   predicate(needs_acquiring_load_exclusive(n));
 9560   match(Set newval (GetAndAddL mem incr));
 9561   ins_cost(VOLATILE_REF_COST + 1);
 9562   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9563   ins_encode %{
 9564     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9565   %}
 9566   ins_pipe(pipe_serial);
 9567 %}
 9568 
 9569 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9570   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9571   match(Set dummy (GetAndAddL mem incr));
 9572   ins_cost(VOLATILE_REF_COST);
 9573   format %{ "get_and_addL_acq [$mem], $incr" %}
 9574   ins_encode %{
 9575     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9576   %}
 9577   ins_pipe(pipe_serial);
 9578 %}
 9579 
 9580 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9581   predicate(needs_acquiring_load_exclusive(n));
 9582   match(Set newval (GetAndAddI mem incr));
 9583   ins_cost(VOLATILE_REF_COST + 1);
 9584   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9585   ins_encode %{
 9586     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9587   %}
 9588   ins_pipe(pipe_serial);
 9589 %}
 9590 
 9591 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9592   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9593   match(Set dummy (GetAndAddI mem incr));
 9594   ins_cost(VOLATILE_REF_COST);
 9595   format %{ "get_and_addI_acq [$mem], $incr" %}
 9596   ins_encode %{
 9597     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9598   %}
 9599   ins_pipe(pipe_serial);
 9600 %}
 9601 
 9602 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9603   predicate(needs_acquiring_load_exclusive(n));
 9604   match(Set newval (GetAndAddI mem incr));
 9605   ins_cost(VOLATILE_REF_COST + 1);
 9606   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9607   ins_encode %{
 9608     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9609   %}
 9610   ins_pipe(pipe_serial);
 9611 %}
 9612 
 9613 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9614   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9615   match(Set dummy (GetAndAddI mem incr));
 9616   ins_cost(VOLATILE_REF_COST);
 9617   format %{ "get_and_addI_acq [$mem], $incr" %}
 9618   ins_encode %{
 9619     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9620   %}
 9621   ins_pipe(pipe_serial);
 9622 %}
 9623 
 9624 // Manifest a CmpL result in an integer register.
 9625 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9626 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9627 %{
 9628   match(Set dst (CmpL3 src1 src2));
 9629   effect(KILL flags);
 9630 
 9631   ins_cost(INSN_COST * 6);
 9632   format %{
 9633       "cmp $src1, $src2"
 9634       "csetw $dst, ne"
 9635       "cnegw $dst, lt"
 9636   %}
 9637   // format %{ "CmpL3 $dst, $src1, $src2" %}
 9638   ins_encode %{
 9639     __ cmp($src1$$Register, $src2$$Register);
 9640     __ csetw($dst$$Register, Assembler::NE);
 9641     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9642   %}
 9643 
 9644   ins_pipe(pipe_class_default);
 9645 %}
 9646 
 9647 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9648 %{
 9649   match(Set dst (CmpL3 src1 src2));
 9650   effect(KILL flags);
 9651 
 9652   ins_cost(INSN_COST * 6);
 9653   format %{
 9654       "cmp $src1, $src2"
 9655       "csetw $dst, ne"
 9656       "cnegw $dst, lt"
 9657   %}
 9658   ins_encode %{
 9659     int32_t con = (int32_t)$src2$$constant;
 9660      if (con < 0) {
 9661       __ adds(zr, $src1$$Register, -con);
 9662     } else {
 9663       __ subs(zr, $src1$$Register, con);
 9664     }
 9665     __ csetw($dst$$Register, Assembler::NE);
 9666     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9667   %}
 9668 
 9669   ins_pipe(pipe_class_default);
 9670 %}
 9671 
 9672 // ============================================================================
 9673 // Conditional Move Instructions
 9674 
 9675 // n.b. we have identical rules for both a signed compare op (cmpOp)
 9676 // and an unsigned compare op (cmpOpU). it would be nice if we could
 9677 // define an op class which merged both inputs and use it to type the
 9678 // argument to a single rule. unfortunatelyt his fails because the
 9679 // opclass does not live up to the COND_INTER interface of its
 9680 // component operands. When the generic code tries to negate the
 9681 // operand it ends up running the generci Machoper::negate method
 9682 // which throws a ShouldNotHappen. So, we have to provide two flavours
 9683 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
 9684 
 9685 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9686   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9687 
 9688   ins_cost(INSN_COST * 2);
 9689   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
 9690 
 9691   ins_encode %{
 9692     __ cselw(as_Register($dst$$reg),
 9693              as_Register($src2$$reg),
 9694              as_Register($src1$$reg),
 9695              (Assembler::Condition)$cmp$$cmpcode);
 9696   %}
 9697 
 9698   ins_pipe(icond_reg_reg);
 9699 %}
 9700 
 9701 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9702   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9703 
 9704   ins_cost(INSN_COST * 2);
 9705   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
 9706 
 9707   ins_encode %{
 9708     __ cselw(as_Register($dst$$reg),
 9709              as_Register($src2$$reg),
 9710              as_Register($src1$$reg),
 9711              (Assembler::Condition)$cmp$$cmpcode);
 9712   %}
 9713 
 9714   ins_pipe(icond_reg_reg);
 9715 %}
 9716 
 9717 // special cases where one arg is zero
 9718 
 9719 // n.b. this is selected in preference to the rule above because it
 9720 // avoids loading constant 0 into a source register
 9721 
 9722 // TODO
 9723 // we ought only to be able to cull one of these variants as the ideal
 9724 // transforms ought always to order the zero consistently (to left/right?)
 9725 
 9726 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9727   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9728 
 9729   ins_cost(INSN_COST * 2);
 9730   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
 9731 
 9732   ins_encode %{
 9733     __ cselw(as_Register($dst$$reg),
 9734              as_Register($src$$reg),
 9735              zr,
 9736              (Assembler::Condition)$cmp$$cmpcode);
 9737   %}
 9738 
 9739   ins_pipe(icond_reg);
 9740 %}
 9741 
 9742 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9743   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9744 
 9745   ins_cost(INSN_COST * 2);
 9746   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
 9747 
 9748   ins_encode %{
 9749     __ cselw(as_Register($dst$$reg),
 9750              as_Register($src$$reg),
 9751              zr,
 9752              (Assembler::Condition)$cmp$$cmpcode);
 9753   %}
 9754 
 9755   ins_pipe(icond_reg);
 9756 %}
 9757 
 9758 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9759   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9760 
 9761   ins_cost(INSN_COST * 2);
 9762   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
 9763 
 9764   ins_encode %{
 9765     __ cselw(as_Register($dst$$reg),
 9766              zr,
 9767              as_Register($src$$reg),
 9768              (Assembler::Condition)$cmp$$cmpcode);
 9769   %}
 9770 
 9771   ins_pipe(icond_reg);
 9772 %}
 9773 
 9774 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9775   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9776 
 9777   ins_cost(INSN_COST * 2);
 9778   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
 9779 
 9780   ins_encode %{
 9781     __ cselw(as_Register($dst$$reg),
 9782              zr,
 9783              as_Register($src$$reg),
 9784              (Assembler::Condition)$cmp$$cmpcode);
 9785   %}
 9786 
 9787   ins_pipe(icond_reg);
 9788 %}
 9789 
 9790 // special case for creating a boolean 0 or 1
 9791 
 9792 // n.b. this is selected in preference to the rule above because it
 9793 // avoids loading constants 0 and 1 into a source register
 9794 
 9795 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9796   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9797 
 9798   ins_cost(INSN_COST * 2);
 9799   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
 9800 
 9801   ins_encode %{
 9802     // equivalently
 9803     // cset(as_Register($dst$$reg),
 9804     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9805     __ csincw(as_Register($dst$$reg),
 9806              zr,
 9807              zr,
 9808              (Assembler::Condition)$cmp$$cmpcode);
 9809   %}
 9810 
 9811   ins_pipe(icond_none);
 9812 %}
 9813 
 9814 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9815   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9816 
 9817   ins_cost(INSN_COST * 2);
 9818   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
 9819 
 9820   ins_encode %{
 9821     // equivalently
 9822     // cset(as_Register($dst$$reg),
 9823     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9824     __ csincw(as_Register($dst$$reg),
 9825              zr,
 9826              zr,
 9827              (Assembler::Condition)$cmp$$cmpcode);
 9828   %}
 9829 
 9830   ins_pipe(icond_none);
 9831 %}
 9832 
 9833 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9834   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9835 
 9836   ins_cost(INSN_COST * 2);
 9837   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
 9838 
 9839   ins_encode %{
 9840     __ csel(as_Register($dst$$reg),
 9841             as_Register($src2$$reg),
 9842             as_Register($src1$$reg),
 9843             (Assembler::Condition)$cmp$$cmpcode);
 9844   %}
 9845 
 9846   ins_pipe(icond_reg_reg);
 9847 %}
 9848 
 9849 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9850   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9851 
 9852   ins_cost(INSN_COST * 2);
 9853   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
 9854 
 9855   ins_encode %{
 9856     __ csel(as_Register($dst$$reg),
 9857             as_Register($src2$$reg),
 9858             as_Register($src1$$reg),
 9859             (Assembler::Condition)$cmp$$cmpcode);
 9860   %}
 9861 
 9862   ins_pipe(icond_reg_reg);
 9863 %}
 9864 
 9865 // special cases where one arg is zero
 9866 
 9867 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9868   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9869 
 9870   ins_cost(INSN_COST * 2);
 9871   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
 9872 
 9873   ins_encode %{
 9874     __ csel(as_Register($dst$$reg),
 9875             zr,
 9876             as_Register($src$$reg),
 9877             (Assembler::Condition)$cmp$$cmpcode);
 9878   %}
 9879 
 9880   ins_pipe(icond_reg);
 9881 %}
 9882 
 9883 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9884   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9885 
 9886   ins_cost(INSN_COST * 2);
 9887   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
 9888 
 9889   ins_encode %{
 9890     __ csel(as_Register($dst$$reg),
 9891             zr,
 9892             as_Register($src$$reg),
 9893             (Assembler::Condition)$cmp$$cmpcode);
 9894   %}
 9895 
 9896   ins_pipe(icond_reg);
 9897 %}
 9898 
 9899 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9900   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9901 
 9902   ins_cost(INSN_COST * 2);
 9903   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
 9904 
 9905   ins_encode %{
 9906     __ csel(as_Register($dst$$reg),
 9907             as_Register($src$$reg),
 9908             zr,
 9909             (Assembler::Condition)$cmp$$cmpcode);
 9910   %}
 9911 
 9912   ins_pipe(icond_reg);
 9913 %}
 9914 
 9915 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9916   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9917 
 9918   ins_cost(INSN_COST * 2);
 9919   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
 9920 
 9921   ins_encode %{
 9922     __ csel(as_Register($dst$$reg),
 9923             as_Register($src$$reg),
 9924             zr,
 9925             (Assembler::Condition)$cmp$$cmpcode);
 9926   %}
 9927 
 9928   ins_pipe(icond_reg);
 9929 %}
 9930 
 9931 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
 9932   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
 9933 
 9934   ins_cost(INSN_COST * 2);
 9935   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
 9936 
 9937   ins_encode %{
 9938     __ csel(as_Register($dst$$reg),
 9939             as_Register($src2$$reg),
 9940             as_Register($src1$$reg),
 9941             (Assembler::Condition)$cmp$$cmpcode);
 9942   %}
 9943 
 9944   ins_pipe(icond_reg_reg);
 9945 %}
 9946 
 9947 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
 9948   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
 9949 
 9950   ins_cost(INSN_COST * 2);
 9951   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
 9952 
 9953   ins_encode %{
 9954     __ csel(as_Register($dst$$reg),
 9955             as_Register($src2$$reg),
 9956             as_Register($src1$$reg),
 9957             (Assembler::Condition)$cmp$$cmpcode);
 9958   %}
 9959 
 9960   ins_pipe(icond_reg_reg);
 9961 %}
 9962 
 9963 // special cases where one arg is zero
 9964 
 9965 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
 9966   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
 9967 
 9968   ins_cost(INSN_COST * 2);
 9969   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
 9970 
 9971   ins_encode %{
 9972     __ csel(as_Register($dst$$reg),
 9973             zr,
 9974             as_Register($src$$reg),
 9975             (Assembler::Condition)$cmp$$cmpcode);
 9976   %}
 9977 
 9978   ins_pipe(icond_reg);
 9979 %}
 9980 
 9981 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
 9982   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
 9983 
 9984   ins_cost(INSN_COST * 2);
 9985   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
 9986 
 9987   ins_encode %{
 9988     __ csel(as_Register($dst$$reg),
 9989             zr,
 9990             as_Register($src$$reg),
 9991             (Assembler::Condition)$cmp$$cmpcode);
 9992   %}
 9993 
 9994   ins_pipe(icond_reg);
 9995 %}
 9996 
 9997 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
 9998   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
 9999 
10000   ins_cost(INSN_COST * 2);
10001   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10002 
10003   ins_encode %{
10004     __ csel(as_Register($dst$$reg),
10005             as_Register($src$$reg),
10006             zr,
10007             (Assembler::Condition)$cmp$$cmpcode);
10008   %}
10009 
10010   ins_pipe(icond_reg);
10011 %}
10012 
10013 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10014   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10015 
10016   ins_cost(INSN_COST * 2);
10017   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10018 
10019   ins_encode %{
10020     __ csel(as_Register($dst$$reg),
10021             as_Register($src$$reg),
10022             zr,
10023             (Assembler::Condition)$cmp$$cmpcode);
10024   %}
10025 
10026   ins_pipe(icond_reg);
10027 %}
10028 
10029 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10030   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10031 
10032   ins_cost(INSN_COST * 2);
10033   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10034 
10035   ins_encode %{
10036     __ cselw(as_Register($dst$$reg),
10037              as_Register($src2$$reg),
10038              as_Register($src1$$reg),
10039              (Assembler::Condition)$cmp$$cmpcode);
10040   %}
10041 
10042   ins_pipe(icond_reg_reg);
10043 %}
10044 
10045 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10046   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10047 
10048   ins_cost(INSN_COST * 2);
10049   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10050 
10051   ins_encode %{
10052     __ cselw(as_Register($dst$$reg),
10053              as_Register($src2$$reg),
10054              as_Register($src1$$reg),
10055              (Assembler::Condition)$cmp$$cmpcode);
10056   %}
10057 
10058   ins_pipe(icond_reg_reg);
10059 %}
10060 
10061 // special cases where one arg is zero
10062 
10063 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10064   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10065 
10066   ins_cost(INSN_COST * 2);
10067   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10068 
10069   ins_encode %{
10070     __ cselw(as_Register($dst$$reg),
10071              zr,
10072              as_Register($src$$reg),
10073              (Assembler::Condition)$cmp$$cmpcode);
10074   %}
10075 
10076   ins_pipe(icond_reg);
10077 %}
10078 
10079 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10080   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10081 
10082   ins_cost(INSN_COST * 2);
10083   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10084 
10085   ins_encode %{
10086     __ cselw(as_Register($dst$$reg),
10087              zr,
10088              as_Register($src$$reg),
10089              (Assembler::Condition)$cmp$$cmpcode);
10090   %}
10091 
10092   ins_pipe(icond_reg);
10093 %}
10094 
10095 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10096   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10097 
10098   ins_cost(INSN_COST * 2);
10099   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10100 
10101   ins_encode %{
10102     __ cselw(as_Register($dst$$reg),
10103              as_Register($src$$reg),
10104              zr,
10105              (Assembler::Condition)$cmp$$cmpcode);
10106   %}
10107 
10108   ins_pipe(icond_reg);
10109 %}
10110 
10111 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10112   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10113 
10114   ins_cost(INSN_COST * 2);
10115   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10116 
10117   ins_encode %{
10118     __ cselw(as_Register($dst$$reg),
10119              as_Register($src$$reg),
10120              zr,
10121              (Assembler::Condition)$cmp$$cmpcode);
10122   %}
10123 
10124   ins_pipe(icond_reg);
10125 %}
10126 
10127 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10128 %{
10129   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10130 
10131   ins_cost(INSN_COST * 3);
10132 
10133   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10134   ins_encode %{
10135     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10136     __ fcsels(as_FloatRegister($dst$$reg),
10137               as_FloatRegister($src2$$reg),
10138               as_FloatRegister($src1$$reg),
10139               cond);
10140   %}
10141 
10142   ins_pipe(fp_cond_reg_reg_s);
10143 %}
10144 
10145 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10146 %{
10147   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10148 
10149   ins_cost(INSN_COST * 3);
10150 
10151   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10152   ins_encode %{
10153     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10154     __ fcsels(as_FloatRegister($dst$$reg),
10155               as_FloatRegister($src2$$reg),
10156               as_FloatRegister($src1$$reg),
10157               cond);
10158   %}
10159 
10160   ins_pipe(fp_cond_reg_reg_s);
10161 %}
10162 
10163 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10164 %{
10165   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10166 
10167   ins_cost(INSN_COST * 3);
10168 
10169   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10170   ins_encode %{
10171     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10172     __ fcseld(as_FloatRegister($dst$$reg),
10173               as_FloatRegister($src2$$reg),
10174               as_FloatRegister($src1$$reg),
10175               cond);
10176   %}
10177 
10178   ins_pipe(fp_cond_reg_reg_d);
10179 %}
10180 
10181 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10182 %{
10183   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10184 
10185   ins_cost(INSN_COST * 3);
10186 
10187   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10188   ins_encode %{
10189     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10190     __ fcseld(as_FloatRegister($dst$$reg),
10191               as_FloatRegister($src2$$reg),
10192               as_FloatRegister($src1$$reg),
10193               cond);
10194   %}
10195 
10196   ins_pipe(fp_cond_reg_reg_d);
10197 %}
10198 
10199 // ============================================================================
10200 // Arithmetic Instructions
10201 //
10202 
10203 // Integer Addition
10204 
10205 // TODO
10206 // these currently employ operations which do not set CR and hence are
10207 // not flagged as killing CR but we would like to isolate the cases
10208 // where we want to set flags from those where we don't. need to work
10209 // out how to do that.
10210 
10211 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10212   match(Set dst (AddI src1 src2));
10213 
10214   ins_cost(INSN_COST);
10215   format %{ "addw  $dst, $src1, $src2" %}
10216 
10217   ins_encode %{
10218     __ addw(as_Register($dst$$reg),
10219             as_Register($src1$$reg),
10220             as_Register($src2$$reg));
10221   %}
10222 
10223   ins_pipe(ialu_reg_reg);
10224 %}
10225 
10226 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10227   match(Set dst (AddI src1 src2));
10228 
10229   ins_cost(INSN_COST);
10230   format %{ "addw $dst, $src1, $src2" %}
10231 
10232   // use opcode to indicate that this is an add not a sub
10233   opcode(0x0);
10234 
10235   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10236 
10237   ins_pipe(ialu_reg_imm);
10238 %}
10239 
10240 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10241   match(Set dst (AddI (ConvL2I src1) src2));
10242 
10243   ins_cost(INSN_COST);
10244   format %{ "addw $dst, $src1, $src2" %}
10245 
10246   // use opcode to indicate that this is an add not a sub
10247   opcode(0x0);
10248 
10249   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10250 
10251   ins_pipe(ialu_reg_imm);
10252 %}
10253 
10254 // Pointer Addition
10255 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10256   match(Set dst (AddP src1 src2));
10257 
10258   ins_cost(INSN_COST);
10259   format %{ "add $dst, $src1, $src2\t# ptr" %}
10260 
10261   ins_encode %{
10262     __ add(as_Register($dst$$reg),
10263            as_Register($src1$$reg),
10264            as_Register($src2$$reg));
10265   %}
10266 
10267   ins_pipe(ialu_reg_reg);
10268 %}
10269 
10270 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10271   match(Set dst (AddP src1 (ConvI2L src2)));
10272 
10273   ins_cost(1.9 * INSN_COST);
10274   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10275 
10276   ins_encode %{
10277     __ add(as_Register($dst$$reg),
10278            as_Register($src1$$reg),
10279            as_Register($src2$$reg), ext::sxtw);
10280   %}
10281 
10282   ins_pipe(ialu_reg_reg);
10283 %}
10284 
10285 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10286   match(Set dst (AddP src1 (LShiftL src2 scale)));
10287 
10288   ins_cost(1.9 * INSN_COST);
10289   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10290 
10291   ins_encode %{
10292     __ lea(as_Register($dst$$reg),
10293            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10294                    Address::lsl($scale$$constant)));
10295   %}
10296 
10297   ins_pipe(ialu_reg_reg_shift);
10298 %}
10299 
10300 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10301   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10302 
10303   ins_cost(1.9 * INSN_COST);
10304   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10305 
10306   ins_encode %{
10307     __ lea(as_Register($dst$$reg),
10308            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10309                    Address::sxtw($scale$$constant)));
10310   %}
10311 
10312   ins_pipe(ialu_reg_reg_shift);
10313 %}
10314 
10315 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10316   match(Set dst (LShiftL (ConvI2L src) scale));
10317 
10318   ins_cost(INSN_COST);
10319   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10320 
10321   ins_encode %{
10322     __ sbfiz(as_Register($dst$$reg),
10323           as_Register($src$$reg),
10324           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10325   %}
10326 
10327   ins_pipe(ialu_reg_shift);
10328 %}
10329 
10330 // Pointer Immediate Addition
10331 // n.b. this needs to be more expensive than using an indirect memory
10332 // operand
10333 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10334   match(Set dst (AddP src1 src2));
10335 
10336   ins_cost(INSN_COST);
10337   format %{ "add $dst, $src1, $src2\t# ptr" %}
10338 
10339   // use opcode to indicate that this is an add not a sub
10340   opcode(0x0);
10341 
10342   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10343 
10344   ins_pipe(ialu_reg_imm);
10345 %}
10346 
10347 // Long Addition
10348 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10349 
10350   match(Set dst (AddL src1 src2));
10351 
10352   ins_cost(INSN_COST);
10353   format %{ "add  $dst, $src1, $src2" %}
10354 
10355   ins_encode %{
10356     __ add(as_Register($dst$$reg),
10357            as_Register($src1$$reg),
10358            as_Register($src2$$reg));
10359   %}
10360 
10361   ins_pipe(ialu_reg_reg);
10362 %}
10363 
10364 // No constant pool entries requiredLong Immediate Addition.
10365 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10366   match(Set dst (AddL src1 src2));
10367 
10368   ins_cost(INSN_COST);
10369   format %{ "add $dst, $src1, $src2" %}
10370 
10371   // use opcode to indicate that this is an add not a sub
10372   opcode(0x0);
10373 
10374   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10375 
10376   ins_pipe(ialu_reg_imm);
10377 %}
10378 
10379 // Integer Subtraction
10380 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10381   match(Set dst (SubI src1 src2));
10382 
10383   ins_cost(INSN_COST);
10384   format %{ "subw  $dst, $src1, $src2" %}
10385 
10386   ins_encode %{
10387     __ subw(as_Register($dst$$reg),
10388             as_Register($src1$$reg),
10389             as_Register($src2$$reg));
10390   %}
10391 
10392   ins_pipe(ialu_reg_reg);
10393 %}
10394 
10395 // Immediate Subtraction
10396 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10397   match(Set dst (SubI src1 src2));
10398 
10399   ins_cost(INSN_COST);
10400   format %{ "subw $dst, $src1, $src2" %}
10401 
10402   // use opcode to indicate that this is a sub not an add
10403   opcode(0x1);
10404 
10405   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10406 
10407   ins_pipe(ialu_reg_imm);
10408 %}
10409 
10410 // Long Subtraction
10411 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10412 
10413   match(Set dst (SubL src1 src2));
10414 
10415   ins_cost(INSN_COST);
10416   format %{ "sub  $dst, $src1, $src2" %}
10417 
10418   ins_encode %{
10419     __ sub(as_Register($dst$$reg),
10420            as_Register($src1$$reg),
10421            as_Register($src2$$reg));
10422   %}
10423 
10424   ins_pipe(ialu_reg_reg);
10425 %}
10426 
10427 // No constant pool entries requiredLong Immediate Subtraction.
10428 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10429   match(Set dst (SubL src1 src2));
10430 
10431   ins_cost(INSN_COST);
10432   format %{ "sub$dst, $src1, $src2" %}
10433 
10434   // use opcode to indicate that this is a sub not an add
10435   opcode(0x1);
10436 
10437   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10438 
10439   ins_pipe(ialu_reg_imm);
10440 %}
10441 
10442 // Integer Negation (special case for sub)
10443 
10444 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10445   match(Set dst (SubI zero src));
10446 
10447   ins_cost(INSN_COST);
10448   format %{ "negw $dst, $src\t# int" %}
10449 
10450   ins_encode %{
10451     __ negw(as_Register($dst$$reg),
10452             as_Register($src$$reg));
10453   %}
10454 
10455   ins_pipe(ialu_reg);
10456 %}
10457 
10458 // Long Negation
10459 
10460 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10461   match(Set dst (SubL zero src));
10462 
10463   ins_cost(INSN_COST);
10464   format %{ "neg $dst, $src\t# long" %}
10465 
10466   ins_encode %{
10467     __ neg(as_Register($dst$$reg),
10468            as_Register($src$$reg));
10469   %}
10470 
10471   ins_pipe(ialu_reg);
10472 %}
10473 
10474 // Integer Multiply
10475 
10476 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10477   match(Set dst (MulI src1 src2));
10478 
10479   ins_cost(INSN_COST * 3);
10480   format %{ "mulw  $dst, $src1, $src2" %}
10481 
10482   ins_encode %{
10483     __ mulw(as_Register($dst$$reg),
10484             as_Register($src1$$reg),
10485             as_Register($src2$$reg));
10486   %}
10487 
10488   ins_pipe(imul_reg_reg);
10489 %}
10490 
10491 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10492   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10493 
10494   ins_cost(INSN_COST * 3);
10495   format %{ "smull  $dst, $src1, $src2" %}
10496 
10497   ins_encode %{
10498     __ smull(as_Register($dst$$reg),
10499              as_Register($src1$$reg),
10500              as_Register($src2$$reg));
10501   %}
10502 
10503   ins_pipe(imul_reg_reg);
10504 %}
10505 
10506 // Long Multiply
10507 
10508 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10509   match(Set dst (MulL src1 src2));
10510 
10511   ins_cost(INSN_COST * 5);
10512   format %{ "mul  $dst, $src1, $src2" %}
10513 
10514   ins_encode %{
10515     __ mul(as_Register($dst$$reg),
10516            as_Register($src1$$reg),
10517            as_Register($src2$$reg));
10518   %}
10519 
10520   ins_pipe(lmul_reg_reg);
10521 %}
10522 
10523 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10524 %{
10525   match(Set dst (MulHiL src1 src2));
10526 
10527   ins_cost(INSN_COST * 7);
10528   format %{ "smulh   $dst, $src1, $src2\t# mulhi" %}
10529 
10530   ins_encode %{
10531     __ smulh(as_Register($dst$$reg),
10532              as_Register($src1$$reg),
10533              as_Register($src2$$reg));
10534   %}
10535 
10536   ins_pipe(lmul_reg_reg);
10537 %}
10538 
10539 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10540 %{
10541   match(Set dst (UMulHiL src1 src2));
10542 
10543   ins_cost(INSN_COST * 7);
10544   format %{ "umulh   $dst, $src1, $src2\t# umulhi" %}
10545 
10546   ins_encode %{
10547     __ umulh(as_Register($dst$$reg),
10548              as_Register($src1$$reg),
10549              as_Register($src2$$reg));
10550   %}
10551 
10552   ins_pipe(lmul_reg_reg);
10553 %}
10554 
10555 // Combined Integer Multiply & Add/Sub
10556 
10557 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10558   match(Set dst (AddI src3 (MulI src1 src2)));
10559 
10560   ins_cost(INSN_COST * 3);
10561   format %{ "madd  $dst, $src1, $src2, $src3" %}
10562 
10563   ins_encode %{
10564     __ maddw(as_Register($dst$$reg),
10565              as_Register($src1$$reg),
10566              as_Register($src2$$reg),
10567              as_Register($src3$$reg));
10568   %}
10569 
10570   ins_pipe(imac_reg_reg);
10571 %}
10572 
10573 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10574   match(Set dst (SubI src3 (MulI src1 src2)));
10575 
10576   ins_cost(INSN_COST * 3);
10577   format %{ "msub  $dst, $src1, $src2, $src3" %}
10578 
10579   ins_encode %{
10580     __ msubw(as_Register($dst$$reg),
10581              as_Register($src1$$reg),
10582              as_Register($src2$$reg),
10583              as_Register($src3$$reg));
10584   %}
10585 
10586   ins_pipe(imac_reg_reg);
10587 %}
10588 
10589 // Combined Integer Multiply & Neg
10590 
10591 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10592   match(Set dst (MulI (SubI zero src1) src2));
10593 
10594   ins_cost(INSN_COST * 3);
10595   format %{ "mneg  $dst, $src1, $src2" %}
10596 
10597   ins_encode %{
10598     __ mnegw(as_Register($dst$$reg),
10599              as_Register($src1$$reg),
10600              as_Register($src2$$reg));
10601   %}
10602 
10603   ins_pipe(imac_reg_reg);
10604 %}
10605 
10606 // Combined Long Multiply & Add/Sub
10607 
10608 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10609   match(Set dst (AddL src3 (MulL src1 src2)));
10610 
10611   ins_cost(INSN_COST * 5);
10612   format %{ "madd  $dst, $src1, $src2, $src3" %}
10613 
10614   ins_encode %{
10615     __ madd(as_Register($dst$$reg),
10616             as_Register($src1$$reg),
10617             as_Register($src2$$reg),
10618             as_Register($src3$$reg));
10619   %}
10620 
10621   ins_pipe(lmac_reg_reg);
10622 %}
10623 
10624 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10625   match(Set dst (SubL src3 (MulL src1 src2)));
10626 
10627   ins_cost(INSN_COST * 5);
10628   format %{ "msub  $dst, $src1, $src2, $src3" %}
10629 
10630   ins_encode %{
10631     __ msub(as_Register($dst$$reg),
10632             as_Register($src1$$reg),
10633             as_Register($src2$$reg),
10634             as_Register($src3$$reg));
10635   %}
10636 
10637   ins_pipe(lmac_reg_reg);
10638 %}
10639 
10640 // Combined Long Multiply & Neg
10641 
10642 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10643   match(Set dst (MulL (SubL zero src1) src2));
10644 
10645   ins_cost(INSN_COST * 5);
10646   format %{ "mneg  $dst, $src1, $src2" %}
10647 
10648   ins_encode %{
10649     __ mneg(as_Register($dst$$reg),
10650             as_Register($src1$$reg),
10651             as_Register($src2$$reg));
10652   %}
10653 
10654   ins_pipe(lmac_reg_reg);
10655 %}
10656 
10657 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10658 
10659 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10660   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10661 
10662   ins_cost(INSN_COST * 3);
10663   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10664 
10665   ins_encode %{
10666     __ smaddl(as_Register($dst$$reg),
10667               as_Register($src1$$reg),
10668               as_Register($src2$$reg),
10669               as_Register($src3$$reg));
10670   %}
10671 
10672   ins_pipe(imac_reg_reg);
10673 %}
10674 
10675 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10676   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10677 
10678   ins_cost(INSN_COST * 3);
10679   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10680 
10681   ins_encode %{
10682     __ smsubl(as_Register($dst$$reg),
10683               as_Register($src1$$reg),
10684               as_Register($src2$$reg),
10685               as_Register($src3$$reg));
10686   %}
10687 
10688   ins_pipe(imac_reg_reg);
10689 %}
10690 
10691 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10692   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10693 
10694   ins_cost(INSN_COST * 3);
10695   format %{ "smnegl  $dst, $src1, $src2" %}
10696 
10697   ins_encode %{
10698     __ smnegl(as_Register($dst$$reg),
10699               as_Register($src1$$reg),
10700               as_Register($src2$$reg));
10701   %}
10702 
10703   ins_pipe(imac_reg_reg);
10704 %}
10705 
10706 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10707 
10708 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10709   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10710 
10711   ins_cost(INSN_COST * 5);
10712   format %{ "mulw  rscratch1, $src1, $src2\n\t"
10713             "maddw $dst, $src3, $src4, rscratch1" %}
10714 
10715   ins_encode %{
10716     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10717     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10718 
10719   ins_pipe(imac_reg_reg);
10720 %}
10721 
10722 // Integer Divide
10723 
10724 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10725   match(Set dst (DivI src1 src2));
10726 
10727   ins_cost(INSN_COST * 19);
10728   format %{ "sdivw  $dst, $src1, $src2" %}
10729 
10730   ins_encode(aarch64_enc_divw(dst, src1, src2));
10731   ins_pipe(idiv_reg_reg);
10732 %}
10733 
10734 // Long Divide
10735 
10736 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10737   match(Set dst (DivL src1 src2));
10738 
10739   ins_cost(INSN_COST * 35);
10740   format %{ "sdiv   $dst, $src1, $src2" %}
10741 
10742   ins_encode(aarch64_enc_div(dst, src1, src2));
10743   ins_pipe(ldiv_reg_reg);
10744 %}
10745 
10746 // Integer Remainder
10747 
10748 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10749   match(Set dst (ModI src1 src2));
10750 
10751   ins_cost(INSN_COST * 22);
10752   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
10753             "msubw  $dst, rscratch1, $src2, $src1" %}
10754 
10755   ins_encode(aarch64_enc_modw(dst, src1, src2));
10756   ins_pipe(idiv_reg_reg);
10757 %}
10758 
10759 // Long Remainder
10760 
10761 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10762   match(Set dst (ModL src1 src2));
10763 
10764   ins_cost(INSN_COST * 38);
10765   format %{ "sdiv   rscratch1, $src1, $src2\n"
10766             "msub   $dst, rscratch1, $src2, $src1" %}
10767 
10768   ins_encode(aarch64_enc_mod(dst, src1, src2));
10769   ins_pipe(ldiv_reg_reg);
10770 %}
10771 
10772 // Unsigned Integer Divide
10773 
10774 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10775   match(Set dst (UDivI src1 src2));
10776 
10777   ins_cost(INSN_COST * 19);
10778   format %{ "udivw  $dst, $src1, $src2" %}
10779 
10780   ins_encode %{
10781     __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10782   %}
10783 
10784   ins_pipe(idiv_reg_reg);
10785 %}
10786 
10787 //  Unsigned Long Divide
10788 
10789 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10790   match(Set dst (UDivL src1 src2));
10791 
10792   ins_cost(INSN_COST * 35);
10793   format %{ "udiv   $dst, $src1, $src2" %}
10794 
10795   ins_encode %{
10796     __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10797   %}
10798 
10799   ins_pipe(ldiv_reg_reg);
10800 %}
10801 
10802 // Unsigned Integer Remainder
10803 
10804 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10805   match(Set dst (UModI src1 src2));
10806 
10807   ins_cost(INSN_COST * 22);
10808   format %{ "udivw  rscratch1, $src1, $src2\n\t"
10809             "msubw  $dst, rscratch1, $src2, $src1" %}
10810 
10811   ins_encode %{
10812     __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10813     __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10814   %}
10815 
10816   ins_pipe(idiv_reg_reg);
10817 %}
10818 
10819 // Unsigned Long Remainder
10820 
10821 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10822   match(Set dst (UModL src1 src2));
10823 
10824   ins_cost(INSN_COST * 38);
10825   format %{ "udiv   rscratch1, $src1, $src2\n"
10826             "msub   $dst, rscratch1, $src2, $src1" %}
10827 
10828   ins_encode %{
10829     __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10830     __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10831   %}
10832 
10833   ins_pipe(ldiv_reg_reg);
10834 %}
10835 
10836 // Integer Shifts
10837 
10838 // Shift Left Register
10839 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10840   match(Set dst (LShiftI src1 src2));
10841 
10842   ins_cost(INSN_COST * 2);
10843   format %{ "lslvw  $dst, $src1, $src2" %}
10844 
10845   ins_encode %{
10846     __ lslvw(as_Register($dst$$reg),
10847              as_Register($src1$$reg),
10848              as_Register($src2$$reg));
10849   %}
10850 
10851   ins_pipe(ialu_reg_reg_vshift);
10852 %}
10853 
10854 // Shift Left Immediate
10855 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10856   match(Set dst (LShiftI src1 src2));
10857 
10858   ins_cost(INSN_COST);
10859   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10860 
10861   ins_encode %{
10862     __ lslw(as_Register($dst$$reg),
10863             as_Register($src1$$reg),
10864             $src2$$constant & 0x1f);
10865   %}
10866 
10867   ins_pipe(ialu_reg_shift);
10868 %}
10869 
10870 // Shift Right Logical Register
10871 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10872   match(Set dst (URShiftI src1 src2));
10873 
10874   ins_cost(INSN_COST * 2);
10875   format %{ "lsrvw  $dst, $src1, $src2" %}
10876 
10877   ins_encode %{
10878     __ lsrvw(as_Register($dst$$reg),
10879              as_Register($src1$$reg),
10880              as_Register($src2$$reg));
10881   %}
10882 
10883   ins_pipe(ialu_reg_reg_vshift);
10884 %}
10885 
10886 // Shift Right Logical Immediate
10887 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10888   match(Set dst (URShiftI src1 src2));
10889 
10890   ins_cost(INSN_COST);
10891   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10892 
10893   ins_encode %{
10894     __ lsrw(as_Register($dst$$reg),
10895             as_Register($src1$$reg),
10896             $src2$$constant & 0x1f);
10897   %}
10898 
10899   ins_pipe(ialu_reg_shift);
10900 %}
10901 
10902 // Shift Right Arithmetic Register
10903 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10904   match(Set dst (RShiftI src1 src2));
10905 
10906   ins_cost(INSN_COST * 2);
10907   format %{ "asrvw  $dst, $src1, $src2" %}
10908 
10909   ins_encode %{
10910     __ asrvw(as_Register($dst$$reg),
10911              as_Register($src1$$reg),
10912              as_Register($src2$$reg));
10913   %}
10914 
10915   ins_pipe(ialu_reg_reg_vshift);
10916 %}
10917 
10918 // Shift Right Arithmetic Immediate
10919 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10920   match(Set dst (RShiftI src1 src2));
10921 
10922   ins_cost(INSN_COST);
10923   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10924 
10925   ins_encode %{
10926     __ asrw(as_Register($dst$$reg),
10927             as_Register($src1$$reg),
10928             $src2$$constant & 0x1f);
10929   %}
10930 
10931   ins_pipe(ialu_reg_shift);
10932 %}
10933 
10934 // Combined Int Mask and Right Shift (using UBFM)
10935 // TODO
10936 
10937 // Long Shifts
10938 
10939 // Shift Left Register
10940 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10941   match(Set dst (LShiftL src1 src2));
10942 
10943   ins_cost(INSN_COST * 2);
10944   format %{ "lslv  $dst, $src1, $src2" %}
10945 
10946   ins_encode %{
10947     __ lslv(as_Register($dst$$reg),
10948             as_Register($src1$$reg),
10949             as_Register($src2$$reg));
10950   %}
10951 
10952   ins_pipe(ialu_reg_reg_vshift);
10953 %}
10954 
10955 // Shift Left Immediate
10956 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10957   match(Set dst (LShiftL src1 src2));
10958 
10959   ins_cost(INSN_COST);
10960   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10961 
10962   ins_encode %{
10963     __ lsl(as_Register($dst$$reg),
10964             as_Register($src1$$reg),
10965             $src2$$constant & 0x3f);
10966   %}
10967 
10968   ins_pipe(ialu_reg_shift);
10969 %}
10970 
10971 // Shift Right Logical Register
10972 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10973   match(Set dst (URShiftL src1 src2));
10974 
10975   ins_cost(INSN_COST * 2);
10976   format %{ "lsrv  $dst, $src1, $src2" %}
10977 
10978   ins_encode %{
10979     __ lsrv(as_Register($dst$$reg),
10980             as_Register($src1$$reg),
10981             as_Register($src2$$reg));
10982   %}
10983 
10984   ins_pipe(ialu_reg_reg_vshift);
10985 %}
10986 
10987 // Shift Right Logical Immediate
10988 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10989   match(Set dst (URShiftL src1 src2));
10990 
10991   ins_cost(INSN_COST);
10992   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10993 
10994   ins_encode %{
10995     __ lsr(as_Register($dst$$reg),
10996            as_Register($src1$$reg),
10997            $src2$$constant & 0x3f);
10998   %}
10999 
11000   ins_pipe(ialu_reg_shift);
11001 %}
11002 
11003 // A special-case pattern for card table stores.
11004 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11005   match(Set dst (URShiftL (CastP2X src1) src2));
11006 
11007   ins_cost(INSN_COST);
11008   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11009 
11010   ins_encode %{
11011     __ lsr(as_Register($dst$$reg),
11012            as_Register($src1$$reg),
11013            $src2$$constant & 0x3f);
11014   %}
11015 
11016   ins_pipe(ialu_reg_shift);
11017 %}
11018 
11019 // Shift Right Arithmetic Register
11020 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11021   match(Set dst (RShiftL src1 src2));
11022 
11023   ins_cost(INSN_COST * 2);
11024   format %{ "asrv  $dst, $src1, $src2" %}
11025 
11026   ins_encode %{
11027     __ asrv(as_Register($dst$$reg),
11028             as_Register($src1$$reg),
11029             as_Register($src2$$reg));
11030   %}
11031 
11032   ins_pipe(ialu_reg_reg_vshift);
11033 %}
11034 
11035 // Shift Right Arithmetic Immediate
11036 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11037   match(Set dst (RShiftL src1 src2));
11038 
11039   ins_cost(INSN_COST);
11040   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11041 
11042   ins_encode %{
11043     __ asr(as_Register($dst$$reg),
11044            as_Register($src1$$reg),
11045            $src2$$constant & 0x3f);
11046   %}
11047 
11048   ins_pipe(ialu_reg_shift);
11049 %}
11050 
11051 // BEGIN This section of the file is automatically generated. Do not edit --------------
11052 // This section is generated from aarch64_ad.m4
11053 
11054 
11055 // This pattern is automatically generated from aarch64_ad.m4
11056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11057 instruct regL_not_reg(iRegLNoSp dst,
11058                          iRegL src1, immL_M1 m1,
11059                          rFlagsReg cr) %{
11060   match(Set dst (XorL src1 m1));
11061   ins_cost(INSN_COST);
11062   format %{ "eon  $dst, $src1, zr" %}
11063 
11064   ins_encode %{
11065     __ eon(as_Register($dst$$reg),
11066               as_Register($src1$$reg),
11067               zr,
11068               Assembler::LSL, 0);
11069   %}
11070 
11071   ins_pipe(ialu_reg);
11072 %}
11073 
11074 // This pattern is automatically generated from aarch64_ad.m4
11075 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11076 instruct regI_not_reg(iRegINoSp dst,
11077                          iRegIorL2I src1, immI_M1 m1,
11078                          rFlagsReg cr) %{
11079   match(Set dst (XorI src1 m1));
11080   ins_cost(INSN_COST);
11081   format %{ "eonw  $dst, $src1, zr" %}
11082 
11083   ins_encode %{
11084     __ eonw(as_Register($dst$$reg),
11085               as_Register($src1$$reg),
11086               zr,
11087               Assembler::LSL, 0);
11088   %}
11089 
11090   ins_pipe(ialu_reg);
11091 %}
11092 
11093 // This pattern is automatically generated from aarch64_ad.m4
11094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11095 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11096                               immI0 zero, iRegIorL2I src1, immI src2) %{
11097   match(Set dst (SubI zero (URShiftI src1 src2)));
11098 
11099   ins_cost(1.9 * INSN_COST);
11100   format %{ "negw  $dst, $src1, LSR $src2" %}
11101 
11102   ins_encode %{
11103     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11104             Assembler::LSR, $src2$$constant & 0x1f);
11105   %}
11106 
11107   ins_pipe(ialu_reg_shift);
11108 %}
11109 
11110 // This pattern is automatically generated from aarch64_ad.m4
11111 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11112 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11113                               immI0 zero, iRegIorL2I src1, immI src2) %{
11114   match(Set dst (SubI zero (RShiftI src1 src2)));
11115 
11116   ins_cost(1.9 * INSN_COST);
11117   format %{ "negw  $dst, $src1, ASR $src2" %}
11118 
11119   ins_encode %{
11120     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11121             Assembler::ASR, $src2$$constant & 0x1f);
11122   %}
11123 
11124   ins_pipe(ialu_reg_shift);
11125 %}
11126 
11127 // This pattern is automatically generated from aarch64_ad.m4
11128 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11129 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11130                               immI0 zero, iRegIorL2I src1, immI src2) %{
11131   match(Set dst (SubI zero (LShiftI src1 src2)));
11132 
11133   ins_cost(1.9 * INSN_COST);
11134   format %{ "negw  $dst, $src1, LSL $src2" %}
11135 
11136   ins_encode %{
11137     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11138             Assembler::LSL, $src2$$constant & 0x1f);
11139   %}
11140 
11141   ins_pipe(ialu_reg_shift);
11142 %}
11143 
11144 // This pattern is automatically generated from aarch64_ad.m4
11145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11146 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11147                               immL0 zero, iRegL src1, immI src2) %{
11148   match(Set dst (SubL zero (URShiftL src1 src2)));
11149 
11150   ins_cost(1.9 * INSN_COST);
11151   format %{ "neg  $dst, $src1, LSR $src2" %}
11152 
11153   ins_encode %{
11154     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11155             Assembler::LSR, $src2$$constant & 0x3f);
11156   %}
11157 
11158   ins_pipe(ialu_reg_shift);
11159 %}
11160 
11161 // This pattern is automatically generated from aarch64_ad.m4
11162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11163 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11164                               immL0 zero, iRegL src1, immI src2) %{
11165   match(Set dst (SubL zero (RShiftL src1 src2)));
11166 
11167   ins_cost(1.9 * INSN_COST);
11168   format %{ "neg  $dst, $src1, ASR $src2" %}
11169 
11170   ins_encode %{
11171     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11172             Assembler::ASR, $src2$$constant & 0x3f);
11173   %}
11174 
11175   ins_pipe(ialu_reg_shift);
11176 %}
11177 
11178 // This pattern is automatically generated from aarch64_ad.m4
11179 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11180 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11181                               immL0 zero, iRegL src1, immI src2) %{
11182   match(Set dst (SubL zero (LShiftL src1 src2)));
11183 
11184   ins_cost(1.9 * INSN_COST);
11185   format %{ "neg  $dst, $src1, LSL $src2" %}
11186 
11187   ins_encode %{
11188     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11189             Assembler::LSL, $src2$$constant & 0x3f);
11190   %}
11191 
11192   ins_pipe(ialu_reg_shift);
11193 %}
11194 
11195 // This pattern is automatically generated from aarch64_ad.m4
11196 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11197 instruct AndI_reg_not_reg(iRegINoSp dst,
11198                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11199   match(Set dst (AndI src1 (XorI src2 m1)));
11200   ins_cost(INSN_COST);
11201   format %{ "bicw  $dst, $src1, $src2" %}
11202 
11203   ins_encode %{
11204     __ bicw(as_Register($dst$$reg),
11205               as_Register($src1$$reg),
11206               as_Register($src2$$reg),
11207               Assembler::LSL, 0);
11208   %}
11209 
11210   ins_pipe(ialu_reg_reg);
11211 %}
11212 
11213 // This pattern is automatically generated from aarch64_ad.m4
11214 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11215 instruct AndL_reg_not_reg(iRegLNoSp dst,
11216                          iRegL src1, iRegL src2, immL_M1 m1) %{
11217   match(Set dst (AndL src1 (XorL src2 m1)));
11218   ins_cost(INSN_COST);
11219   format %{ "bic  $dst, $src1, $src2" %}
11220 
11221   ins_encode %{
11222     __ bic(as_Register($dst$$reg),
11223               as_Register($src1$$reg),
11224               as_Register($src2$$reg),
11225               Assembler::LSL, 0);
11226   %}
11227 
11228   ins_pipe(ialu_reg_reg);
11229 %}
11230 
11231 // This pattern is automatically generated from aarch64_ad.m4
11232 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11233 instruct OrI_reg_not_reg(iRegINoSp dst,
11234                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11235   match(Set dst (OrI src1 (XorI src2 m1)));
11236   ins_cost(INSN_COST);
11237   format %{ "ornw  $dst, $src1, $src2" %}
11238 
11239   ins_encode %{
11240     __ ornw(as_Register($dst$$reg),
11241               as_Register($src1$$reg),
11242               as_Register($src2$$reg),
11243               Assembler::LSL, 0);
11244   %}
11245 
11246   ins_pipe(ialu_reg_reg);
11247 %}
11248 
11249 // This pattern is automatically generated from aarch64_ad.m4
11250 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11251 instruct OrL_reg_not_reg(iRegLNoSp dst,
11252                          iRegL src1, iRegL src2, immL_M1 m1) %{
11253   match(Set dst (OrL src1 (XorL src2 m1)));
11254   ins_cost(INSN_COST);
11255   format %{ "orn  $dst, $src1, $src2" %}
11256 
11257   ins_encode %{
11258     __ orn(as_Register($dst$$reg),
11259               as_Register($src1$$reg),
11260               as_Register($src2$$reg),
11261               Assembler::LSL, 0);
11262   %}
11263 
11264   ins_pipe(ialu_reg_reg);
11265 %}
11266 
11267 // This pattern is automatically generated from aarch64_ad.m4
11268 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11269 instruct XorI_reg_not_reg(iRegINoSp dst,
11270                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11271   match(Set dst (XorI m1 (XorI src2 src1)));
11272   ins_cost(INSN_COST);
11273   format %{ "eonw  $dst, $src1, $src2" %}
11274 
11275   ins_encode %{
11276     __ eonw(as_Register($dst$$reg),
11277               as_Register($src1$$reg),
11278               as_Register($src2$$reg),
11279               Assembler::LSL, 0);
11280   %}
11281 
11282   ins_pipe(ialu_reg_reg);
11283 %}
11284 
11285 // This pattern is automatically generated from aarch64_ad.m4
11286 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11287 instruct XorL_reg_not_reg(iRegLNoSp dst,
11288                          iRegL src1, iRegL src2, immL_M1 m1) %{
11289   match(Set dst (XorL m1 (XorL src2 src1)));
11290   ins_cost(INSN_COST);
11291   format %{ "eon  $dst, $src1, $src2" %}
11292 
11293   ins_encode %{
11294     __ eon(as_Register($dst$$reg),
11295               as_Register($src1$$reg),
11296               as_Register($src2$$reg),
11297               Assembler::LSL, 0);
11298   %}
11299 
11300   ins_pipe(ialu_reg_reg);
11301 %}
11302 
11303 // This pattern is automatically generated from aarch64_ad.m4
11304 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11305 // val & (-1 ^ (val >>> shift)) ==> bicw
11306 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11307                          iRegIorL2I src1, iRegIorL2I src2,
11308                          immI src3, immI_M1 src4) %{
11309   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11310   ins_cost(1.9 * INSN_COST);
11311   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11312 
11313   ins_encode %{
11314     __ bicw(as_Register($dst$$reg),
11315               as_Register($src1$$reg),
11316               as_Register($src2$$reg),
11317               Assembler::LSR,
11318               $src3$$constant & 0x1f);
11319   %}
11320 
11321   ins_pipe(ialu_reg_reg_shift);
11322 %}
11323 
11324 // This pattern is automatically generated from aarch64_ad.m4
11325 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11326 // val & (-1 ^ (val >>> shift)) ==> bic
11327 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11328                          iRegL src1, iRegL src2,
11329                          immI src3, immL_M1 src4) %{
11330   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11331   ins_cost(1.9 * INSN_COST);
11332   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11333 
11334   ins_encode %{
11335     __ bic(as_Register($dst$$reg),
11336               as_Register($src1$$reg),
11337               as_Register($src2$$reg),
11338               Assembler::LSR,
11339               $src3$$constant & 0x3f);
11340   %}
11341 
11342   ins_pipe(ialu_reg_reg_shift);
11343 %}
11344 
11345 // This pattern is automatically generated from aarch64_ad.m4
11346 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11347 // val & (-1 ^ (val >> shift)) ==> bicw
11348 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11349                          iRegIorL2I src1, iRegIorL2I src2,
11350                          immI src3, immI_M1 src4) %{
11351   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11352   ins_cost(1.9 * INSN_COST);
11353   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11354 
11355   ins_encode %{
11356     __ bicw(as_Register($dst$$reg),
11357               as_Register($src1$$reg),
11358               as_Register($src2$$reg),
11359               Assembler::ASR,
11360               $src3$$constant & 0x1f);
11361   %}
11362 
11363   ins_pipe(ialu_reg_reg_shift);
11364 %}
11365 
11366 // This pattern is automatically generated from aarch64_ad.m4
11367 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11368 // val & (-1 ^ (val >> shift)) ==> bic
11369 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11370                          iRegL src1, iRegL src2,
11371                          immI src3, immL_M1 src4) %{
11372   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11373   ins_cost(1.9 * INSN_COST);
11374   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11375 
11376   ins_encode %{
11377     __ bic(as_Register($dst$$reg),
11378               as_Register($src1$$reg),
11379               as_Register($src2$$reg),
11380               Assembler::ASR,
11381               $src3$$constant & 0x3f);
11382   %}
11383 
11384   ins_pipe(ialu_reg_reg_shift);
11385 %}
11386 
11387 // This pattern is automatically generated from aarch64_ad.m4
11388 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11389 // val & (-1 ^ (val ror shift)) ==> bicw
11390 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11391                          iRegIorL2I src1, iRegIorL2I src2,
11392                          immI src3, immI_M1 src4) %{
11393   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11394   ins_cost(1.9 * INSN_COST);
11395   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11396 
11397   ins_encode %{
11398     __ bicw(as_Register($dst$$reg),
11399               as_Register($src1$$reg),
11400               as_Register($src2$$reg),
11401               Assembler::ROR,
11402               $src3$$constant & 0x1f);
11403   %}
11404 
11405   ins_pipe(ialu_reg_reg_shift);
11406 %}
11407 
11408 // This pattern is automatically generated from aarch64_ad.m4
11409 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11410 // val & (-1 ^ (val ror shift)) ==> bic
11411 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11412                          iRegL src1, iRegL src2,
11413                          immI src3, immL_M1 src4) %{
11414   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11415   ins_cost(1.9 * INSN_COST);
11416   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11417 
11418   ins_encode %{
11419     __ bic(as_Register($dst$$reg),
11420               as_Register($src1$$reg),
11421               as_Register($src2$$reg),
11422               Assembler::ROR,
11423               $src3$$constant & 0x3f);
11424   %}
11425 
11426   ins_pipe(ialu_reg_reg_shift);
11427 %}
11428 
11429 // This pattern is automatically generated from aarch64_ad.m4
11430 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11431 // val & (-1 ^ (val << shift)) ==> bicw
11432 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11433                          iRegIorL2I src1, iRegIorL2I src2,
11434                          immI src3, immI_M1 src4) %{
11435   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11436   ins_cost(1.9 * INSN_COST);
11437   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11438 
11439   ins_encode %{
11440     __ bicw(as_Register($dst$$reg),
11441               as_Register($src1$$reg),
11442               as_Register($src2$$reg),
11443               Assembler::LSL,
11444               $src3$$constant & 0x1f);
11445   %}
11446 
11447   ins_pipe(ialu_reg_reg_shift);
11448 %}
11449 
11450 // This pattern is automatically generated from aarch64_ad.m4
11451 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11452 // val & (-1 ^ (val << shift)) ==> bic
11453 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11454                          iRegL src1, iRegL src2,
11455                          immI src3, immL_M1 src4) %{
11456   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11457   ins_cost(1.9 * INSN_COST);
11458   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11459 
11460   ins_encode %{
11461     __ bic(as_Register($dst$$reg),
11462               as_Register($src1$$reg),
11463               as_Register($src2$$reg),
11464               Assembler::LSL,
11465               $src3$$constant & 0x3f);
11466   %}
11467 
11468   ins_pipe(ialu_reg_reg_shift);
11469 %}
11470 
11471 // This pattern is automatically generated from aarch64_ad.m4
11472 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11473 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11474 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11475                          iRegIorL2I src1, iRegIorL2I src2,
11476                          immI src3, immI_M1 src4) %{
11477   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11478   ins_cost(1.9 * INSN_COST);
11479   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11480 
11481   ins_encode %{
11482     __ eonw(as_Register($dst$$reg),
11483               as_Register($src1$$reg),
11484               as_Register($src2$$reg),
11485               Assembler::LSR,
11486               $src3$$constant & 0x1f);
11487   %}
11488 
11489   ins_pipe(ialu_reg_reg_shift);
11490 %}
11491 
11492 // This pattern is automatically generated from aarch64_ad.m4
11493 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11494 // val ^ (-1 ^ (val >>> shift)) ==> eon
11495 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11496                          iRegL src1, iRegL src2,
11497                          immI src3, immL_M1 src4) %{
11498   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11499   ins_cost(1.9 * INSN_COST);
11500   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11501 
11502   ins_encode %{
11503     __ eon(as_Register($dst$$reg),
11504               as_Register($src1$$reg),
11505               as_Register($src2$$reg),
11506               Assembler::LSR,
11507               $src3$$constant & 0x3f);
11508   %}
11509 
11510   ins_pipe(ialu_reg_reg_shift);
11511 %}
11512 
11513 // This pattern is automatically generated from aarch64_ad.m4
11514 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11515 // val ^ (-1 ^ (val >> shift)) ==> eonw
11516 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11517                          iRegIorL2I src1, iRegIorL2I src2,
11518                          immI src3, immI_M1 src4) %{
11519   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11520   ins_cost(1.9 * INSN_COST);
11521   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11522 
11523   ins_encode %{
11524     __ eonw(as_Register($dst$$reg),
11525               as_Register($src1$$reg),
11526               as_Register($src2$$reg),
11527               Assembler::ASR,
11528               $src3$$constant & 0x1f);
11529   %}
11530 
11531   ins_pipe(ialu_reg_reg_shift);
11532 %}
11533 
11534 // This pattern is automatically generated from aarch64_ad.m4
11535 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11536 // val ^ (-1 ^ (val >> shift)) ==> eon
11537 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11538                          iRegL src1, iRegL src2,
11539                          immI src3, immL_M1 src4) %{
11540   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11541   ins_cost(1.9 * INSN_COST);
11542   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11543 
11544   ins_encode %{
11545     __ eon(as_Register($dst$$reg),
11546               as_Register($src1$$reg),
11547               as_Register($src2$$reg),
11548               Assembler::ASR,
11549               $src3$$constant & 0x3f);
11550   %}
11551 
11552   ins_pipe(ialu_reg_reg_shift);
11553 %}
11554 
11555 // This pattern is automatically generated from aarch64_ad.m4
11556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11557 // val ^ (-1 ^ (val ror shift)) ==> eonw
11558 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11559                          iRegIorL2I src1, iRegIorL2I src2,
11560                          immI src3, immI_M1 src4) %{
11561   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11562   ins_cost(1.9 * INSN_COST);
11563   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11564 
11565   ins_encode %{
11566     __ eonw(as_Register($dst$$reg),
11567               as_Register($src1$$reg),
11568               as_Register($src2$$reg),
11569               Assembler::ROR,
11570               $src3$$constant & 0x1f);
11571   %}
11572 
11573   ins_pipe(ialu_reg_reg_shift);
11574 %}
11575 
11576 // This pattern is automatically generated from aarch64_ad.m4
11577 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11578 // val ^ (-1 ^ (val ror shift)) ==> eon
11579 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11580                          iRegL src1, iRegL src2,
11581                          immI src3, immL_M1 src4) %{
11582   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11583   ins_cost(1.9 * INSN_COST);
11584   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11585 
11586   ins_encode %{
11587     __ eon(as_Register($dst$$reg),
11588               as_Register($src1$$reg),
11589               as_Register($src2$$reg),
11590               Assembler::ROR,
11591               $src3$$constant & 0x3f);
11592   %}
11593 
11594   ins_pipe(ialu_reg_reg_shift);
11595 %}
11596 
11597 // This pattern is automatically generated from aarch64_ad.m4
11598 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11599 // val ^ (-1 ^ (val << shift)) ==> eonw
11600 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11601                          iRegIorL2I src1, iRegIorL2I src2,
11602                          immI src3, immI_M1 src4) %{
11603   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11604   ins_cost(1.9 * INSN_COST);
11605   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11606 
11607   ins_encode %{
11608     __ eonw(as_Register($dst$$reg),
11609               as_Register($src1$$reg),
11610               as_Register($src2$$reg),
11611               Assembler::LSL,
11612               $src3$$constant & 0x1f);
11613   %}
11614 
11615   ins_pipe(ialu_reg_reg_shift);
11616 %}
11617 
11618 // This pattern is automatically generated from aarch64_ad.m4
11619 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11620 // val ^ (-1 ^ (val << shift)) ==> eon
11621 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11622                          iRegL src1, iRegL src2,
11623                          immI src3, immL_M1 src4) %{
11624   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11625   ins_cost(1.9 * INSN_COST);
11626   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11627 
11628   ins_encode %{
11629     __ eon(as_Register($dst$$reg),
11630               as_Register($src1$$reg),
11631               as_Register($src2$$reg),
11632               Assembler::LSL,
11633               $src3$$constant & 0x3f);
11634   %}
11635 
11636   ins_pipe(ialu_reg_reg_shift);
11637 %}
11638 
11639 // This pattern is automatically generated from aarch64_ad.m4
11640 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11641 // val | (-1 ^ (val >>> shift)) ==> ornw
11642 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11643                          iRegIorL2I src1, iRegIorL2I src2,
11644                          immI src3, immI_M1 src4) %{
11645   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11646   ins_cost(1.9 * INSN_COST);
11647   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11648 
11649   ins_encode %{
11650     __ ornw(as_Register($dst$$reg),
11651               as_Register($src1$$reg),
11652               as_Register($src2$$reg),
11653               Assembler::LSR,
11654               $src3$$constant & 0x1f);
11655   %}
11656 
11657   ins_pipe(ialu_reg_reg_shift);
11658 %}
11659 
11660 // This pattern is automatically generated from aarch64_ad.m4
11661 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11662 // val | (-1 ^ (val >>> shift)) ==> orn
11663 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11664                          iRegL src1, iRegL src2,
11665                          immI src3, immL_M1 src4) %{
11666   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11667   ins_cost(1.9 * INSN_COST);
11668   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11669 
11670   ins_encode %{
11671     __ orn(as_Register($dst$$reg),
11672               as_Register($src1$$reg),
11673               as_Register($src2$$reg),
11674               Assembler::LSR,
11675               $src3$$constant & 0x3f);
11676   %}
11677 
11678   ins_pipe(ialu_reg_reg_shift);
11679 %}
11680 
11681 // This pattern is automatically generated from aarch64_ad.m4
11682 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11683 // val | (-1 ^ (val >> shift)) ==> ornw
11684 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11685                          iRegIorL2I src1, iRegIorL2I src2,
11686                          immI src3, immI_M1 src4) %{
11687   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11688   ins_cost(1.9 * INSN_COST);
11689   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11690 
11691   ins_encode %{
11692     __ ornw(as_Register($dst$$reg),
11693               as_Register($src1$$reg),
11694               as_Register($src2$$reg),
11695               Assembler::ASR,
11696               $src3$$constant & 0x1f);
11697   %}
11698 
11699   ins_pipe(ialu_reg_reg_shift);
11700 %}
11701 
11702 // This pattern is automatically generated from aarch64_ad.m4
11703 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11704 // val | (-1 ^ (val >> shift)) ==> orn
11705 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11706                          iRegL src1, iRegL src2,
11707                          immI src3, immL_M1 src4) %{
11708   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11709   ins_cost(1.9 * INSN_COST);
11710   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11711 
11712   ins_encode %{
11713     __ orn(as_Register($dst$$reg),
11714               as_Register($src1$$reg),
11715               as_Register($src2$$reg),
11716               Assembler::ASR,
11717               $src3$$constant & 0x3f);
11718   %}
11719 
11720   ins_pipe(ialu_reg_reg_shift);
11721 %}
11722 
11723 // This pattern is automatically generated from aarch64_ad.m4
11724 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11725 // val | (-1 ^ (val ror shift)) ==> ornw
11726 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11727                          iRegIorL2I src1, iRegIorL2I src2,
11728                          immI src3, immI_M1 src4) %{
11729   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11730   ins_cost(1.9 * INSN_COST);
11731   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11732 
11733   ins_encode %{
11734     __ ornw(as_Register($dst$$reg),
11735               as_Register($src1$$reg),
11736               as_Register($src2$$reg),
11737               Assembler::ROR,
11738               $src3$$constant & 0x1f);
11739   %}
11740 
11741   ins_pipe(ialu_reg_reg_shift);
11742 %}
11743 
11744 // This pattern is automatically generated from aarch64_ad.m4
11745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11746 // val | (-1 ^ (val ror shift)) ==> orn
11747 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11748                          iRegL src1, iRegL src2,
11749                          immI src3, immL_M1 src4) %{
11750   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11751   ins_cost(1.9 * INSN_COST);
11752   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11753 
11754   ins_encode %{
11755     __ orn(as_Register($dst$$reg),
11756               as_Register($src1$$reg),
11757               as_Register($src2$$reg),
11758               Assembler::ROR,
11759               $src3$$constant & 0x3f);
11760   %}
11761 
11762   ins_pipe(ialu_reg_reg_shift);
11763 %}
11764 
11765 // This pattern is automatically generated from aarch64_ad.m4
11766 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11767 // val | (-1 ^ (val << shift)) ==> ornw
11768 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11769                          iRegIorL2I src1, iRegIorL2I src2,
11770                          immI src3, immI_M1 src4) %{
11771   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11772   ins_cost(1.9 * INSN_COST);
11773   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11774 
11775   ins_encode %{
11776     __ ornw(as_Register($dst$$reg),
11777               as_Register($src1$$reg),
11778               as_Register($src2$$reg),
11779               Assembler::LSL,
11780               $src3$$constant & 0x1f);
11781   %}
11782 
11783   ins_pipe(ialu_reg_reg_shift);
11784 %}
11785 
11786 // This pattern is automatically generated from aarch64_ad.m4
11787 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11788 // val | (-1 ^ (val << shift)) ==> orn
11789 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11790                          iRegL src1, iRegL src2,
11791                          immI src3, immL_M1 src4) %{
11792   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11793   ins_cost(1.9 * INSN_COST);
11794   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11795 
11796   ins_encode %{
11797     __ orn(as_Register($dst$$reg),
11798               as_Register($src1$$reg),
11799               as_Register($src2$$reg),
11800               Assembler::LSL,
11801               $src3$$constant & 0x3f);
11802   %}
11803 
11804   ins_pipe(ialu_reg_reg_shift);
11805 %}
11806 
11807 // This pattern is automatically generated from aarch64_ad.m4
11808 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11809 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11810                          iRegIorL2I src1, iRegIorL2I src2,
11811                          immI src3) %{
11812   match(Set dst (AndI src1 (URShiftI src2 src3)));
11813 
11814   ins_cost(1.9 * INSN_COST);
11815   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
11816 
11817   ins_encode %{
11818     __ andw(as_Register($dst$$reg),
11819               as_Register($src1$$reg),
11820               as_Register($src2$$reg),
11821               Assembler::LSR,
11822               $src3$$constant & 0x1f);
11823   %}
11824 
11825   ins_pipe(ialu_reg_reg_shift);
11826 %}
11827 
11828 // This pattern is automatically generated from aarch64_ad.m4
11829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11830 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11831                          iRegL src1, iRegL src2,
11832                          immI src3) %{
11833   match(Set dst (AndL src1 (URShiftL src2 src3)));
11834 
11835   ins_cost(1.9 * INSN_COST);
11836   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
11837 
11838   ins_encode %{
11839     __ andr(as_Register($dst$$reg),
11840               as_Register($src1$$reg),
11841               as_Register($src2$$reg),
11842               Assembler::LSR,
11843               $src3$$constant & 0x3f);
11844   %}
11845 
11846   ins_pipe(ialu_reg_reg_shift);
11847 %}
11848 
11849 // This pattern is automatically generated from aarch64_ad.m4
11850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11851 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11852                          iRegIorL2I src1, iRegIorL2I src2,
11853                          immI src3) %{
11854   match(Set dst (AndI src1 (RShiftI src2 src3)));
11855 
11856   ins_cost(1.9 * INSN_COST);
11857   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
11858 
11859   ins_encode %{
11860     __ andw(as_Register($dst$$reg),
11861               as_Register($src1$$reg),
11862               as_Register($src2$$reg),
11863               Assembler::ASR,
11864               $src3$$constant & 0x1f);
11865   %}
11866 
11867   ins_pipe(ialu_reg_reg_shift);
11868 %}
11869 
11870 // This pattern is automatically generated from aarch64_ad.m4
11871 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11872 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11873                          iRegL src1, iRegL src2,
11874                          immI src3) %{
11875   match(Set dst (AndL src1 (RShiftL src2 src3)));
11876 
11877   ins_cost(1.9 * INSN_COST);
11878   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
11879 
11880   ins_encode %{
11881     __ andr(as_Register($dst$$reg),
11882               as_Register($src1$$reg),
11883               as_Register($src2$$reg),
11884               Assembler::ASR,
11885               $src3$$constant & 0x3f);
11886   %}
11887 
11888   ins_pipe(ialu_reg_reg_shift);
11889 %}
11890 
11891 // This pattern is automatically generated from aarch64_ad.m4
11892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11893 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11894                          iRegIorL2I src1, iRegIorL2I src2,
11895                          immI src3) %{
11896   match(Set dst (AndI src1 (LShiftI src2 src3)));
11897 
11898   ins_cost(1.9 * INSN_COST);
11899   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
11900 
11901   ins_encode %{
11902     __ andw(as_Register($dst$$reg),
11903               as_Register($src1$$reg),
11904               as_Register($src2$$reg),
11905               Assembler::LSL,
11906               $src3$$constant & 0x1f);
11907   %}
11908 
11909   ins_pipe(ialu_reg_reg_shift);
11910 %}
11911 
11912 // This pattern is automatically generated from aarch64_ad.m4
11913 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11914 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11915                          iRegL src1, iRegL src2,
11916                          immI src3) %{
11917   match(Set dst (AndL src1 (LShiftL src2 src3)));
11918 
11919   ins_cost(1.9 * INSN_COST);
11920   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
11921 
11922   ins_encode %{
11923     __ andr(as_Register($dst$$reg),
11924               as_Register($src1$$reg),
11925               as_Register($src2$$reg),
11926               Assembler::LSL,
11927               $src3$$constant & 0x3f);
11928   %}
11929 
11930   ins_pipe(ialu_reg_reg_shift);
11931 %}
11932 
11933 // This pattern is automatically generated from aarch64_ad.m4
11934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11935 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11936                          iRegIorL2I src1, iRegIorL2I src2,
11937                          immI src3) %{
11938   match(Set dst (AndI src1 (RotateRight src2 src3)));
11939 
11940   ins_cost(1.9 * INSN_COST);
11941   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
11942 
11943   ins_encode %{
11944     __ andw(as_Register($dst$$reg),
11945               as_Register($src1$$reg),
11946               as_Register($src2$$reg),
11947               Assembler::ROR,
11948               $src3$$constant & 0x1f);
11949   %}
11950 
11951   ins_pipe(ialu_reg_reg_shift);
11952 %}
11953 
11954 // This pattern is automatically generated from aarch64_ad.m4
11955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11956 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11957                          iRegL src1, iRegL src2,
11958                          immI src3) %{
11959   match(Set dst (AndL src1 (RotateRight src2 src3)));
11960 
11961   ins_cost(1.9 * INSN_COST);
11962   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
11963 
11964   ins_encode %{
11965     __ andr(as_Register($dst$$reg),
11966               as_Register($src1$$reg),
11967               as_Register($src2$$reg),
11968               Assembler::ROR,
11969               $src3$$constant & 0x3f);
11970   %}
11971 
11972   ins_pipe(ialu_reg_reg_shift);
11973 %}
11974 
11975 // This pattern is automatically generated from aarch64_ad.m4
11976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11977 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11978                          iRegIorL2I src1, iRegIorL2I src2,
11979                          immI src3) %{
11980   match(Set dst (XorI src1 (URShiftI src2 src3)));
11981 
11982   ins_cost(1.9 * INSN_COST);
11983   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
11984 
11985   ins_encode %{
11986     __ eorw(as_Register($dst$$reg),
11987               as_Register($src1$$reg),
11988               as_Register($src2$$reg),
11989               Assembler::LSR,
11990               $src3$$constant & 0x1f);
11991   %}
11992 
11993   ins_pipe(ialu_reg_reg_shift);
11994 %}
11995 
11996 // This pattern is automatically generated from aarch64_ad.m4
11997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11998 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11999                          iRegL src1, iRegL src2,
12000                          immI src3) %{
12001   match(Set dst (XorL src1 (URShiftL src2 src3)));
12002 
12003   ins_cost(1.9 * INSN_COST);
12004   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12005 
12006   ins_encode %{
12007     __ eor(as_Register($dst$$reg),
12008               as_Register($src1$$reg),
12009               as_Register($src2$$reg),
12010               Assembler::LSR,
12011               $src3$$constant & 0x3f);
12012   %}
12013 
12014   ins_pipe(ialu_reg_reg_shift);
12015 %}
12016 
12017 // This pattern is automatically generated from aarch64_ad.m4
12018 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12019 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12020                          iRegIorL2I src1, iRegIorL2I src2,
12021                          immI src3) %{
12022   match(Set dst (XorI src1 (RShiftI src2 src3)));
12023 
12024   ins_cost(1.9 * INSN_COST);
12025   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12026 
12027   ins_encode %{
12028     __ eorw(as_Register($dst$$reg),
12029               as_Register($src1$$reg),
12030               as_Register($src2$$reg),
12031               Assembler::ASR,
12032               $src3$$constant & 0x1f);
12033   %}
12034 
12035   ins_pipe(ialu_reg_reg_shift);
12036 %}
12037 
12038 // This pattern is automatically generated from aarch64_ad.m4
12039 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12040 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12041                          iRegL src1, iRegL src2,
12042                          immI src3) %{
12043   match(Set dst (XorL src1 (RShiftL src2 src3)));
12044 
12045   ins_cost(1.9 * INSN_COST);
12046   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12047 
12048   ins_encode %{
12049     __ eor(as_Register($dst$$reg),
12050               as_Register($src1$$reg),
12051               as_Register($src2$$reg),
12052               Assembler::ASR,
12053               $src3$$constant & 0x3f);
12054   %}
12055 
12056   ins_pipe(ialu_reg_reg_shift);
12057 %}
12058 
12059 // This pattern is automatically generated from aarch64_ad.m4
12060 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12061 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12062                          iRegIorL2I src1, iRegIorL2I src2,
12063                          immI src3) %{
12064   match(Set dst (XorI src1 (LShiftI src2 src3)));
12065 
12066   ins_cost(1.9 * INSN_COST);
12067   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12068 
12069   ins_encode %{
12070     __ eorw(as_Register($dst$$reg),
12071               as_Register($src1$$reg),
12072               as_Register($src2$$reg),
12073               Assembler::LSL,
12074               $src3$$constant & 0x1f);
12075   %}
12076 
12077   ins_pipe(ialu_reg_reg_shift);
12078 %}
12079 
12080 // This pattern is automatically generated from aarch64_ad.m4
12081 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12082 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12083                          iRegL src1, iRegL src2,
12084                          immI src3) %{
12085   match(Set dst (XorL src1 (LShiftL src2 src3)));
12086 
12087   ins_cost(1.9 * INSN_COST);
12088   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12089 
12090   ins_encode %{
12091     __ eor(as_Register($dst$$reg),
12092               as_Register($src1$$reg),
12093               as_Register($src2$$reg),
12094               Assembler::LSL,
12095               $src3$$constant & 0x3f);
12096   %}
12097 
12098   ins_pipe(ialu_reg_reg_shift);
12099 %}
12100 
12101 // This pattern is automatically generated from aarch64_ad.m4
12102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12103 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12104                          iRegIorL2I src1, iRegIorL2I src2,
12105                          immI src3) %{
12106   match(Set dst (XorI src1 (RotateRight src2 src3)));
12107 
12108   ins_cost(1.9 * INSN_COST);
12109   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12110 
12111   ins_encode %{
12112     __ eorw(as_Register($dst$$reg),
12113               as_Register($src1$$reg),
12114               as_Register($src2$$reg),
12115               Assembler::ROR,
12116               $src3$$constant & 0x1f);
12117   %}
12118 
12119   ins_pipe(ialu_reg_reg_shift);
12120 %}
12121 
12122 // This pattern is automatically generated from aarch64_ad.m4
12123 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12124 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12125                          iRegL src1, iRegL src2,
12126                          immI src3) %{
12127   match(Set dst (XorL src1 (RotateRight src2 src3)));
12128 
12129   ins_cost(1.9 * INSN_COST);
12130   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12131 
12132   ins_encode %{
12133     __ eor(as_Register($dst$$reg),
12134               as_Register($src1$$reg),
12135               as_Register($src2$$reg),
12136               Assembler::ROR,
12137               $src3$$constant & 0x3f);
12138   %}
12139 
12140   ins_pipe(ialu_reg_reg_shift);
12141 %}
12142 
12143 // This pattern is automatically generated from aarch64_ad.m4
12144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12145 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12146                          iRegIorL2I src1, iRegIorL2I src2,
12147                          immI src3) %{
12148   match(Set dst (OrI src1 (URShiftI src2 src3)));
12149 
12150   ins_cost(1.9 * INSN_COST);
12151   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12152 
12153   ins_encode %{
12154     __ orrw(as_Register($dst$$reg),
12155               as_Register($src1$$reg),
12156               as_Register($src2$$reg),
12157               Assembler::LSR,
12158               $src3$$constant & 0x1f);
12159   %}
12160 
12161   ins_pipe(ialu_reg_reg_shift);
12162 %}
12163 
12164 // This pattern is automatically generated from aarch64_ad.m4
12165 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12166 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12167                          iRegL src1, iRegL src2,
12168                          immI src3) %{
12169   match(Set dst (OrL src1 (URShiftL src2 src3)));
12170 
12171   ins_cost(1.9 * INSN_COST);
12172   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12173 
12174   ins_encode %{
12175     __ orr(as_Register($dst$$reg),
12176               as_Register($src1$$reg),
12177               as_Register($src2$$reg),
12178               Assembler::LSR,
12179               $src3$$constant & 0x3f);
12180   %}
12181 
12182   ins_pipe(ialu_reg_reg_shift);
12183 %}
12184 
12185 // This pattern is automatically generated from aarch64_ad.m4
12186 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12187 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12188                          iRegIorL2I src1, iRegIorL2I src2,
12189                          immI src3) %{
12190   match(Set dst (OrI src1 (RShiftI src2 src3)));
12191 
12192   ins_cost(1.9 * INSN_COST);
12193   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12194 
12195   ins_encode %{
12196     __ orrw(as_Register($dst$$reg),
12197               as_Register($src1$$reg),
12198               as_Register($src2$$reg),
12199               Assembler::ASR,
12200               $src3$$constant & 0x1f);
12201   %}
12202 
12203   ins_pipe(ialu_reg_reg_shift);
12204 %}
12205 
12206 // This pattern is automatically generated from aarch64_ad.m4
12207 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12208 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12209                          iRegL src1, iRegL src2,
12210                          immI src3) %{
12211   match(Set dst (OrL src1 (RShiftL src2 src3)));
12212 
12213   ins_cost(1.9 * INSN_COST);
12214   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12215 
12216   ins_encode %{
12217     __ orr(as_Register($dst$$reg),
12218               as_Register($src1$$reg),
12219               as_Register($src2$$reg),
12220               Assembler::ASR,
12221               $src3$$constant & 0x3f);
12222   %}
12223 
12224   ins_pipe(ialu_reg_reg_shift);
12225 %}
12226 
12227 // This pattern is automatically generated from aarch64_ad.m4
12228 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12229 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12230                          iRegIorL2I src1, iRegIorL2I src2,
12231                          immI src3) %{
12232   match(Set dst (OrI src1 (LShiftI src2 src3)));
12233 
12234   ins_cost(1.9 * INSN_COST);
12235   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12236 
12237   ins_encode %{
12238     __ orrw(as_Register($dst$$reg),
12239               as_Register($src1$$reg),
12240               as_Register($src2$$reg),
12241               Assembler::LSL,
12242               $src3$$constant & 0x1f);
12243   %}
12244 
12245   ins_pipe(ialu_reg_reg_shift);
12246 %}
12247 
12248 // This pattern is automatically generated from aarch64_ad.m4
12249 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12250 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12251                          iRegL src1, iRegL src2,
12252                          immI src3) %{
12253   match(Set dst (OrL src1 (LShiftL src2 src3)));
12254 
12255   ins_cost(1.9 * INSN_COST);
12256   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12257 
12258   ins_encode %{
12259     __ orr(as_Register($dst$$reg),
12260               as_Register($src1$$reg),
12261               as_Register($src2$$reg),
12262               Assembler::LSL,
12263               $src3$$constant & 0x3f);
12264   %}
12265 
12266   ins_pipe(ialu_reg_reg_shift);
12267 %}
12268 
12269 // This pattern is automatically generated from aarch64_ad.m4
12270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12271 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12272                          iRegIorL2I src1, iRegIorL2I src2,
12273                          immI src3) %{
12274   match(Set dst (OrI src1 (RotateRight src2 src3)));
12275 
12276   ins_cost(1.9 * INSN_COST);
12277   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12278 
12279   ins_encode %{
12280     __ orrw(as_Register($dst$$reg),
12281               as_Register($src1$$reg),
12282               as_Register($src2$$reg),
12283               Assembler::ROR,
12284               $src3$$constant & 0x1f);
12285   %}
12286 
12287   ins_pipe(ialu_reg_reg_shift);
12288 %}
12289 
12290 // This pattern is automatically generated from aarch64_ad.m4
12291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12292 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12293                          iRegL src1, iRegL src2,
12294                          immI src3) %{
12295   match(Set dst (OrL src1 (RotateRight src2 src3)));
12296 
12297   ins_cost(1.9 * INSN_COST);
12298   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12299 
12300   ins_encode %{
12301     __ orr(as_Register($dst$$reg),
12302               as_Register($src1$$reg),
12303               as_Register($src2$$reg),
12304               Assembler::ROR,
12305               $src3$$constant & 0x3f);
12306   %}
12307 
12308   ins_pipe(ialu_reg_reg_shift);
12309 %}
12310 
12311 // This pattern is automatically generated from aarch64_ad.m4
12312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12313 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12314                          iRegIorL2I src1, iRegIorL2I src2,
12315                          immI src3) %{
12316   match(Set dst (AddI src1 (URShiftI src2 src3)));
12317 
12318   ins_cost(1.9 * INSN_COST);
12319   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12320 
12321   ins_encode %{
12322     __ addw(as_Register($dst$$reg),
12323               as_Register($src1$$reg),
12324               as_Register($src2$$reg),
12325               Assembler::LSR,
12326               $src3$$constant & 0x1f);
12327   %}
12328 
12329   ins_pipe(ialu_reg_reg_shift);
12330 %}
12331 
12332 // This pattern is automatically generated from aarch64_ad.m4
12333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12334 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12335                          iRegL src1, iRegL src2,
12336                          immI src3) %{
12337   match(Set dst (AddL src1 (URShiftL src2 src3)));
12338 
12339   ins_cost(1.9 * INSN_COST);
12340   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12341 
12342   ins_encode %{
12343     __ add(as_Register($dst$$reg),
12344               as_Register($src1$$reg),
12345               as_Register($src2$$reg),
12346               Assembler::LSR,
12347               $src3$$constant & 0x3f);
12348   %}
12349 
12350   ins_pipe(ialu_reg_reg_shift);
12351 %}
12352 
12353 // This pattern is automatically generated from aarch64_ad.m4
12354 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12355 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12356                          iRegIorL2I src1, iRegIorL2I src2,
12357                          immI src3) %{
12358   match(Set dst (AddI src1 (RShiftI src2 src3)));
12359 
12360   ins_cost(1.9 * INSN_COST);
12361   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12362 
12363   ins_encode %{
12364     __ addw(as_Register($dst$$reg),
12365               as_Register($src1$$reg),
12366               as_Register($src2$$reg),
12367               Assembler::ASR,
12368               $src3$$constant & 0x1f);
12369   %}
12370 
12371   ins_pipe(ialu_reg_reg_shift);
12372 %}
12373 
12374 // This pattern is automatically generated from aarch64_ad.m4
12375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12376 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12377                          iRegL src1, iRegL src2,
12378                          immI src3) %{
12379   match(Set dst (AddL src1 (RShiftL src2 src3)));
12380 
12381   ins_cost(1.9 * INSN_COST);
12382   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12383 
12384   ins_encode %{
12385     __ add(as_Register($dst$$reg),
12386               as_Register($src1$$reg),
12387               as_Register($src2$$reg),
12388               Assembler::ASR,
12389               $src3$$constant & 0x3f);
12390   %}
12391 
12392   ins_pipe(ialu_reg_reg_shift);
12393 %}
12394 
12395 // This pattern is automatically generated from aarch64_ad.m4
12396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12397 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12398                          iRegIorL2I src1, iRegIorL2I src2,
12399                          immI src3) %{
12400   match(Set dst (AddI src1 (LShiftI src2 src3)));
12401 
12402   ins_cost(1.9 * INSN_COST);
12403   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12404 
12405   ins_encode %{
12406     __ addw(as_Register($dst$$reg),
12407               as_Register($src1$$reg),
12408               as_Register($src2$$reg),
12409               Assembler::LSL,
12410               $src3$$constant & 0x1f);
12411   %}
12412 
12413   ins_pipe(ialu_reg_reg_shift);
12414 %}
12415 
12416 // This pattern is automatically generated from aarch64_ad.m4
12417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12418 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12419                          iRegL src1, iRegL src2,
12420                          immI src3) %{
12421   match(Set dst (AddL src1 (LShiftL src2 src3)));
12422 
12423   ins_cost(1.9 * INSN_COST);
12424   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12425 
12426   ins_encode %{
12427     __ add(as_Register($dst$$reg),
12428               as_Register($src1$$reg),
12429               as_Register($src2$$reg),
12430               Assembler::LSL,
12431               $src3$$constant & 0x3f);
12432   %}
12433 
12434   ins_pipe(ialu_reg_reg_shift);
12435 %}
12436 
12437 // This pattern is automatically generated from aarch64_ad.m4
12438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12439 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12440                          iRegIorL2I src1, iRegIorL2I src2,
12441                          immI src3) %{
12442   match(Set dst (SubI src1 (URShiftI src2 src3)));
12443 
12444   ins_cost(1.9 * INSN_COST);
12445   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12446 
12447   ins_encode %{
12448     __ subw(as_Register($dst$$reg),
12449               as_Register($src1$$reg),
12450               as_Register($src2$$reg),
12451               Assembler::LSR,
12452               $src3$$constant & 0x1f);
12453   %}
12454 
12455   ins_pipe(ialu_reg_reg_shift);
12456 %}
12457 
12458 // This pattern is automatically generated from aarch64_ad.m4
12459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12460 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12461                          iRegL src1, iRegL src2,
12462                          immI src3) %{
12463   match(Set dst (SubL src1 (URShiftL src2 src3)));
12464 
12465   ins_cost(1.9 * INSN_COST);
12466   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12467 
12468   ins_encode %{
12469     __ sub(as_Register($dst$$reg),
12470               as_Register($src1$$reg),
12471               as_Register($src2$$reg),
12472               Assembler::LSR,
12473               $src3$$constant & 0x3f);
12474   %}
12475 
12476   ins_pipe(ialu_reg_reg_shift);
12477 %}
12478 
12479 // This pattern is automatically generated from aarch64_ad.m4
12480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12481 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12482                          iRegIorL2I src1, iRegIorL2I src2,
12483                          immI src3) %{
12484   match(Set dst (SubI src1 (RShiftI src2 src3)));
12485 
12486   ins_cost(1.9 * INSN_COST);
12487   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12488 
12489   ins_encode %{
12490     __ subw(as_Register($dst$$reg),
12491               as_Register($src1$$reg),
12492               as_Register($src2$$reg),
12493               Assembler::ASR,
12494               $src3$$constant & 0x1f);
12495   %}
12496 
12497   ins_pipe(ialu_reg_reg_shift);
12498 %}
12499 
12500 // This pattern is automatically generated from aarch64_ad.m4
12501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12502 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12503                          iRegL src1, iRegL src2,
12504                          immI src3) %{
12505   match(Set dst (SubL src1 (RShiftL src2 src3)));
12506 
12507   ins_cost(1.9 * INSN_COST);
12508   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12509 
12510   ins_encode %{
12511     __ sub(as_Register($dst$$reg),
12512               as_Register($src1$$reg),
12513               as_Register($src2$$reg),
12514               Assembler::ASR,
12515               $src3$$constant & 0x3f);
12516   %}
12517 
12518   ins_pipe(ialu_reg_reg_shift);
12519 %}
12520 
12521 // This pattern is automatically generated from aarch64_ad.m4
12522 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12523 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12524                          iRegIorL2I src1, iRegIorL2I src2,
12525                          immI src3) %{
12526   match(Set dst (SubI src1 (LShiftI src2 src3)));
12527 
12528   ins_cost(1.9 * INSN_COST);
12529   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12530 
12531   ins_encode %{
12532     __ subw(as_Register($dst$$reg),
12533               as_Register($src1$$reg),
12534               as_Register($src2$$reg),
12535               Assembler::LSL,
12536               $src3$$constant & 0x1f);
12537   %}
12538 
12539   ins_pipe(ialu_reg_reg_shift);
12540 %}
12541 
12542 // This pattern is automatically generated from aarch64_ad.m4
12543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12544 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12545                          iRegL src1, iRegL src2,
12546                          immI src3) %{
12547   match(Set dst (SubL src1 (LShiftL src2 src3)));
12548 
12549   ins_cost(1.9 * INSN_COST);
12550   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12551 
12552   ins_encode %{
12553     __ sub(as_Register($dst$$reg),
12554               as_Register($src1$$reg),
12555               as_Register($src2$$reg),
12556               Assembler::LSL,
12557               $src3$$constant & 0x3f);
12558   %}
12559 
12560   ins_pipe(ialu_reg_reg_shift);
12561 %}
12562 
12563 // This pattern is automatically generated from aarch64_ad.m4
12564 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12565 
12566 // Shift Left followed by Shift Right.
12567 // This idiom is used by the compiler for the i2b bytecode etc.
12568 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12569 %{
12570   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12571   ins_cost(INSN_COST * 2);
12572   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12573   ins_encode %{
12574     int lshift = $lshift_count$$constant & 63;
12575     int rshift = $rshift_count$$constant & 63;
12576     int s = 63 - lshift;
12577     int r = (rshift - lshift) & 63;
12578     __ sbfm(as_Register($dst$$reg),
12579             as_Register($src$$reg),
12580             r, s);
12581   %}
12582 
12583   ins_pipe(ialu_reg_shift);
12584 %}
12585 
12586 // This pattern is automatically generated from aarch64_ad.m4
12587 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12588 
12589 // Shift Left followed by Shift Right.
12590 // This idiom is used by the compiler for the i2b bytecode etc.
12591 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12592 %{
12593   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12594   ins_cost(INSN_COST * 2);
12595   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12596   ins_encode %{
12597     int lshift = $lshift_count$$constant & 31;
12598     int rshift = $rshift_count$$constant & 31;
12599     int s = 31 - lshift;
12600     int r = (rshift - lshift) & 31;
12601     __ sbfmw(as_Register($dst$$reg),
12602             as_Register($src$$reg),
12603             r, s);
12604   %}
12605 
12606   ins_pipe(ialu_reg_shift);
12607 %}
12608 
12609 // This pattern is automatically generated from aarch64_ad.m4
12610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12611 
12612 // Shift Left followed by Shift Right.
12613 // This idiom is used by the compiler for the i2b bytecode etc.
12614 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12615 %{
12616   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12617   ins_cost(INSN_COST * 2);
12618   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12619   ins_encode %{
12620     int lshift = $lshift_count$$constant & 63;
12621     int rshift = $rshift_count$$constant & 63;
12622     int s = 63 - lshift;
12623     int r = (rshift - lshift) & 63;
12624     __ ubfm(as_Register($dst$$reg),
12625             as_Register($src$$reg),
12626             r, s);
12627   %}
12628 
12629   ins_pipe(ialu_reg_shift);
12630 %}
12631 
12632 // This pattern is automatically generated from aarch64_ad.m4
12633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12634 
12635 // Shift Left followed by Shift Right.
12636 // This idiom is used by the compiler for the i2b bytecode etc.
12637 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12638 %{
12639   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12640   ins_cost(INSN_COST * 2);
12641   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12642   ins_encode %{
12643     int lshift = $lshift_count$$constant & 31;
12644     int rshift = $rshift_count$$constant & 31;
12645     int s = 31 - lshift;
12646     int r = (rshift - lshift) & 31;
12647     __ ubfmw(as_Register($dst$$reg),
12648             as_Register($src$$reg),
12649             r, s);
12650   %}
12651 
12652   ins_pipe(ialu_reg_shift);
12653 %}
12654 
12655 // Bitfield extract with shift & mask
12656 
12657 // This pattern is automatically generated from aarch64_ad.m4
12658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12659 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12660 %{
12661   match(Set dst (AndI (URShiftI src rshift) mask));
12662   // Make sure we are not going to exceed what ubfxw can do.
12663   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12664 
12665   ins_cost(INSN_COST);
12666   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12667   ins_encode %{
12668     int rshift = $rshift$$constant & 31;
12669     intptr_t mask = $mask$$constant;
12670     int width = exact_log2(mask+1);
12671     __ ubfxw(as_Register($dst$$reg),
12672             as_Register($src$$reg), rshift, width);
12673   %}
12674   ins_pipe(ialu_reg_shift);
12675 %}
12676 
12677 // This pattern is automatically generated from aarch64_ad.m4
12678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12679 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12680 %{
12681   match(Set dst (AndL (URShiftL src rshift) mask));
12682   // Make sure we are not going to exceed what ubfx can do.
12683   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12684 
12685   ins_cost(INSN_COST);
12686   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12687   ins_encode %{
12688     int rshift = $rshift$$constant & 63;
12689     intptr_t mask = $mask$$constant;
12690     int width = exact_log2_long(mask+1);
12691     __ ubfx(as_Register($dst$$reg),
12692             as_Register($src$$reg), rshift, width);
12693   %}
12694   ins_pipe(ialu_reg_shift);
12695 %}
12696 
12697 
12698 // This pattern is automatically generated from aarch64_ad.m4
12699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12700 
12701 // We can use ubfx when extending an And with a mask when we know mask
12702 // is positive.  We know that because immI_bitmask guarantees it.
12703 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12704 %{
12705   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12706   // Make sure we are not going to exceed what ubfxw can do.
12707   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12708 
12709   ins_cost(INSN_COST * 2);
12710   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12711   ins_encode %{
12712     int rshift = $rshift$$constant & 31;
12713     intptr_t mask = $mask$$constant;
12714     int width = exact_log2(mask+1);
12715     __ ubfx(as_Register($dst$$reg),
12716             as_Register($src$$reg), rshift, width);
12717   %}
12718   ins_pipe(ialu_reg_shift);
12719 %}
12720 
12721 
12722 // This pattern is automatically generated from aarch64_ad.m4
12723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12724 
12725 // We can use ubfiz when masking by a positive number and then left shifting the result.
12726 // We know that the mask is positive because immI_bitmask guarantees it.
12727 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12728 %{
12729   match(Set dst (LShiftI (AndI src mask) lshift));
12730   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12731 
12732   ins_cost(INSN_COST);
12733   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12734   ins_encode %{
12735     int lshift = $lshift$$constant & 31;
12736     intptr_t mask = $mask$$constant;
12737     int width = exact_log2(mask+1);
12738     __ ubfizw(as_Register($dst$$reg),
12739           as_Register($src$$reg), lshift, width);
12740   %}
12741   ins_pipe(ialu_reg_shift);
12742 %}
12743 
12744 // This pattern is automatically generated from aarch64_ad.m4
12745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12746 
12747 // We can use ubfiz when masking by a positive number and then left shifting the result.
12748 // We know that the mask is positive because immL_bitmask guarantees it.
12749 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12750 %{
12751   match(Set dst (LShiftL (AndL src mask) lshift));
12752   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12753 
12754   ins_cost(INSN_COST);
12755   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12756   ins_encode %{
12757     int lshift = $lshift$$constant & 63;
12758     intptr_t mask = $mask$$constant;
12759     int width = exact_log2_long(mask+1);
12760     __ ubfiz(as_Register($dst$$reg),
12761           as_Register($src$$reg), lshift, width);
12762   %}
12763   ins_pipe(ialu_reg_shift);
12764 %}
12765 
12766 // This pattern is automatically generated from aarch64_ad.m4
12767 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12768 
12769 // We can use ubfiz when masking by a positive number and then left shifting the result.
12770 // We know that the mask is positive because immI_bitmask guarantees it.
12771 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12772 %{
12773   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12774   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12775 
12776   ins_cost(INSN_COST);
12777   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12778   ins_encode %{
12779     int lshift = $lshift$$constant & 31;
12780     intptr_t mask = $mask$$constant;
12781     int width = exact_log2(mask+1);
12782     __ ubfizw(as_Register($dst$$reg),
12783           as_Register($src$$reg), lshift, width);
12784   %}
12785   ins_pipe(ialu_reg_shift);
12786 %}
12787 
12788 // This pattern is automatically generated from aarch64_ad.m4
12789 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12790 
12791 // We can use ubfiz when masking by a positive number and then left shifting the result.
12792 // We know that the mask is positive because immL_bitmask guarantees it.
12793 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12794 %{
12795   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12796   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12797 
12798   ins_cost(INSN_COST);
12799   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12800   ins_encode %{
12801     int lshift = $lshift$$constant & 63;
12802     intptr_t mask = $mask$$constant;
12803     int width = exact_log2_long(mask+1);
12804     __ ubfiz(as_Register($dst$$reg),
12805           as_Register($src$$reg), lshift, width);
12806   %}
12807   ins_pipe(ialu_reg_shift);
12808 %}
12809 
12810 
12811 // This pattern is automatically generated from aarch64_ad.m4
12812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12813 
12814 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12815 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12816 %{
12817   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12818   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12819 
12820   ins_cost(INSN_COST);
12821   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12822   ins_encode %{
12823     int lshift = $lshift$$constant & 63;
12824     intptr_t mask = $mask$$constant;
12825     int width = exact_log2(mask+1);
12826     __ ubfiz(as_Register($dst$$reg),
12827              as_Register($src$$reg), lshift, width);
12828   %}
12829   ins_pipe(ialu_reg_shift);
12830 %}
12831 
12832 // This pattern is automatically generated from aarch64_ad.m4
12833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12834 
12835 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12836 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12837 %{
12838   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12839   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12840 
12841   ins_cost(INSN_COST);
12842   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12843   ins_encode %{
12844     int lshift = $lshift$$constant & 31;
12845     intptr_t mask = $mask$$constant;
12846     int width = exact_log2(mask+1);
12847     __ ubfiz(as_Register($dst$$reg),
12848              as_Register($src$$reg), lshift, width);
12849   %}
12850   ins_pipe(ialu_reg_shift);
12851 %}
12852 
12853 // This pattern is automatically generated from aarch64_ad.m4
12854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12855 
12856 // Can skip int2long conversions after AND with small bitmask
12857 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12858 %{
12859   match(Set dst (ConvI2L (AndI src msk)));
12860   ins_cost(INSN_COST);
12861   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12862   ins_encode %{
12863     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12864   %}
12865   ins_pipe(ialu_reg_shift);
12866 %}
12867 
12868 
12869 // Rotations
12870 // This pattern is automatically generated from aarch64_ad.m4
12871 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12872 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12873 %{
12874   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12875   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12876 
12877   ins_cost(INSN_COST);
12878   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12879 
12880   ins_encode %{
12881     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12882             $rshift$$constant & 63);
12883   %}
12884   ins_pipe(ialu_reg_reg_extr);
12885 %}
12886 
12887 
12888 // This pattern is automatically generated from aarch64_ad.m4
12889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12890 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12891 %{
12892   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12893   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12894 
12895   ins_cost(INSN_COST);
12896   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12897 
12898   ins_encode %{
12899     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12900             $rshift$$constant & 31);
12901   %}
12902   ins_pipe(ialu_reg_reg_extr);
12903 %}
12904 
12905 
12906 // This pattern is automatically generated from aarch64_ad.m4
12907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12908 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12909 %{
12910   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12911   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12912 
12913   ins_cost(INSN_COST);
12914   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12915 
12916   ins_encode %{
12917     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12918             $rshift$$constant & 63);
12919   %}
12920   ins_pipe(ialu_reg_reg_extr);
12921 %}
12922 
12923 
12924 // This pattern is automatically generated from aarch64_ad.m4
12925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12926 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12927 %{
12928   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12929   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12930 
12931   ins_cost(INSN_COST);
12932   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12933 
12934   ins_encode %{
12935     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12936             $rshift$$constant & 31);
12937   %}
12938   ins_pipe(ialu_reg_reg_extr);
12939 %}
12940 
12941 
12942 // This pattern is automatically generated from aarch64_ad.m4
12943 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12944 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12945 %{
12946   match(Set dst (RotateRight src shift));
12947 
12948   ins_cost(INSN_COST);
12949   format %{ "ror    $dst, $src, $shift" %}
12950 
12951   ins_encode %{
12952      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12953                $shift$$constant & 0x1f);
12954   %}
12955   ins_pipe(ialu_reg_reg_vshift);
12956 %}
12957 
12958 // This pattern is automatically generated from aarch64_ad.m4
12959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12960 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12961 %{
12962   match(Set dst (RotateRight src shift));
12963 
12964   ins_cost(INSN_COST);
12965   format %{ "ror    $dst, $src, $shift" %}
12966 
12967   ins_encode %{
12968      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12969                $shift$$constant & 0x3f);
12970   %}
12971   ins_pipe(ialu_reg_reg_vshift);
12972 %}
12973 
12974 // This pattern is automatically generated from aarch64_ad.m4
12975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12976 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12977 %{
12978   match(Set dst (RotateRight src shift));
12979 
12980   ins_cost(INSN_COST);
12981   format %{ "ror    $dst, $src, $shift" %}
12982 
12983   ins_encode %{
12984      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12985   %}
12986   ins_pipe(ialu_reg_reg_vshift);
12987 %}
12988 
12989 // This pattern is automatically generated from aarch64_ad.m4
12990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12991 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12992 %{
12993   match(Set dst (RotateRight src shift));
12994 
12995   ins_cost(INSN_COST);
12996   format %{ "ror    $dst, $src, $shift" %}
12997 
12998   ins_encode %{
12999      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13000   %}
13001   ins_pipe(ialu_reg_reg_vshift);
13002 %}
13003 
13004 // This pattern is automatically generated from aarch64_ad.m4
13005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13006 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13007 %{
13008   match(Set dst (RotateLeft src shift));
13009 
13010   ins_cost(INSN_COST);
13011   format %{ "rol    $dst, $src, $shift" %}
13012 
13013   ins_encode %{
13014      __ subw(rscratch1, zr, as_Register($shift$$reg));
13015      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13016   %}
13017   ins_pipe(ialu_reg_reg_vshift);
13018 %}
13019 
13020 // This pattern is automatically generated from aarch64_ad.m4
13021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13022 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13023 %{
13024   match(Set dst (RotateLeft src shift));
13025 
13026   ins_cost(INSN_COST);
13027   format %{ "rol    $dst, $src, $shift" %}
13028 
13029   ins_encode %{
13030      __ subw(rscratch1, zr, as_Register($shift$$reg));
13031      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13032   %}
13033   ins_pipe(ialu_reg_reg_vshift);
13034 %}
13035 
13036 
13037 // Add/subtract (extended)
13038 
13039 // This pattern is automatically generated from aarch64_ad.m4
13040 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13041 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13042 %{
13043   match(Set dst (AddL src1 (ConvI2L src2)));
13044   ins_cost(INSN_COST);
13045   format %{ "add  $dst, $src1, $src2, sxtw" %}
13046 
13047    ins_encode %{
13048      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13049             as_Register($src2$$reg), ext::sxtw);
13050    %}
13051   ins_pipe(ialu_reg_reg);
13052 %}
13053 
13054 // This pattern is automatically generated from aarch64_ad.m4
13055 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13056 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13057 %{
13058   match(Set dst (SubL src1 (ConvI2L src2)));
13059   ins_cost(INSN_COST);
13060   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13061 
13062    ins_encode %{
13063      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13064             as_Register($src2$$reg), ext::sxtw);
13065    %}
13066   ins_pipe(ialu_reg_reg);
13067 %}
13068 
13069 // This pattern is automatically generated from aarch64_ad.m4
13070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13071 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13072 %{
13073   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13074   ins_cost(INSN_COST);
13075   format %{ "add  $dst, $src1, $src2, sxth" %}
13076 
13077    ins_encode %{
13078      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13079             as_Register($src2$$reg), ext::sxth);
13080    %}
13081   ins_pipe(ialu_reg_reg);
13082 %}
13083 
13084 // This pattern is automatically generated from aarch64_ad.m4
13085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13086 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13087 %{
13088   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13089   ins_cost(INSN_COST);
13090   format %{ "add  $dst, $src1, $src2, sxtb" %}
13091 
13092    ins_encode %{
13093      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13094             as_Register($src2$$reg), ext::sxtb);
13095    %}
13096   ins_pipe(ialu_reg_reg);
13097 %}
13098 
13099 // This pattern is automatically generated from aarch64_ad.m4
13100 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13101 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13102 %{
13103   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13104   ins_cost(INSN_COST);
13105   format %{ "add  $dst, $src1, $src2, uxtb" %}
13106 
13107    ins_encode %{
13108      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13109             as_Register($src2$$reg), ext::uxtb);
13110    %}
13111   ins_pipe(ialu_reg_reg);
13112 %}
13113 
13114 // This pattern is automatically generated from aarch64_ad.m4
13115 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13116 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13117 %{
13118   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13119   ins_cost(INSN_COST);
13120   format %{ "add  $dst, $src1, $src2, sxth" %}
13121 
13122    ins_encode %{
13123      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13124             as_Register($src2$$reg), ext::sxth);
13125    %}
13126   ins_pipe(ialu_reg_reg);
13127 %}
13128 
13129 // This pattern is automatically generated from aarch64_ad.m4
13130 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13131 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13132 %{
13133   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13134   ins_cost(INSN_COST);
13135   format %{ "add  $dst, $src1, $src2, sxtw" %}
13136 
13137    ins_encode %{
13138      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13139             as_Register($src2$$reg), ext::sxtw);
13140    %}
13141   ins_pipe(ialu_reg_reg);
13142 %}
13143 
13144 // This pattern is automatically generated from aarch64_ad.m4
13145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13146 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13147 %{
13148   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13149   ins_cost(INSN_COST);
13150   format %{ "add  $dst, $src1, $src2, sxtb" %}
13151 
13152    ins_encode %{
13153      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13154             as_Register($src2$$reg), ext::sxtb);
13155    %}
13156   ins_pipe(ialu_reg_reg);
13157 %}
13158 
13159 // This pattern is automatically generated from aarch64_ad.m4
13160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13161 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13162 %{
13163   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13164   ins_cost(INSN_COST);
13165   format %{ "add  $dst, $src1, $src2, uxtb" %}
13166 
13167    ins_encode %{
13168      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13169             as_Register($src2$$reg), ext::uxtb);
13170    %}
13171   ins_pipe(ialu_reg_reg);
13172 %}
13173 
13174 // This pattern is automatically generated from aarch64_ad.m4
13175 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13176 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13177 %{
13178   match(Set dst (AddI src1 (AndI src2 mask)));
13179   ins_cost(INSN_COST);
13180   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13181 
13182    ins_encode %{
13183      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13184             as_Register($src2$$reg), ext::uxtb);
13185    %}
13186   ins_pipe(ialu_reg_reg);
13187 %}
13188 
13189 // This pattern is automatically generated from aarch64_ad.m4
13190 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13191 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13192 %{
13193   match(Set dst (AddI src1 (AndI src2 mask)));
13194   ins_cost(INSN_COST);
13195   format %{ "addw  $dst, $src1, $src2, uxth" %}
13196 
13197    ins_encode %{
13198      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13199             as_Register($src2$$reg), ext::uxth);
13200    %}
13201   ins_pipe(ialu_reg_reg);
13202 %}
13203 
13204 // This pattern is automatically generated from aarch64_ad.m4
13205 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13206 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13207 %{
13208   match(Set dst (AddL src1 (AndL src2 mask)));
13209   ins_cost(INSN_COST);
13210   format %{ "add  $dst, $src1, $src2, uxtb" %}
13211 
13212    ins_encode %{
13213      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13214             as_Register($src2$$reg), ext::uxtb);
13215    %}
13216   ins_pipe(ialu_reg_reg);
13217 %}
13218 
13219 // This pattern is automatically generated from aarch64_ad.m4
13220 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13221 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13222 %{
13223   match(Set dst (AddL src1 (AndL src2 mask)));
13224   ins_cost(INSN_COST);
13225   format %{ "add  $dst, $src1, $src2, uxth" %}
13226 
13227    ins_encode %{
13228      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13229             as_Register($src2$$reg), ext::uxth);
13230    %}
13231   ins_pipe(ialu_reg_reg);
13232 %}
13233 
13234 // This pattern is automatically generated from aarch64_ad.m4
13235 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13236 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13237 %{
13238   match(Set dst (AddL src1 (AndL src2 mask)));
13239   ins_cost(INSN_COST);
13240   format %{ "add  $dst, $src1, $src2, uxtw" %}
13241 
13242    ins_encode %{
13243      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13244             as_Register($src2$$reg), ext::uxtw);
13245    %}
13246   ins_pipe(ialu_reg_reg);
13247 %}
13248 
13249 // This pattern is automatically generated from aarch64_ad.m4
13250 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13251 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13252 %{
13253   match(Set dst (SubI src1 (AndI src2 mask)));
13254   ins_cost(INSN_COST);
13255   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13256 
13257    ins_encode %{
13258      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13259             as_Register($src2$$reg), ext::uxtb);
13260    %}
13261   ins_pipe(ialu_reg_reg);
13262 %}
13263 
13264 // This pattern is automatically generated from aarch64_ad.m4
13265 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13266 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13267 %{
13268   match(Set dst (SubI src1 (AndI src2 mask)));
13269   ins_cost(INSN_COST);
13270   format %{ "subw  $dst, $src1, $src2, uxth" %}
13271 
13272    ins_encode %{
13273      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13274             as_Register($src2$$reg), ext::uxth);
13275    %}
13276   ins_pipe(ialu_reg_reg);
13277 %}
13278 
13279 // This pattern is automatically generated from aarch64_ad.m4
13280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13281 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13282 %{
13283   match(Set dst (SubL src1 (AndL src2 mask)));
13284   ins_cost(INSN_COST);
13285   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13286 
13287    ins_encode %{
13288      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13289             as_Register($src2$$reg), ext::uxtb);
13290    %}
13291   ins_pipe(ialu_reg_reg);
13292 %}
13293 
13294 // This pattern is automatically generated from aarch64_ad.m4
13295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13296 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13297 %{
13298   match(Set dst (SubL src1 (AndL src2 mask)));
13299   ins_cost(INSN_COST);
13300   format %{ "sub  $dst, $src1, $src2, uxth" %}
13301 
13302    ins_encode %{
13303      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13304             as_Register($src2$$reg), ext::uxth);
13305    %}
13306   ins_pipe(ialu_reg_reg);
13307 %}
13308 
13309 // This pattern is automatically generated from aarch64_ad.m4
13310 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13311 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13312 %{
13313   match(Set dst (SubL src1 (AndL src2 mask)));
13314   ins_cost(INSN_COST);
13315   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13316 
13317    ins_encode %{
13318      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13319             as_Register($src2$$reg), ext::uxtw);
13320    %}
13321   ins_pipe(ialu_reg_reg);
13322 %}
13323 
13324 
13325 // This pattern is automatically generated from aarch64_ad.m4
13326 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13327 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13328 %{
13329   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13330   ins_cost(1.9 * INSN_COST);
13331   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13332 
13333    ins_encode %{
13334      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13335             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13336    %}
13337   ins_pipe(ialu_reg_reg_shift);
13338 %}
13339 
13340 // This pattern is automatically generated from aarch64_ad.m4
13341 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13342 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13343 %{
13344   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13345   ins_cost(1.9 * INSN_COST);
13346   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13347 
13348    ins_encode %{
13349      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13350             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13351    %}
13352   ins_pipe(ialu_reg_reg_shift);
13353 %}
13354 
13355 // This pattern is automatically generated from aarch64_ad.m4
13356 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13357 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13358 %{
13359   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13360   ins_cost(1.9 * INSN_COST);
13361   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13362 
13363    ins_encode %{
13364      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13365             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13366    %}
13367   ins_pipe(ialu_reg_reg_shift);
13368 %}
13369 
13370 // This pattern is automatically generated from aarch64_ad.m4
13371 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13372 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13373 %{
13374   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13375   ins_cost(1.9 * INSN_COST);
13376   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13377 
13378    ins_encode %{
13379      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13380             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13381    %}
13382   ins_pipe(ialu_reg_reg_shift);
13383 %}
13384 
13385 // This pattern is automatically generated from aarch64_ad.m4
13386 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13387 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13388 %{
13389   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13390   ins_cost(1.9 * INSN_COST);
13391   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13392 
13393    ins_encode %{
13394      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13395             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13396    %}
13397   ins_pipe(ialu_reg_reg_shift);
13398 %}
13399 
13400 // This pattern is automatically generated from aarch64_ad.m4
13401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13402 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13403 %{
13404   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13405   ins_cost(1.9 * INSN_COST);
13406   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13407 
13408    ins_encode %{
13409      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13410             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13411    %}
13412   ins_pipe(ialu_reg_reg_shift);
13413 %}
13414 
13415 // This pattern is automatically generated from aarch64_ad.m4
13416 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13417 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13418 %{
13419   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13420   ins_cost(1.9 * INSN_COST);
13421   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13422 
13423    ins_encode %{
13424      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13425             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13426    %}
13427   ins_pipe(ialu_reg_reg_shift);
13428 %}
13429 
13430 // This pattern is automatically generated from aarch64_ad.m4
13431 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13432 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13433 %{
13434   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13435   ins_cost(1.9 * INSN_COST);
13436   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13437 
13438    ins_encode %{
13439      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13440             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13441    %}
13442   ins_pipe(ialu_reg_reg_shift);
13443 %}
13444 
13445 // This pattern is automatically generated from aarch64_ad.m4
13446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13447 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13448 %{
13449   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13450   ins_cost(1.9 * INSN_COST);
13451   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13452 
13453    ins_encode %{
13454      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13455             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13456    %}
13457   ins_pipe(ialu_reg_reg_shift);
13458 %}
13459 
13460 // This pattern is automatically generated from aarch64_ad.m4
13461 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13462 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13463 %{
13464   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13465   ins_cost(1.9 * INSN_COST);
13466   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13467 
13468    ins_encode %{
13469      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13470             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13471    %}
13472   ins_pipe(ialu_reg_reg_shift);
13473 %}
13474 
13475 // This pattern is automatically generated from aarch64_ad.m4
13476 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13477 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13478 %{
13479   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13480   ins_cost(1.9 * INSN_COST);
13481   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13482 
13483    ins_encode %{
13484      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13485             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13486    %}
13487   ins_pipe(ialu_reg_reg_shift);
13488 %}
13489 
13490 // This pattern is automatically generated from aarch64_ad.m4
13491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13492 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13493 %{
13494   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13495   ins_cost(1.9 * INSN_COST);
13496   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13497 
13498    ins_encode %{
13499      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13500             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13501    %}
13502   ins_pipe(ialu_reg_reg_shift);
13503 %}
13504 
13505 // This pattern is automatically generated from aarch64_ad.m4
13506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13507 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13508 %{
13509   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13510   ins_cost(1.9 * INSN_COST);
13511   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13512 
13513    ins_encode %{
13514      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13515             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13516    %}
13517   ins_pipe(ialu_reg_reg_shift);
13518 %}
13519 
13520 // This pattern is automatically generated from aarch64_ad.m4
13521 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13522 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13523 %{
13524   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13525   ins_cost(1.9 * INSN_COST);
13526   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13527 
13528    ins_encode %{
13529      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13530             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13531    %}
13532   ins_pipe(ialu_reg_reg_shift);
13533 %}
13534 
13535 // This pattern is automatically generated from aarch64_ad.m4
13536 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13537 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13538 %{
13539   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13540   ins_cost(1.9 * INSN_COST);
13541   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13542 
13543    ins_encode %{
13544      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13545             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13546    %}
13547   ins_pipe(ialu_reg_reg_shift);
13548 %}
13549 
13550 // This pattern is automatically generated from aarch64_ad.m4
13551 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13552 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13553 %{
13554   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13555   ins_cost(1.9 * INSN_COST);
13556   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13557 
13558    ins_encode %{
13559      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13560             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13561    %}
13562   ins_pipe(ialu_reg_reg_shift);
13563 %}
13564 
13565 // This pattern is automatically generated from aarch64_ad.m4
13566 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13567 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13568 %{
13569   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13570   ins_cost(1.9 * INSN_COST);
13571   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13572 
13573    ins_encode %{
13574      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13575             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13576    %}
13577   ins_pipe(ialu_reg_reg_shift);
13578 %}
13579 
13580 // This pattern is automatically generated from aarch64_ad.m4
13581 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13582 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13583 %{
13584   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13585   ins_cost(1.9 * INSN_COST);
13586   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13587 
13588    ins_encode %{
13589      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13590             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13591    %}
13592   ins_pipe(ialu_reg_reg_shift);
13593 %}
13594 
13595 // This pattern is automatically generated from aarch64_ad.m4
13596 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13597 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13598 %{
13599   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13600   ins_cost(1.9 * INSN_COST);
13601   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13602 
13603    ins_encode %{
13604      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13605             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13606    %}
13607   ins_pipe(ialu_reg_reg_shift);
13608 %}
13609 
13610 // This pattern is automatically generated from aarch64_ad.m4
13611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13612 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13613 %{
13614   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13615   ins_cost(1.9 * INSN_COST);
13616   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13617 
13618    ins_encode %{
13619      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13620             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13621    %}
13622   ins_pipe(ialu_reg_reg_shift);
13623 %}
13624 
13625 // This pattern is automatically generated from aarch64_ad.m4
13626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13627 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13628 %{
13629   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13630   ins_cost(1.9 * INSN_COST);
13631   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13632 
13633    ins_encode %{
13634      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13635             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13636    %}
13637   ins_pipe(ialu_reg_reg_shift);
13638 %}
13639 
13640 // This pattern is automatically generated from aarch64_ad.m4
13641 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13642 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13643 %{
13644   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13645   ins_cost(1.9 * INSN_COST);
13646   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13647 
13648    ins_encode %{
13649      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13650             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13651    %}
13652   ins_pipe(ialu_reg_reg_shift);
13653 %}
13654 
13655 
13656 
13657 // END This section of the file is automatically generated. Do not edit --------------
13658 
13659 
13660 // ============================================================================
13661 // Floating Point Arithmetic Instructions
13662 
13663 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13664   match(Set dst (AddF src1 src2));
13665 
13666   ins_cost(INSN_COST * 5);
13667   format %{ "fadds   $dst, $src1, $src2" %}
13668 
13669   ins_encode %{
13670     __ fadds(as_FloatRegister($dst$$reg),
13671              as_FloatRegister($src1$$reg),
13672              as_FloatRegister($src2$$reg));
13673   %}
13674 
13675   ins_pipe(fp_dop_reg_reg_s);
13676 %}
13677 
13678 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13679   match(Set dst (AddD src1 src2));
13680 
13681   ins_cost(INSN_COST * 5);
13682   format %{ "faddd   $dst, $src1, $src2" %}
13683 
13684   ins_encode %{
13685     __ faddd(as_FloatRegister($dst$$reg),
13686              as_FloatRegister($src1$$reg),
13687              as_FloatRegister($src2$$reg));
13688   %}
13689 
13690   ins_pipe(fp_dop_reg_reg_d);
13691 %}
13692 
13693 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13694   match(Set dst (SubF src1 src2));
13695 
13696   ins_cost(INSN_COST * 5);
13697   format %{ "fsubs   $dst, $src1, $src2" %}
13698 
13699   ins_encode %{
13700     __ fsubs(as_FloatRegister($dst$$reg),
13701              as_FloatRegister($src1$$reg),
13702              as_FloatRegister($src2$$reg));
13703   %}
13704 
13705   ins_pipe(fp_dop_reg_reg_s);
13706 %}
13707 
13708 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13709   match(Set dst (SubD src1 src2));
13710 
13711   ins_cost(INSN_COST * 5);
13712   format %{ "fsubd   $dst, $src1, $src2" %}
13713 
13714   ins_encode %{
13715     __ fsubd(as_FloatRegister($dst$$reg),
13716              as_FloatRegister($src1$$reg),
13717              as_FloatRegister($src2$$reg));
13718   %}
13719 
13720   ins_pipe(fp_dop_reg_reg_d);
13721 %}
13722 
13723 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13724   match(Set dst (MulF src1 src2));
13725 
13726   ins_cost(INSN_COST * 6);
13727   format %{ "fmuls   $dst, $src1, $src2" %}
13728 
13729   ins_encode %{
13730     __ fmuls(as_FloatRegister($dst$$reg),
13731              as_FloatRegister($src1$$reg),
13732              as_FloatRegister($src2$$reg));
13733   %}
13734 
13735   ins_pipe(fp_dop_reg_reg_s);
13736 %}
13737 
13738 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13739   match(Set dst (MulD src1 src2));
13740 
13741   ins_cost(INSN_COST * 6);
13742   format %{ "fmuld   $dst, $src1, $src2" %}
13743 
13744   ins_encode %{
13745     __ fmuld(as_FloatRegister($dst$$reg),
13746              as_FloatRegister($src1$$reg),
13747              as_FloatRegister($src2$$reg));
13748   %}
13749 
13750   ins_pipe(fp_dop_reg_reg_d);
13751 %}
13752 
13753 // src1 * src2 + src3
13754 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13755   predicate(UseFMA);
13756   match(Set dst (FmaF src3 (Binary src1 src2)));
13757 
13758   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
13759 
13760   ins_encode %{
13761     __ fmadds(as_FloatRegister($dst$$reg),
13762              as_FloatRegister($src1$$reg),
13763              as_FloatRegister($src2$$reg),
13764              as_FloatRegister($src3$$reg));
13765   %}
13766 
13767   ins_pipe(pipe_class_default);
13768 %}
13769 
13770 // src1 * src2 + src3
13771 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13772   predicate(UseFMA);
13773   match(Set dst (FmaD src3 (Binary src1 src2)));
13774 
13775   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
13776 
13777   ins_encode %{
13778     __ fmaddd(as_FloatRegister($dst$$reg),
13779              as_FloatRegister($src1$$reg),
13780              as_FloatRegister($src2$$reg),
13781              as_FloatRegister($src3$$reg));
13782   %}
13783 
13784   ins_pipe(pipe_class_default);
13785 %}
13786 
13787 // -src1 * src2 + src3
13788 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13789   predicate(UseFMA);
13790   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
13791   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13792 
13793   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
13794 
13795   ins_encode %{
13796     __ fmsubs(as_FloatRegister($dst$$reg),
13797               as_FloatRegister($src1$$reg),
13798               as_FloatRegister($src2$$reg),
13799               as_FloatRegister($src3$$reg));
13800   %}
13801 
13802   ins_pipe(pipe_class_default);
13803 %}
13804 
13805 // -src1 * src2 + src3
13806 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13807   predicate(UseFMA);
13808   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
13809   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13810 
13811   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
13812 
13813   ins_encode %{
13814     __ fmsubd(as_FloatRegister($dst$$reg),
13815               as_FloatRegister($src1$$reg),
13816               as_FloatRegister($src2$$reg),
13817               as_FloatRegister($src3$$reg));
13818   %}
13819 
13820   ins_pipe(pipe_class_default);
13821 %}
13822 
13823 // -src1 * src2 - src3
13824 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13825   predicate(UseFMA);
13826   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
13827   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13828 
13829   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
13830 
13831   ins_encode %{
13832     __ fnmadds(as_FloatRegister($dst$$reg),
13833                as_FloatRegister($src1$$reg),
13834                as_FloatRegister($src2$$reg),
13835                as_FloatRegister($src3$$reg));
13836   %}
13837 
13838   ins_pipe(pipe_class_default);
13839 %}
13840 
13841 // -src1 * src2 - src3
13842 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13843   predicate(UseFMA);
13844   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
13845   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13846 
13847   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
13848 
13849   ins_encode %{
13850     __ fnmaddd(as_FloatRegister($dst$$reg),
13851                as_FloatRegister($src1$$reg),
13852                as_FloatRegister($src2$$reg),
13853                as_FloatRegister($src3$$reg));
13854   %}
13855 
13856   ins_pipe(pipe_class_default);
13857 %}
13858 
13859 // src1 * src2 - src3
13860 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13861   predicate(UseFMA);
13862   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13863 
13864   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
13865 
13866   ins_encode %{
13867     __ fnmsubs(as_FloatRegister($dst$$reg),
13868                as_FloatRegister($src1$$reg),
13869                as_FloatRegister($src2$$reg),
13870                as_FloatRegister($src3$$reg));
13871   %}
13872 
13873   ins_pipe(pipe_class_default);
13874 %}
13875 
13876 // src1 * src2 - src3
13877 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13878   predicate(UseFMA);
13879   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13880 
13881   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
13882 
13883   ins_encode %{
13884   // n.b. insn name should be fnmsubd
13885     __ fnmsub(as_FloatRegister($dst$$reg),
13886               as_FloatRegister($src1$$reg),
13887               as_FloatRegister($src2$$reg),
13888               as_FloatRegister($src3$$reg));
13889   %}
13890 
13891   ins_pipe(pipe_class_default);
13892 %}
13893 
13894 
13895 // Math.max(FF)F
13896 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13897   match(Set dst (MaxF src1 src2));
13898 
13899   format %{ "fmaxs   $dst, $src1, $src2" %}
13900   ins_encode %{
13901     __ fmaxs(as_FloatRegister($dst$$reg),
13902              as_FloatRegister($src1$$reg),
13903              as_FloatRegister($src2$$reg));
13904   %}
13905 
13906   ins_pipe(fp_dop_reg_reg_s);
13907 %}
13908 
13909 // Math.min(FF)F
13910 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13911   match(Set dst (MinF src1 src2));
13912 
13913   format %{ "fmins   $dst, $src1, $src2" %}
13914   ins_encode %{
13915     __ fmins(as_FloatRegister($dst$$reg),
13916              as_FloatRegister($src1$$reg),
13917              as_FloatRegister($src2$$reg));
13918   %}
13919 
13920   ins_pipe(fp_dop_reg_reg_s);
13921 %}
13922 
13923 // Math.max(DD)D
13924 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13925   match(Set dst (MaxD src1 src2));
13926 
13927   format %{ "fmaxd   $dst, $src1, $src2" %}
13928   ins_encode %{
13929     __ fmaxd(as_FloatRegister($dst$$reg),
13930              as_FloatRegister($src1$$reg),
13931              as_FloatRegister($src2$$reg));
13932   %}
13933 
13934   ins_pipe(fp_dop_reg_reg_d);
13935 %}
13936 
13937 // Math.min(DD)D
13938 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13939   match(Set dst (MinD src1 src2));
13940 
13941   format %{ "fmind   $dst, $src1, $src2" %}
13942   ins_encode %{
13943     __ fmind(as_FloatRegister($dst$$reg),
13944              as_FloatRegister($src1$$reg),
13945              as_FloatRegister($src2$$reg));
13946   %}
13947 
13948   ins_pipe(fp_dop_reg_reg_d);
13949 %}
13950 
13951 
13952 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13953   match(Set dst (DivF src1  src2));
13954 
13955   ins_cost(INSN_COST * 18);
13956   format %{ "fdivs   $dst, $src1, $src2" %}
13957 
13958   ins_encode %{
13959     __ fdivs(as_FloatRegister($dst$$reg),
13960              as_FloatRegister($src1$$reg),
13961              as_FloatRegister($src2$$reg));
13962   %}
13963 
13964   ins_pipe(fp_div_s);
13965 %}
13966 
13967 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13968   match(Set dst (DivD src1  src2));
13969 
13970   ins_cost(INSN_COST * 32);
13971   format %{ "fdivd   $dst, $src1, $src2" %}
13972 
13973   ins_encode %{
13974     __ fdivd(as_FloatRegister($dst$$reg),
13975              as_FloatRegister($src1$$reg),
13976              as_FloatRegister($src2$$reg));
13977   %}
13978 
13979   ins_pipe(fp_div_d);
13980 %}
13981 
13982 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13983   match(Set dst (NegF src));
13984 
13985   ins_cost(INSN_COST * 3);
13986   format %{ "fneg   $dst, $src" %}
13987 
13988   ins_encode %{
13989     __ fnegs(as_FloatRegister($dst$$reg),
13990              as_FloatRegister($src$$reg));
13991   %}
13992 
13993   ins_pipe(fp_uop_s);
13994 %}
13995 
13996 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13997   match(Set dst (NegD src));
13998 
13999   ins_cost(INSN_COST * 3);
14000   format %{ "fnegd   $dst, $src" %}
14001 
14002   ins_encode %{
14003     __ fnegd(as_FloatRegister($dst$$reg),
14004              as_FloatRegister($src$$reg));
14005   %}
14006 
14007   ins_pipe(fp_uop_d);
14008 %}
14009 
14010 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14011 %{
14012   match(Set dst (AbsI src));
14013 
14014   effect(KILL cr);
14015   ins_cost(INSN_COST * 2);
14016   format %{ "cmpw  $src, zr\n\t"
14017             "cnegw $dst, $src, Assembler::LT\t# int abs"
14018   %}
14019 
14020   ins_encode %{
14021     __ cmpw(as_Register($src$$reg), zr);
14022     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14023   %}
14024   ins_pipe(pipe_class_default);
14025 %}
14026 
14027 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14028 %{
14029   match(Set dst (AbsL src));
14030 
14031   effect(KILL cr);
14032   ins_cost(INSN_COST * 2);
14033   format %{ "cmp  $src, zr\n\t"
14034             "cneg $dst, $src, Assembler::LT\t# long abs"
14035   %}
14036 
14037   ins_encode %{
14038     __ cmp(as_Register($src$$reg), zr);
14039     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14040   %}
14041   ins_pipe(pipe_class_default);
14042 %}
14043 
14044 instruct absF_reg(vRegF dst, vRegF src) %{
14045   match(Set dst (AbsF src));
14046 
14047   ins_cost(INSN_COST * 3);
14048   format %{ "fabss   $dst, $src" %}
14049   ins_encode %{
14050     __ fabss(as_FloatRegister($dst$$reg),
14051              as_FloatRegister($src$$reg));
14052   %}
14053 
14054   ins_pipe(fp_uop_s);
14055 %}
14056 
14057 instruct absD_reg(vRegD dst, vRegD src) %{
14058   match(Set dst (AbsD src));
14059 
14060   ins_cost(INSN_COST * 3);
14061   format %{ "fabsd   $dst, $src" %}
14062   ins_encode %{
14063     __ fabsd(as_FloatRegister($dst$$reg),
14064              as_FloatRegister($src$$reg));
14065   %}
14066 
14067   ins_pipe(fp_uop_d);
14068 %}
14069 
14070 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14071   match(Set dst (AbsF (SubF src1 src2)));
14072 
14073   ins_cost(INSN_COST * 3);
14074   format %{ "fabds   $dst, $src1, $src2" %}
14075   ins_encode %{
14076     __ fabds(as_FloatRegister($dst$$reg),
14077              as_FloatRegister($src1$$reg),
14078              as_FloatRegister($src2$$reg));
14079   %}
14080 
14081   ins_pipe(fp_uop_s);
14082 %}
14083 
14084 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14085   match(Set dst (AbsD (SubD src1 src2)));
14086 
14087   ins_cost(INSN_COST * 3);
14088   format %{ "fabdd   $dst, $src1, $src2" %}
14089   ins_encode %{
14090     __ fabdd(as_FloatRegister($dst$$reg),
14091              as_FloatRegister($src1$$reg),
14092              as_FloatRegister($src2$$reg));
14093   %}
14094 
14095   ins_pipe(fp_uop_d);
14096 %}
14097 
14098 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14099   match(Set dst (SqrtD src));
14100 
14101   ins_cost(INSN_COST * 50);
14102   format %{ "fsqrtd  $dst, $src" %}
14103   ins_encode %{
14104     __ fsqrtd(as_FloatRegister($dst$$reg),
14105              as_FloatRegister($src$$reg));
14106   %}
14107 
14108   ins_pipe(fp_div_s);
14109 %}
14110 
14111 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14112   match(Set dst (SqrtF src));
14113 
14114   ins_cost(INSN_COST * 50);
14115   format %{ "fsqrts  $dst, $src" %}
14116   ins_encode %{
14117     __ fsqrts(as_FloatRegister($dst$$reg),
14118              as_FloatRegister($src$$reg));
14119   %}
14120 
14121   ins_pipe(fp_div_d);
14122 %}
14123 
14124 // Math.rint, floor, ceil
14125 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14126   match(Set dst (RoundDoubleMode src rmode));
14127   format %{ "frint  $dst, $src, $rmode" %}
14128   ins_encode %{
14129     switch ($rmode$$constant) {
14130       case RoundDoubleModeNode::rmode_rint:
14131         __ frintnd(as_FloatRegister($dst$$reg),
14132                    as_FloatRegister($src$$reg));
14133         break;
14134       case RoundDoubleModeNode::rmode_floor:
14135         __ frintmd(as_FloatRegister($dst$$reg),
14136                    as_FloatRegister($src$$reg));
14137         break;
14138       case RoundDoubleModeNode::rmode_ceil:
14139         __ frintpd(as_FloatRegister($dst$$reg),
14140                    as_FloatRegister($src$$reg));
14141         break;
14142     }
14143   %}
14144   ins_pipe(fp_uop_d);
14145 %}
14146 
14147 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14148   match(Set dst (CopySignD src1 (Binary src2 zero)));
14149   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14150   format %{ "CopySignD  $dst $src1 $src2" %}
14151   ins_encode %{
14152     FloatRegister dst = as_FloatRegister($dst$$reg),
14153                   src1 = as_FloatRegister($src1$$reg),
14154                   src2 = as_FloatRegister($src2$$reg),
14155                   zero = as_FloatRegister($zero$$reg);
14156     __ fnegd(dst, zero);
14157     __ bsl(dst, __ T8B, src2, src1);
14158   %}
14159   ins_pipe(fp_uop_d);
14160 %}
14161 
14162 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14163   match(Set dst (CopySignF src1 src2));
14164   effect(TEMP_DEF dst, USE src1, USE src2);
14165   format %{ "CopySignF  $dst $src1 $src2" %}
14166   ins_encode %{
14167     FloatRegister dst = as_FloatRegister($dst$$reg),
14168                   src1 = as_FloatRegister($src1$$reg),
14169                   src2 = as_FloatRegister($src2$$reg);
14170     __ movi(dst, __ T2S, 0x80, 24);
14171     __ bsl(dst, __ T8B, src2, src1);
14172   %}
14173   ins_pipe(fp_uop_d);
14174 %}
14175 
14176 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14177   match(Set dst (SignumD src (Binary zero one)));
14178   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14179   format %{ "signumD  $dst, $src" %}
14180   ins_encode %{
14181     FloatRegister src = as_FloatRegister($src$$reg),
14182                   dst = as_FloatRegister($dst$$reg),
14183                   zero = as_FloatRegister($zero$$reg),
14184                   one = as_FloatRegister($one$$reg);
14185     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14186     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14187     // Bit selection instruction gets bit from "one" for each enabled bit in
14188     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14189     // NaN the whole "src" will be copied because "dst" is zero. For all other
14190     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14191     // from "src", and all other bits are copied from 1.0.
14192     __ bsl(dst, __ T8B, one, src);
14193   %}
14194   ins_pipe(fp_uop_d);
14195 %}
14196 
14197 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14198   match(Set dst (SignumF src (Binary zero one)));
14199   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14200   format %{ "signumF  $dst, $src" %}
14201   ins_encode %{
14202     FloatRegister src = as_FloatRegister($src$$reg),
14203                   dst = as_FloatRegister($dst$$reg),
14204                   zero = as_FloatRegister($zero$$reg),
14205                   one = as_FloatRegister($one$$reg);
14206     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14207     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14208     // Bit selection instruction gets bit from "one" for each enabled bit in
14209     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14210     // NaN the whole "src" will be copied because "dst" is zero. For all other
14211     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14212     // from "src", and all other bits are copied from 1.0.
14213     __ bsl(dst, __ T8B, one, src);
14214   %}
14215   ins_pipe(fp_uop_d);
14216 %}
14217 
14218 instruct onspinwait() %{
14219   match(OnSpinWait);
14220   ins_cost(INSN_COST);
14221 
14222   format %{ "onspinwait" %}
14223 
14224   ins_encode %{
14225     __ spin_wait();
14226   %}
14227   ins_pipe(pipe_class_empty);
14228 %}
14229 
14230 // ============================================================================
14231 // Logical Instructions
14232 
14233 // Integer Logical Instructions
14234 
14235 // And Instructions
14236 
14237 
14238 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14239   match(Set dst (AndI src1 src2));
14240 
14241   format %{ "andw  $dst, $src1, $src2\t# int" %}
14242 
14243   ins_cost(INSN_COST);
14244   ins_encode %{
14245     __ andw(as_Register($dst$$reg),
14246             as_Register($src1$$reg),
14247             as_Register($src2$$reg));
14248   %}
14249 
14250   ins_pipe(ialu_reg_reg);
14251 %}
14252 
14253 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14254   match(Set dst (AndI src1 src2));
14255 
14256   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14257 
14258   ins_cost(INSN_COST);
14259   ins_encode %{
14260     __ andw(as_Register($dst$$reg),
14261             as_Register($src1$$reg),
14262             (uint64_t)($src2$$constant));
14263   %}
14264 
14265   ins_pipe(ialu_reg_imm);
14266 %}
14267 
14268 // Or Instructions
14269 
14270 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14271   match(Set dst (OrI src1 src2));
14272 
14273   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14274 
14275   ins_cost(INSN_COST);
14276   ins_encode %{
14277     __ orrw(as_Register($dst$$reg),
14278             as_Register($src1$$reg),
14279             as_Register($src2$$reg));
14280   %}
14281 
14282   ins_pipe(ialu_reg_reg);
14283 %}
14284 
14285 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14286   match(Set dst (OrI src1 src2));
14287 
14288   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14289 
14290   ins_cost(INSN_COST);
14291   ins_encode %{
14292     __ orrw(as_Register($dst$$reg),
14293             as_Register($src1$$reg),
14294             (uint64_t)($src2$$constant));
14295   %}
14296 
14297   ins_pipe(ialu_reg_imm);
14298 %}
14299 
14300 // Xor Instructions
14301 
14302 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14303   match(Set dst (XorI src1 src2));
14304 
14305   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14306 
14307   ins_cost(INSN_COST);
14308   ins_encode %{
14309     __ eorw(as_Register($dst$$reg),
14310             as_Register($src1$$reg),
14311             as_Register($src2$$reg));
14312   %}
14313 
14314   ins_pipe(ialu_reg_reg);
14315 %}
14316 
14317 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14318   match(Set dst (XorI src1 src2));
14319 
14320   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14321 
14322   ins_cost(INSN_COST);
14323   ins_encode %{
14324     __ eorw(as_Register($dst$$reg),
14325             as_Register($src1$$reg),
14326             (uint64_t)($src2$$constant));
14327   %}
14328 
14329   ins_pipe(ialu_reg_imm);
14330 %}
14331 
14332 // Long Logical Instructions
14333 // TODO
14334 
14335 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14336   match(Set dst (AndL src1 src2));
14337 
14338   format %{ "and  $dst, $src1, $src2\t# int" %}
14339 
14340   ins_cost(INSN_COST);
14341   ins_encode %{
14342     __ andr(as_Register($dst$$reg),
14343             as_Register($src1$$reg),
14344             as_Register($src2$$reg));
14345   %}
14346 
14347   ins_pipe(ialu_reg_reg);
14348 %}
14349 
14350 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14351   match(Set dst (AndL src1 src2));
14352 
14353   format %{ "and  $dst, $src1, $src2\t# int" %}
14354 
14355   ins_cost(INSN_COST);
14356   ins_encode %{
14357     __ andr(as_Register($dst$$reg),
14358             as_Register($src1$$reg),
14359             (uint64_t)($src2$$constant));
14360   %}
14361 
14362   ins_pipe(ialu_reg_imm);
14363 %}
14364 
14365 // Or Instructions
14366 
14367 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14368   match(Set dst (OrL src1 src2));
14369 
14370   format %{ "orr  $dst, $src1, $src2\t# int" %}
14371 
14372   ins_cost(INSN_COST);
14373   ins_encode %{
14374     __ orr(as_Register($dst$$reg),
14375            as_Register($src1$$reg),
14376            as_Register($src2$$reg));
14377   %}
14378 
14379   ins_pipe(ialu_reg_reg);
14380 %}
14381 
14382 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14383   match(Set dst (OrL src1 src2));
14384 
14385   format %{ "orr  $dst, $src1, $src2\t# int" %}
14386 
14387   ins_cost(INSN_COST);
14388   ins_encode %{
14389     __ orr(as_Register($dst$$reg),
14390            as_Register($src1$$reg),
14391            (uint64_t)($src2$$constant));
14392   %}
14393 
14394   ins_pipe(ialu_reg_imm);
14395 %}
14396 
14397 // Xor Instructions
14398 
14399 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14400   match(Set dst (XorL src1 src2));
14401 
14402   format %{ "eor  $dst, $src1, $src2\t# int" %}
14403 
14404   ins_cost(INSN_COST);
14405   ins_encode %{
14406     __ eor(as_Register($dst$$reg),
14407            as_Register($src1$$reg),
14408            as_Register($src2$$reg));
14409   %}
14410 
14411   ins_pipe(ialu_reg_reg);
14412 %}
14413 
14414 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14415   match(Set dst (XorL src1 src2));
14416 
14417   ins_cost(INSN_COST);
14418   format %{ "eor  $dst, $src1, $src2\t# int" %}
14419 
14420   ins_encode %{
14421     __ eor(as_Register($dst$$reg),
14422            as_Register($src1$$reg),
14423            (uint64_t)($src2$$constant));
14424   %}
14425 
14426   ins_pipe(ialu_reg_imm);
14427 %}
14428 
14429 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14430 %{
14431   match(Set dst (ConvI2L src));
14432 
14433   ins_cost(INSN_COST);
14434   format %{ "sxtw  $dst, $src\t# i2l" %}
14435   ins_encode %{
14436     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14437   %}
14438   ins_pipe(ialu_reg_shift);
14439 %}
14440 
14441 // this pattern occurs in bigmath arithmetic
14442 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14443 %{
14444   match(Set dst (AndL (ConvI2L src) mask));
14445 
14446   ins_cost(INSN_COST);
14447   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14448   ins_encode %{
14449     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14450   %}
14451 
14452   ins_pipe(ialu_reg_shift);
14453 %}
14454 
14455 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14456   match(Set dst (ConvL2I src));
14457 
14458   ins_cost(INSN_COST);
14459   format %{ "movw  $dst, $src \t// l2i" %}
14460 
14461   ins_encode %{
14462     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14463   %}
14464 
14465   ins_pipe(ialu_reg);
14466 %}
14467 
14468 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14469 %{
14470   match(Set dst (Conv2B src));
14471   effect(KILL cr);
14472 
14473   format %{
14474     "cmpw $src, zr\n\t"
14475     "cset $dst, ne"
14476   %}
14477 
14478   ins_encode %{
14479     __ cmpw(as_Register($src$$reg), zr);
14480     __ cset(as_Register($dst$$reg), Assembler::NE);
14481   %}
14482 
14483   ins_pipe(ialu_reg);
14484 %}
14485 
14486 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14487 %{
14488   match(Set dst (Conv2B src));
14489   effect(KILL cr);
14490 
14491   format %{
14492     "cmp  $src, zr\n\t"
14493     "cset $dst, ne"
14494   %}
14495 
14496   ins_encode %{
14497     __ cmp(as_Register($src$$reg), zr);
14498     __ cset(as_Register($dst$$reg), Assembler::NE);
14499   %}
14500 
14501   ins_pipe(ialu_reg);
14502 %}
14503 
14504 instruct convD2F_reg(vRegF dst, vRegD src) %{
14505   match(Set dst (ConvD2F src));
14506 
14507   ins_cost(INSN_COST * 5);
14508   format %{ "fcvtd  $dst, $src \t// d2f" %}
14509 
14510   ins_encode %{
14511     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14512   %}
14513 
14514   ins_pipe(fp_d2f);
14515 %}
14516 
14517 instruct convF2D_reg(vRegD dst, vRegF src) %{
14518   match(Set dst (ConvF2D src));
14519 
14520   ins_cost(INSN_COST * 5);
14521   format %{ "fcvts  $dst, $src \t// f2d" %}
14522 
14523   ins_encode %{
14524     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14525   %}
14526 
14527   ins_pipe(fp_f2d);
14528 %}
14529 
14530 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14531   match(Set dst (ConvF2I src));
14532 
14533   ins_cost(INSN_COST * 5);
14534   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14535 
14536   ins_encode %{
14537     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14538   %}
14539 
14540   ins_pipe(fp_f2i);
14541 %}
14542 
14543 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14544   match(Set dst (ConvF2L src));
14545 
14546   ins_cost(INSN_COST * 5);
14547   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14548 
14549   ins_encode %{
14550     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14551   %}
14552 
14553   ins_pipe(fp_f2l);
14554 %}
14555 
14556 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14557   match(Set dst (ConvI2F src));
14558 
14559   ins_cost(INSN_COST * 5);
14560   format %{ "scvtfws  $dst, $src \t// i2f" %}
14561 
14562   ins_encode %{
14563     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14564   %}
14565 
14566   ins_pipe(fp_i2f);
14567 %}
14568 
14569 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14570   match(Set dst (ConvL2F src));
14571 
14572   ins_cost(INSN_COST * 5);
14573   format %{ "scvtfs  $dst, $src \t// l2f" %}
14574 
14575   ins_encode %{
14576     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14577   %}
14578 
14579   ins_pipe(fp_l2f);
14580 %}
14581 
14582 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14583   match(Set dst (ConvD2I src));
14584 
14585   ins_cost(INSN_COST * 5);
14586   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14587 
14588   ins_encode %{
14589     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14590   %}
14591 
14592   ins_pipe(fp_d2i);
14593 %}
14594 
14595 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14596   match(Set dst (ConvD2L src));
14597 
14598   ins_cost(INSN_COST * 5);
14599   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14600 
14601   ins_encode %{
14602     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14603   %}
14604 
14605   ins_pipe(fp_d2l);
14606 %}
14607 
14608 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14609   match(Set dst (ConvI2D src));
14610 
14611   ins_cost(INSN_COST * 5);
14612   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14613 
14614   ins_encode %{
14615     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14616   %}
14617 
14618   ins_pipe(fp_i2d);
14619 %}
14620 
14621 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14622   match(Set dst (ConvL2D src));
14623 
14624   ins_cost(INSN_COST * 5);
14625   format %{ "scvtfd  $dst, $src \t// l2d" %}
14626 
14627   ins_encode %{
14628     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14629   %}
14630 
14631   ins_pipe(fp_l2d);
14632 %}
14633 
14634 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14635 %{
14636   match(Set dst (RoundD src));
14637   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14638   format %{ "java_round_double $dst,$src"%}
14639   ins_encode %{
14640     __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14641                          as_FloatRegister($ftmp$$reg));
14642   %}
14643   ins_pipe(pipe_slow);
14644 %}
14645 
14646 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14647 %{
14648   match(Set dst (RoundF src));
14649   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14650   format %{ "java_round_float $dst,$src"%}
14651   ins_encode %{
14652     __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14653                         as_FloatRegister($ftmp$$reg));
14654   %}
14655   ins_pipe(pipe_slow);
14656 %}
14657 
14658 // stack <-> reg and reg <-> reg shuffles with no conversion
14659 
14660 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14661 
14662   match(Set dst (MoveF2I src));
14663 
14664   effect(DEF dst, USE src);
14665 
14666   ins_cost(4 * INSN_COST);
14667 
14668   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14669 
14670   ins_encode %{
14671     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14672   %}
14673 
14674   ins_pipe(iload_reg_reg);
14675 
14676 %}
14677 
14678 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14679 
14680   match(Set dst (MoveI2F src));
14681 
14682   effect(DEF dst, USE src);
14683 
14684   ins_cost(4 * INSN_COST);
14685 
14686   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14687 
14688   ins_encode %{
14689     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14690   %}
14691 
14692   ins_pipe(pipe_class_memory);
14693 
14694 %}
14695 
14696 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14697 
14698   match(Set dst (MoveD2L src));
14699 
14700   effect(DEF dst, USE src);
14701 
14702   ins_cost(4 * INSN_COST);
14703 
14704   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14705 
14706   ins_encode %{
14707     __ ldr($dst$$Register, Address(sp, $src$$disp));
14708   %}
14709 
14710   ins_pipe(iload_reg_reg);
14711 
14712 %}
14713 
14714 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14715 
14716   match(Set dst (MoveL2D src));
14717 
14718   effect(DEF dst, USE src);
14719 
14720   ins_cost(4 * INSN_COST);
14721 
14722   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14723 
14724   ins_encode %{
14725     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14726   %}
14727 
14728   ins_pipe(pipe_class_memory);
14729 
14730 %}
14731 
14732 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14733 
14734   match(Set dst (MoveF2I src));
14735 
14736   effect(DEF dst, USE src);
14737 
14738   ins_cost(INSN_COST);
14739 
14740   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14741 
14742   ins_encode %{
14743     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14744   %}
14745 
14746   ins_pipe(pipe_class_memory);
14747 
14748 %}
14749 
14750 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14751 
14752   match(Set dst (MoveI2F src));
14753 
14754   effect(DEF dst, USE src);
14755 
14756   ins_cost(INSN_COST);
14757 
14758   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14759 
14760   ins_encode %{
14761     __ strw($src$$Register, Address(sp, $dst$$disp));
14762   %}
14763 
14764   ins_pipe(istore_reg_reg);
14765 
14766 %}
14767 
14768 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14769 
14770   match(Set dst (MoveD2L src));
14771 
14772   effect(DEF dst, USE src);
14773 
14774   ins_cost(INSN_COST);
14775 
14776   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14777 
14778   ins_encode %{
14779     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14780   %}
14781 
14782   ins_pipe(pipe_class_memory);
14783 
14784 %}
14785 
14786 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14787 
14788   match(Set dst (MoveL2D src));
14789 
14790   effect(DEF dst, USE src);
14791 
14792   ins_cost(INSN_COST);
14793 
14794   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14795 
14796   ins_encode %{
14797     __ str($src$$Register, Address(sp, $dst$$disp));
14798   %}
14799 
14800   ins_pipe(istore_reg_reg);
14801 
14802 %}
14803 
14804 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14805 
14806   match(Set dst (MoveF2I src));
14807 
14808   effect(DEF dst, USE src);
14809 
14810   ins_cost(INSN_COST);
14811 
14812   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14813 
14814   ins_encode %{
14815     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14816   %}
14817 
14818   ins_pipe(fp_f2i);
14819 
14820 %}
14821 
14822 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14823 
14824   match(Set dst (MoveI2F src));
14825 
14826   effect(DEF dst, USE src);
14827 
14828   ins_cost(INSN_COST);
14829 
14830   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14831 
14832   ins_encode %{
14833     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14834   %}
14835 
14836   ins_pipe(fp_i2f);
14837 
14838 %}
14839 
14840 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14841 
14842   match(Set dst (MoveD2L src));
14843 
14844   effect(DEF dst, USE src);
14845 
14846   ins_cost(INSN_COST);
14847 
14848   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14849 
14850   ins_encode %{
14851     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14852   %}
14853 
14854   ins_pipe(fp_d2l);
14855 
14856 %}
14857 
14858 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14859 
14860   match(Set dst (MoveL2D src));
14861 
14862   effect(DEF dst, USE src);
14863 
14864   ins_cost(INSN_COST);
14865 
14866   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14867 
14868   ins_encode %{
14869     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14870   %}
14871 
14872   ins_pipe(fp_l2d);
14873 
14874 %}
14875 
14876 // ============================================================================
14877 // clearing of an array
14878 
14879 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14880 %{
14881   match(Set dummy (ClearArray cnt base));
14882   effect(USE_KILL cnt, USE_KILL base, KILL cr);
14883 
14884   ins_cost(4 * INSN_COST);
14885   format %{ "ClearArray $cnt, $base" %}
14886 
14887   ins_encode %{
14888     address tpc = __ zero_words($base$$Register, $cnt$$Register);
14889     if (tpc == NULL) {
14890       ciEnv::current()->record_failure("CodeCache is full");
14891       return;
14892     }
14893   %}
14894 
14895   ins_pipe(pipe_class_memory);
14896 %}
14897 
14898 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14899 %{
14900   predicate((uint64_t)n->in(2)->get_long()
14901             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14902   match(Set dummy (ClearArray cnt base));
14903   effect(TEMP temp, USE_KILL base, KILL cr);
14904 
14905   ins_cost(4 * INSN_COST);
14906   format %{ "ClearArray $cnt, $base" %}
14907 
14908   ins_encode %{
14909     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14910     if (tpc == NULL) {
14911       ciEnv::current()->record_failure("CodeCache is full");
14912       return;
14913     }
14914   %}
14915 
14916   ins_pipe(pipe_class_memory);
14917 %}
14918 
14919 // ============================================================================
14920 // Overflow Math Instructions
14921 
14922 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14923 %{
14924   match(Set cr (OverflowAddI op1 op2));
14925 
14926   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
14927   ins_cost(INSN_COST);
14928   ins_encode %{
14929     __ cmnw($op1$$Register, $op2$$Register);
14930   %}
14931 
14932   ins_pipe(icmp_reg_reg);
14933 %}
14934 
14935 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14936 %{
14937   match(Set cr (OverflowAddI op1 op2));
14938 
14939   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
14940   ins_cost(INSN_COST);
14941   ins_encode %{
14942     __ cmnw($op1$$Register, $op2$$constant);
14943   %}
14944 
14945   ins_pipe(icmp_reg_imm);
14946 %}
14947 
14948 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14949 %{
14950   match(Set cr (OverflowAddL op1 op2));
14951 
14952   format %{ "cmn   $op1, $op2\t# overflow check long" %}
14953   ins_cost(INSN_COST);
14954   ins_encode %{
14955     __ cmn($op1$$Register, $op2$$Register);
14956   %}
14957 
14958   ins_pipe(icmp_reg_reg);
14959 %}
14960 
14961 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14962 %{
14963   match(Set cr (OverflowAddL op1 op2));
14964 
14965   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
14966   ins_cost(INSN_COST);
14967   ins_encode %{
14968     __ adds(zr, $op1$$Register, $op2$$constant);
14969   %}
14970 
14971   ins_pipe(icmp_reg_imm);
14972 %}
14973 
14974 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14975 %{
14976   match(Set cr (OverflowSubI op1 op2));
14977 
14978   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
14979   ins_cost(INSN_COST);
14980   ins_encode %{
14981     __ cmpw($op1$$Register, $op2$$Register);
14982   %}
14983 
14984   ins_pipe(icmp_reg_reg);
14985 %}
14986 
14987 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14988 %{
14989   match(Set cr (OverflowSubI op1 op2));
14990 
14991   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
14992   ins_cost(INSN_COST);
14993   ins_encode %{
14994     __ cmpw($op1$$Register, $op2$$constant);
14995   %}
14996 
14997   ins_pipe(icmp_reg_imm);
14998 %}
14999 
15000 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15001 %{
15002   match(Set cr (OverflowSubL op1 op2));
15003 
15004   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15005   ins_cost(INSN_COST);
15006   ins_encode %{
15007     __ cmp($op1$$Register, $op2$$Register);
15008   %}
15009 
15010   ins_pipe(icmp_reg_reg);
15011 %}
15012 
15013 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15014 %{
15015   match(Set cr (OverflowSubL op1 op2));
15016 
15017   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15018   ins_cost(INSN_COST);
15019   ins_encode %{
15020     __ subs(zr, $op1$$Register, $op2$$constant);
15021   %}
15022 
15023   ins_pipe(icmp_reg_imm);
15024 %}
15025 
15026 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15027 %{
15028   match(Set cr (OverflowSubI zero op1));
15029 
15030   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15031   ins_cost(INSN_COST);
15032   ins_encode %{
15033     __ cmpw(zr, $op1$$Register);
15034   %}
15035 
15036   ins_pipe(icmp_reg_imm);
15037 %}
15038 
15039 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15040 %{
15041   match(Set cr (OverflowSubL zero op1));
15042 
15043   format %{ "cmp   zr, $op1\t# overflow check long" %}
15044   ins_cost(INSN_COST);
15045   ins_encode %{
15046     __ cmp(zr, $op1$$Register);
15047   %}
15048 
15049   ins_pipe(icmp_reg_imm);
15050 %}
15051 
15052 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15053 %{
15054   match(Set cr (OverflowMulI op1 op2));
15055 
15056   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15057             "cmp   rscratch1, rscratch1, sxtw\n\t"
15058             "movw  rscratch1, #0x80000000\n\t"
15059             "cselw rscratch1, rscratch1, zr, NE\n\t"
15060             "cmpw  rscratch1, #1" %}
15061   ins_cost(5 * INSN_COST);
15062   ins_encode %{
15063     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15064     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15065     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15066     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15067     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15068   %}
15069 
15070   ins_pipe(pipe_slow);
15071 %}
15072 
15073 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15074 %{
15075   match(If cmp (OverflowMulI op1 op2));
15076   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15077             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15078   effect(USE labl, KILL cr);
15079 
15080   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15081             "cmp   rscratch1, rscratch1, sxtw\n\t"
15082             "b$cmp   $labl" %}
15083   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15084   ins_encode %{
15085     Label* L = $labl$$label;
15086     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15087     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15088     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15089     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15090   %}
15091 
15092   ins_pipe(pipe_serial);
15093 %}
15094 
15095 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15096 %{
15097   match(Set cr (OverflowMulL op1 op2));
15098 
15099   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15100             "smulh rscratch2, $op1, $op2\n\t"
15101             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15102             "movw  rscratch1, #0x80000000\n\t"
15103             "cselw rscratch1, rscratch1, zr, NE\n\t"
15104             "cmpw  rscratch1, #1" %}
15105   ins_cost(6 * INSN_COST);
15106   ins_encode %{
15107     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15108     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15109     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15110     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15111     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15112     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15113   %}
15114 
15115   ins_pipe(pipe_slow);
15116 %}
15117 
15118 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15119 %{
15120   match(If cmp (OverflowMulL op1 op2));
15121   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15122             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15123   effect(USE labl, KILL cr);
15124 
15125   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15126             "smulh rscratch2, $op1, $op2\n\t"
15127             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15128             "b$cmp $labl" %}
15129   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15130   ins_encode %{
15131     Label* L = $labl$$label;
15132     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15133     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15134     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15135     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15136     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15137   %}
15138 
15139   ins_pipe(pipe_serial);
15140 %}
15141 
15142 // ============================================================================
15143 // Compare Instructions
15144 
15145 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15146 %{
15147   match(Set cr (CmpI op1 op2));
15148 
15149   effect(DEF cr, USE op1, USE op2);
15150 
15151   ins_cost(INSN_COST);
15152   format %{ "cmpw  $op1, $op2" %}
15153 
15154   ins_encode(aarch64_enc_cmpw(op1, op2));
15155 
15156   ins_pipe(icmp_reg_reg);
15157 %}
15158 
15159 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15160 %{
15161   match(Set cr (CmpI op1 zero));
15162 
15163   effect(DEF cr, USE op1);
15164 
15165   ins_cost(INSN_COST);
15166   format %{ "cmpw $op1, 0" %}
15167 
15168   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15169 
15170   ins_pipe(icmp_reg_imm);
15171 %}
15172 
15173 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15174 %{
15175   match(Set cr (CmpI op1 op2));
15176 
15177   effect(DEF cr, USE op1);
15178 
15179   ins_cost(INSN_COST);
15180   format %{ "cmpw  $op1, $op2" %}
15181 
15182   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15183 
15184   ins_pipe(icmp_reg_imm);
15185 %}
15186 
15187 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15188 %{
15189   match(Set cr (CmpI op1 op2));
15190 
15191   effect(DEF cr, USE op1);
15192 
15193   ins_cost(INSN_COST * 2);
15194   format %{ "cmpw  $op1, $op2" %}
15195 
15196   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15197 
15198   ins_pipe(icmp_reg_imm);
15199 %}
15200 
15201 // Unsigned compare Instructions; really, same as signed compare
15202 // except it should only be used to feed an If or a CMovI which takes a
15203 // cmpOpU.
15204 
15205 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15206 %{
15207   match(Set cr (CmpU op1 op2));
15208 
15209   effect(DEF cr, USE op1, USE op2);
15210 
15211   ins_cost(INSN_COST);
15212   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15213 
15214   ins_encode(aarch64_enc_cmpw(op1, op2));
15215 
15216   ins_pipe(icmp_reg_reg);
15217 %}
15218 
15219 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15220 %{
15221   match(Set cr (CmpU op1 zero));
15222 
15223   effect(DEF cr, USE op1);
15224 
15225   ins_cost(INSN_COST);
15226   format %{ "cmpw $op1, #0\t# unsigned" %}
15227 
15228   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15229 
15230   ins_pipe(icmp_reg_imm);
15231 %}
15232 
15233 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15234 %{
15235   match(Set cr (CmpU op1 op2));
15236 
15237   effect(DEF cr, USE op1);
15238 
15239   ins_cost(INSN_COST);
15240   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15241 
15242   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15243 
15244   ins_pipe(icmp_reg_imm);
15245 %}
15246 
15247 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15248 %{
15249   match(Set cr (CmpU op1 op2));
15250 
15251   effect(DEF cr, USE op1);
15252 
15253   ins_cost(INSN_COST * 2);
15254   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15255 
15256   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15257 
15258   ins_pipe(icmp_reg_imm);
15259 %}
15260 
15261 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15262 %{
15263   match(Set cr (CmpL op1 op2));
15264 
15265   effect(DEF cr, USE op1, USE op2);
15266 
15267   ins_cost(INSN_COST);
15268   format %{ "cmp  $op1, $op2" %}
15269 
15270   ins_encode(aarch64_enc_cmp(op1, op2));
15271 
15272   ins_pipe(icmp_reg_reg);
15273 %}
15274 
15275 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15276 %{
15277   match(Set cr (CmpL op1 zero));
15278 
15279   effect(DEF cr, USE op1);
15280 
15281   ins_cost(INSN_COST);
15282   format %{ "tst  $op1" %}
15283 
15284   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15285 
15286   ins_pipe(icmp_reg_imm);
15287 %}
15288 
15289 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15290 %{
15291   match(Set cr (CmpL op1 op2));
15292 
15293   effect(DEF cr, USE op1);
15294 
15295   ins_cost(INSN_COST);
15296   format %{ "cmp  $op1, $op2" %}
15297 
15298   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15299 
15300   ins_pipe(icmp_reg_imm);
15301 %}
15302 
15303 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15304 %{
15305   match(Set cr (CmpL op1 op2));
15306 
15307   effect(DEF cr, USE op1);
15308 
15309   ins_cost(INSN_COST * 2);
15310   format %{ "cmp  $op1, $op2" %}
15311 
15312   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15313 
15314   ins_pipe(icmp_reg_imm);
15315 %}
15316 
15317 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15318 %{
15319   match(Set cr (CmpUL op1 op2));
15320 
15321   effect(DEF cr, USE op1, USE op2);
15322 
15323   ins_cost(INSN_COST);
15324   format %{ "cmp  $op1, $op2" %}
15325 
15326   ins_encode(aarch64_enc_cmp(op1, op2));
15327 
15328   ins_pipe(icmp_reg_reg);
15329 %}
15330 
15331 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15332 %{
15333   match(Set cr (CmpUL op1 zero));
15334 
15335   effect(DEF cr, USE op1);
15336 
15337   ins_cost(INSN_COST);
15338   format %{ "tst  $op1" %}
15339 
15340   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15341 
15342   ins_pipe(icmp_reg_imm);
15343 %}
15344 
15345 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15346 %{
15347   match(Set cr (CmpUL op1 op2));
15348 
15349   effect(DEF cr, USE op1);
15350 
15351   ins_cost(INSN_COST);
15352   format %{ "cmp  $op1, $op2" %}
15353 
15354   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15355 
15356   ins_pipe(icmp_reg_imm);
15357 %}
15358 
15359 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15360 %{
15361   match(Set cr (CmpUL op1 op2));
15362 
15363   effect(DEF cr, USE op1);
15364 
15365   ins_cost(INSN_COST * 2);
15366   format %{ "cmp  $op1, $op2" %}
15367 
15368   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15369 
15370   ins_pipe(icmp_reg_imm);
15371 %}
15372 
15373 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15374 %{
15375   match(Set cr (CmpP op1 op2));
15376 
15377   effect(DEF cr, USE op1, USE op2);
15378 
15379   ins_cost(INSN_COST);
15380   format %{ "cmp  $op1, $op2\t // ptr" %}
15381 
15382   ins_encode(aarch64_enc_cmpp(op1, op2));
15383 
15384   ins_pipe(icmp_reg_reg);
15385 %}
15386 
15387 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15388 %{
15389   match(Set cr (CmpN op1 op2));
15390 
15391   effect(DEF cr, USE op1, USE op2);
15392 
15393   ins_cost(INSN_COST);
15394   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15395 
15396   ins_encode(aarch64_enc_cmpn(op1, op2));
15397 
15398   ins_pipe(icmp_reg_reg);
15399 %}
15400 
15401 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15402 %{
15403   match(Set cr (CmpP op1 zero));
15404 
15405   effect(DEF cr, USE op1, USE zero);
15406 
15407   ins_cost(INSN_COST);
15408   format %{ "cmp  $op1, 0\t // ptr" %}
15409 
15410   ins_encode(aarch64_enc_testp(op1));
15411 
15412   ins_pipe(icmp_reg_imm);
15413 %}
15414 
15415 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15416 %{
15417   match(Set cr (CmpN op1 zero));
15418 
15419   effect(DEF cr, USE op1, USE zero);
15420 
15421   ins_cost(INSN_COST);
15422   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15423 
15424   ins_encode(aarch64_enc_testn(op1));
15425 
15426   ins_pipe(icmp_reg_imm);
15427 %}
15428 
15429 // FP comparisons
15430 //
15431 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15432 // using normal cmpOp. See declaration of rFlagsReg for details.
15433 
15434 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15435 %{
15436   match(Set cr (CmpF src1 src2));
15437 
15438   ins_cost(3 * INSN_COST);
15439   format %{ "fcmps $src1, $src2" %}
15440 
15441   ins_encode %{
15442     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15443   %}
15444 
15445   ins_pipe(pipe_class_compare);
15446 %}
15447 
15448 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15449 %{
15450   match(Set cr (CmpF src1 src2));
15451 
15452   ins_cost(3 * INSN_COST);
15453   format %{ "fcmps $src1, 0.0" %}
15454 
15455   ins_encode %{
15456     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15457   %}
15458 
15459   ins_pipe(pipe_class_compare);
15460 %}
15461 // FROM HERE
15462 
15463 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15464 %{
15465   match(Set cr (CmpD src1 src2));
15466 
15467   ins_cost(3 * INSN_COST);
15468   format %{ "fcmpd $src1, $src2" %}
15469 
15470   ins_encode %{
15471     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15472   %}
15473 
15474   ins_pipe(pipe_class_compare);
15475 %}
15476 
15477 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15478 %{
15479   match(Set cr (CmpD src1 src2));
15480 
15481   ins_cost(3 * INSN_COST);
15482   format %{ "fcmpd $src1, 0.0" %}
15483 
15484   ins_encode %{
15485     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15486   %}
15487 
15488   ins_pipe(pipe_class_compare);
15489 %}
15490 
15491 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15492 %{
15493   match(Set dst (CmpF3 src1 src2));
15494   effect(KILL cr);
15495 
15496   ins_cost(5 * INSN_COST);
15497   format %{ "fcmps $src1, $src2\n\t"
15498             "csinvw($dst, zr, zr, eq\n\t"
15499             "csnegw($dst, $dst, $dst, lt)"
15500   %}
15501 
15502   ins_encode %{
15503     Label done;
15504     FloatRegister s1 = as_FloatRegister($src1$$reg);
15505     FloatRegister s2 = as_FloatRegister($src2$$reg);
15506     Register d = as_Register($dst$$reg);
15507     __ fcmps(s1, s2);
15508     // installs 0 if EQ else -1
15509     __ csinvw(d, zr, zr, Assembler::EQ);
15510     // keeps -1 if less or unordered else installs 1
15511     __ csnegw(d, d, d, Assembler::LT);
15512     __ bind(done);
15513   %}
15514 
15515   ins_pipe(pipe_class_default);
15516 
15517 %}
15518 
15519 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15520 %{
15521   match(Set dst (CmpD3 src1 src2));
15522   effect(KILL cr);
15523 
15524   ins_cost(5 * INSN_COST);
15525   format %{ "fcmpd $src1, $src2\n\t"
15526             "csinvw($dst, zr, zr, eq\n\t"
15527             "csnegw($dst, $dst, $dst, lt)"
15528   %}
15529 
15530   ins_encode %{
15531     Label done;
15532     FloatRegister s1 = as_FloatRegister($src1$$reg);
15533     FloatRegister s2 = as_FloatRegister($src2$$reg);
15534     Register d = as_Register($dst$$reg);
15535     __ fcmpd(s1, s2);
15536     // installs 0 if EQ else -1
15537     __ csinvw(d, zr, zr, Assembler::EQ);
15538     // keeps -1 if less or unordered else installs 1
15539     __ csnegw(d, d, d, Assembler::LT);
15540     __ bind(done);
15541   %}
15542   ins_pipe(pipe_class_default);
15543 
15544 %}
15545 
15546 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15547 %{
15548   match(Set dst (CmpF3 src1 zero));
15549   effect(KILL cr);
15550 
15551   ins_cost(5 * INSN_COST);
15552   format %{ "fcmps $src1, 0.0\n\t"
15553             "csinvw($dst, zr, zr, eq\n\t"
15554             "csnegw($dst, $dst, $dst, lt)"
15555   %}
15556 
15557   ins_encode %{
15558     Label done;
15559     FloatRegister s1 = as_FloatRegister($src1$$reg);
15560     Register d = as_Register($dst$$reg);
15561     __ fcmps(s1, 0.0);
15562     // installs 0 if EQ else -1
15563     __ csinvw(d, zr, zr, Assembler::EQ);
15564     // keeps -1 if less or unordered else installs 1
15565     __ csnegw(d, d, d, Assembler::LT);
15566     __ bind(done);
15567   %}
15568 
15569   ins_pipe(pipe_class_default);
15570 
15571 %}
15572 
15573 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15574 %{
15575   match(Set dst (CmpD3 src1 zero));
15576   effect(KILL cr);
15577 
15578   ins_cost(5 * INSN_COST);
15579   format %{ "fcmpd $src1, 0.0\n\t"
15580             "csinvw($dst, zr, zr, eq\n\t"
15581             "csnegw($dst, $dst, $dst, lt)"
15582   %}
15583 
15584   ins_encode %{
15585     Label done;
15586     FloatRegister s1 = as_FloatRegister($src1$$reg);
15587     Register d = as_Register($dst$$reg);
15588     __ fcmpd(s1, 0.0);
15589     // installs 0 if EQ else -1
15590     __ csinvw(d, zr, zr, Assembler::EQ);
15591     // keeps -1 if less or unordered else installs 1
15592     __ csnegw(d, d, d, Assembler::LT);
15593     __ bind(done);
15594   %}
15595   ins_pipe(pipe_class_default);
15596 
15597 %}
15598 
15599 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15600 %{
15601   match(Set dst (CmpLTMask p q));
15602   effect(KILL cr);
15603 
15604   ins_cost(3 * INSN_COST);
15605 
15606   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15607             "csetw $dst, lt\n\t"
15608             "subw $dst, zr, $dst"
15609   %}
15610 
15611   ins_encode %{
15612     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15613     __ csetw(as_Register($dst$$reg), Assembler::LT);
15614     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15615   %}
15616 
15617   ins_pipe(ialu_reg_reg);
15618 %}
15619 
15620 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15621 %{
15622   match(Set dst (CmpLTMask src zero));
15623   effect(KILL cr);
15624 
15625   ins_cost(INSN_COST);
15626 
15627   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15628 
15629   ins_encode %{
15630     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15631   %}
15632 
15633   ins_pipe(ialu_reg_shift);
15634 %}
15635 
15636 // ============================================================================
15637 // Max and Min
15638 
15639 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15640 %{
15641   effect( DEF dst, USE src1, USE src2, USE cr );
15642 
15643   ins_cost(INSN_COST * 2);
15644   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
15645 
15646   ins_encode %{
15647     __ cselw(as_Register($dst$$reg),
15648              as_Register($src1$$reg),
15649              as_Register($src2$$reg),
15650              Assembler::LT);
15651   %}
15652 
15653   ins_pipe(icond_reg_reg);
15654 %}
15655 
15656 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15657 %{
15658   match(Set dst (MinI src1 src2));
15659   ins_cost(INSN_COST * 3);
15660 
15661   expand %{
15662     rFlagsReg cr;
15663     compI_reg_reg(cr, src1, src2);
15664     cmovI_reg_reg_lt(dst, src1, src2, cr);
15665   %}
15666 
15667 %}
15668 // FROM HERE
15669 
15670 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15671 %{
15672   effect( DEF dst, USE src1, USE src2, USE cr );
15673 
15674   ins_cost(INSN_COST * 2);
15675   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
15676 
15677   ins_encode %{
15678     __ cselw(as_Register($dst$$reg),
15679              as_Register($src1$$reg),
15680              as_Register($src2$$reg),
15681              Assembler::GT);
15682   %}
15683 
15684   ins_pipe(icond_reg_reg);
15685 %}
15686 
15687 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15688 %{
15689   match(Set dst (MaxI src1 src2));
15690   ins_cost(INSN_COST * 3);
15691   expand %{
15692     rFlagsReg cr;
15693     compI_reg_reg(cr, src1, src2);
15694     cmovI_reg_reg_gt(dst, src1, src2, cr);
15695   %}
15696 %}
15697 
15698 // ============================================================================
15699 // Branch Instructions
15700 
15701 // Direct Branch.
15702 instruct branch(label lbl)
15703 %{
15704   match(Goto);
15705 
15706   effect(USE lbl);
15707 
15708   ins_cost(BRANCH_COST);
15709   format %{ "b  $lbl" %}
15710 
15711   ins_encode(aarch64_enc_b(lbl));
15712 
15713   ins_pipe(pipe_branch);
15714 %}
15715 
15716 // Conditional Near Branch
15717 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15718 %{
15719   // Same match rule as `branchConFar'.
15720   match(If cmp cr);
15721 
15722   effect(USE lbl);
15723 
15724   ins_cost(BRANCH_COST);
15725   // If set to 1 this indicates that the current instruction is a
15726   // short variant of a long branch. This avoids using this
15727   // instruction in first-pass matching. It will then only be used in
15728   // the `Shorten_branches' pass.
15729   // ins_short_branch(1);
15730   format %{ "b$cmp  $lbl" %}
15731 
15732   ins_encode(aarch64_enc_br_con(cmp, lbl));
15733 
15734   ins_pipe(pipe_branch_cond);
15735 %}
15736 
15737 // Conditional Near Branch Unsigned
15738 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15739 %{
15740   // Same match rule as `branchConFar'.
15741   match(If cmp cr);
15742 
15743   effect(USE lbl);
15744 
15745   ins_cost(BRANCH_COST);
15746   // If set to 1 this indicates that the current instruction is a
15747   // short variant of a long branch. This avoids using this
15748   // instruction in first-pass matching. It will then only be used in
15749   // the `Shorten_branches' pass.
15750   // ins_short_branch(1);
15751   format %{ "b$cmp  $lbl\t# unsigned" %}
15752 
15753   ins_encode(aarch64_enc_br_conU(cmp, lbl));
15754 
15755   ins_pipe(pipe_branch_cond);
15756 %}
15757 
15758 // Make use of CBZ and CBNZ.  These instructions, as well as being
15759 // shorter than (cmp; branch), have the additional benefit of not
15760 // killing the flags.
15761 
15762 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15763   match(If cmp (CmpI op1 op2));
15764   effect(USE labl);
15765 
15766   ins_cost(BRANCH_COST);
15767   format %{ "cbw$cmp   $op1, $labl" %}
15768   ins_encode %{
15769     Label* L = $labl$$label;
15770     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15771     if (cond == Assembler::EQ)
15772       __ cbzw($op1$$Register, *L);
15773     else
15774       __ cbnzw($op1$$Register, *L);
15775   %}
15776   ins_pipe(pipe_cmp_branch);
15777 %}
15778 
15779 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15780   match(If cmp (CmpL op1 op2));
15781   effect(USE labl);
15782 
15783   ins_cost(BRANCH_COST);
15784   format %{ "cb$cmp   $op1, $labl" %}
15785   ins_encode %{
15786     Label* L = $labl$$label;
15787     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15788     if (cond == Assembler::EQ)
15789       __ cbz($op1$$Register, *L);
15790     else
15791       __ cbnz($op1$$Register, *L);
15792   %}
15793   ins_pipe(pipe_cmp_branch);
15794 %}
15795 
15796 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15797   match(If cmp (CmpP op1 op2));
15798   effect(USE labl);
15799 
15800   ins_cost(BRANCH_COST);
15801   format %{ "cb$cmp   $op1, $labl" %}
15802   ins_encode %{
15803     Label* L = $labl$$label;
15804     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15805     if (cond == Assembler::EQ)
15806       __ cbz($op1$$Register, *L);
15807     else
15808       __ cbnz($op1$$Register, *L);
15809   %}
15810   ins_pipe(pipe_cmp_branch);
15811 %}
15812 
15813 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15814   match(If cmp (CmpN op1 op2));
15815   effect(USE labl);
15816 
15817   ins_cost(BRANCH_COST);
15818   format %{ "cbw$cmp   $op1, $labl" %}
15819   ins_encode %{
15820     Label* L = $labl$$label;
15821     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15822     if (cond == Assembler::EQ)
15823       __ cbzw($op1$$Register, *L);
15824     else
15825       __ cbnzw($op1$$Register, *L);
15826   %}
15827   ins_pipe(pipe_cmp_branch);
15828 %}
15829 
15830 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15831   match(If cmp (CmpP (DecodeN oop) zero));
15832   effect(USE labl);
15833 
15834   ins_cost(BRANCH_COST);
15835   format %{ "cb$cmp   $oop, $labl" %}
15836   ins_encode %{
15837     Label* L = $labl$$label;
15838     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15839     if (cond == Assembler::EQ)
15840       __ cbzw($oop$$Register, *L);
15841     else
15842       __ cbnzw($oop$$Register, *L);
15843   %}
15844   ins_pipe(pipe_cmp_branch);
15845 %}
15846 
15847 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
15848   match(If cmp (CmpU op1 op2));
15849   effect(USE labl);
15850 
15851   ins_cost(BRANCH_COST);
15852   format %{ "cbw$cmp   $op1, $labl" %}
15853   ins_encode %{
15854     Label* L = $labl$$label;
15855     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15856     if (cond == Assembler::EQ || cond == Assembler::LS)
15857       __ cbzw($op1$$Register, *L);
15858     else
15859       __ cbnzw($op1$$Register, *L);
15860   %}
15861   ins_pipe(pipe_cmp_branch);
15862 %}
15863 
15864 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
15865   match(If cmp (CmpUL op1 op2));
15866   effect(USE labl);
15867 
15868   ins_cost(BRANCH_COST);
15869   format %{ "cb$cmp   $op1, $labl" %}
15870   ins_encode %{
15871     Label* L = $labl$$label;
15872     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15873     if (cond == Assembler::EQ || cond == Assembler::LS)
15874       __ cbz($op1$$Register, *L);
15875     else
15876       __ cbnz($op1$$Register, *L);
15877   %}
15878   ins_pipe(pipe_cmp_branch);
15879 %}
15880 
15881 // Test bit and Branch
15882 
15883 // Patterns for short (< 32KiB) variants
15884 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15885   match(If cmp (CmpL op1 op2));
15886   effect(USE labl);
15887 
15888   ins_cost(BRANCH_COST);
15889   format %{ "cb$cmp   $op1, $labl # long" %}
15890   ins_encode %{
15891     Label* L = $labl$$label;
15892     Assembler::Condition cond =
15893       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15894     __ tbr(cond, $op1$$Register, 63, *L);
15895   %}
15896   ins_pipe(pipe_cmp_branch);
15897   ins_short_branch(1);
15898 %}
15899 
15900 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15901   match(If cmp (CmpI op1 op2));
15902   effect(USE labl);
15903 
15904   ins_cost(BRANCH_COST);
15905   format %{ "cb$cmp   $op1, $labl # int" %}
15906   ins_encode %{
15907     Label* L = $labl$$label;
15908     Assembler::Condition cond =
15909       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15910     __ tbr(cond, $op1$$Register, 31, *L);
15911   %}
15912   ins_pipe(pipe_cmp_branch);
15913   ins_short_branch(1);
15914 %}
15915 
15916 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15917   match(If cmp (CmpL (AndL op1 op2) op3));
15918   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15919   effect(USE labl);
15920 
15921   ins_cost(BRANCH_COST);
15922   format %{ "tb$cmp   $op1, $op2, $labl" %}
15923   ins_encode %{
15924     Label* L = $labl$$label;
15925     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15926     int bit = exact_log2_long($op2$$constant);
15927     __ tbr(cond, $op1$$Register, bit, *L);
15928   %}
15929   ins_pipe(pipe_cmp_branch);
15930   ins_short_branch(1);
15931 %}
15932 
15933 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15934   match(If cmp (CmpI (AndI op1 op2) op3));
15935   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15936   effect(USE labl);
15937 
15938   ins_cost(BRANCH_COST);
15939   format %{ "tb$cmp   $op1, $op2, $labl" %}
15940   ins_encode %{
15941     Label* L = $labl$$label;
15942     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15943     int bit = exact_log2((juint)$op2$$constant);
15944     __ tbr(cond, $op1$$Register, bit, *L);
15945   %}
15946   ins_pipe(pipe_cmp_branch);
15947   ins_short_branch(1);
15948 %}
15949 
15950 // And far variants
15951 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15952   match(If cmp (CmpL op1 op2));
15953   effect(USE labl);
15954 
15955   ins_cost(BRANCH_COST);
15956   format %{ "cb$cmp   $op1, $labl # long" %}
15957   ins_encode %{
15958     Label* L = $labl$$label;
15959     Assembler::Condition cond =
15960       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15961     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15962   %}
15963   ins_pipe(pipe_cmp_branch);
15964 %}
15965 
15966 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15967   match(If cmp (CmpI op1 op2));
15968   effect(USE labl);
15969 
15970   ins_cost(BRANCH_COST);
15971   format %{ "cb$cmp   $op1, $labl # int" %}
15972   ins_encode %{
15973     Label* L = $labl$$label;
15974     Assembler::Condition cond =
15975       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15976     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15977   %}
15978   ins_pipe(pipe_cmp_branch);
15979 %}
15980 
15981 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15982   match(If cmp (CmpL (AndL op1 op2) op3));
15983   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15984   effect(USE labl);
15985 
15986   ins_cost(BRANCH_COST);
15987   format %{ "tb$cmp   $op1, $op2, $labl" %}
15988   ins_encode %{
15989     Label* L = $labl$$label;
15990     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15991     int bit = exact_log2_long($op2$$constant);
15992     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15993   %}
15994   ins_pipe(pipe_cmp_branch);
15995 %}
15996 
15997 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15998   match(If cmp (CmpI (AndI op1 op2) op3));
15999   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16000   effect(USE labl);
16001 
16002   ins_cost(BRANCH_COST);
16003   format %{ "tb$cmp   $op1, $op2, $labl" %}
16004   ins_encode %{
16005     Label* L = $labl$$label;
16006     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16007     int bit = exact_log2((juint)$op2$$constant);
16008     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16009   %}
16010   ins_pipe(pipe_cmp_branch);
16011 %}
16012 
16013 // Test bits
16014 
16015 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16016   match(Set cr (CmpL (AndL op1 op2) op3));
16017   predicate(Assembler::operand_valid_for_logical_immediate
16018             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16019 
16020   ins_cost(INSN_COST);
16021   format %{ "tst $op1, $op2 # long" %}
16022   ins_encode %{
16023     __ tst($op1$$Register, $op2$$constant);
16024   %}
16025   ins_pipe(ialu_reg_reg);
16026 %}
16027 
16028 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16029   match(Set cr (CmpI (AndI op1 op2) op3));
16030   predicate(Assembler::operand_valid_for_logical_immediate
16031             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16032 
16033   ins_cost(INSN_COST);
16034   format %{ "tst $op1, $op2 # int" %}
16035   ins_encode %{
16036     __ tstw($op1$$Register, $op2$$constant);
16037   %}
16038   ins_pipe(ialu_reg_reg);
16039 %}
16040 
16041 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16042   match(Set cr (CmpL (AndL op1 op2) op3));
16043 
16044   ins_cost(INSN_COST);
16045   format %{ "tst $op1, $op2 # long" %}
16046   ins_encode %{
16047     __ tst($op1$$Register, $op2$$Register);
16048   %}
16049   ins_pipe(ialu_reg_reg);
16050 %}
16051 
16052 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16053   match(Set cr (CmpI (AndI op1 op2) op3));
16054 
16055   ins_cost(INSN_COST);
16056   format %{ "tstw $op1, $op2 # int" %}
16057   ins_encode %{
16058     __ tstw($op1$$Register, $op2$$Register);
16059   %}
16060   ins_pipe(ialu_reg_reg);
16061 %}
16062 
16063 
16064 // Conditional Far Branch
16065 // Conditional Far Branch Unsigned
16066 // TODO: fixme
16067 
16068 // counted loop end branch near
16069 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16070 %{
16071   match(CountedLoopEnd cmp cr);
16072 
16073   effect(USE lbl);
16074 
16075   ins_cost(BRANCH_COST);
16076   // short variant.
16077   // ins_short_branch(1);
16078   format %{ "b$cmp $lbl \t// counted loop end" %}
16079 
16080   ins_encode(aarch64_enc_br_con(cmp, lbl));
16081 
16082   ins_pipe(pipe_branch);
16083 %}
16084 
16085 // counted loop end branch near Unsigned
16086 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16087 %{
16088   match(CountedLoopEnd cmp cr);
16089 
16090   effect(USE lbl);
16091 
16092   ins_cost(BRANCH_COST);
16093   // short variant.
16094   // ins_short_branch(1);
16095   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16096 
16097   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16098 
16099   ins_pipe(pipe_branch);
16100 %}
16101 
16102 // counted loop end branch far
16103 // counted loop end branch far unsigned
16104 // TODO: fixme
16105 
16106 // ============================================================================
16107 // inlined locking and unlocking
16108 
16109 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16110 %{
16111   match(Set cr (FastLock object));
16112   effect(TEMP tmp, TEMP tmp2, TEMP box);
16113 
16114   // TODO
16115   // identify correct cost
16116   ins_cost(5 * INSN_COST);
16117   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16118 
16119   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16120 
16121   ins_pipe(pipe_serial);
16122 %}
16123 
16124 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16125 %{
16126   match(Set cr (FastUnlock object));
16127   effect(TEMP box, TEMP tmp, TEMP tmp2);
16128 
16129   ins_cost(5 * INSN_COST);
16130   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16131 
16132   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16133 
16134   ins_pipe(pipe_serial);
16135 %}
16136 
16137 
16138 // ============================================================================
16139 // Safepoint Instructions
16140 
16141 // TODO
16142 // provide a near and far version of this code
16143 
16144 instruct safePoint(rFlagsReg cr, iRegP poll)
16145 %{
16146   match(SafePoint poll);
16147   effect(KILL cr);
16148 
16149   format %{
16150     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16151   %}
16152   ins_encode %{
16153     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16154   %}
16155   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16156 %}
16157 
16158 
16159 // ============================================================================
16160 // Procedure Call/Return Instructions
16161 
16162 // Call Java Static Instruction
16163 
16164 instruct CallStaticJavaDirect(method meth)
16165 %{
16166   match(CallStaticJava);
16167 
16168   effect(USE meth);
16169 
16170   ins_cost(CALL_COST);
16171 
16172   format %{ "call,static $meth \t// ==> " %}
16173 
16174   ins_encode(aarch64_enc_java_static_call(meth),
16175              aarch64_enc_call_epilog);
16176 
16177   ins_pipe(pipe_class_call);
16178 %}
16179 
16180 // TO HERE
16181 
16182 // Call Java Dynamic Instruction
16183 instruct CallDynamicJavaDirect(method meth)
16184 %{
16185   match(CallDynamicJava);
16186 
16187   effect(USE meth);
16188 
16189   ins_cost(CALL_COST);
16190 
16191   format %{ "CALL,dynamic $meth \t// ==> " %}
16192 
16193   ins_encode(aarch64_enc_java_dynamic_call(meth),
16194              aarch64_enc_call_epilog);
16195 
16196   ins_pipe(pipe_class_call);
16197 %}
16198 
16199 // Call Runtime Instruction
16200 
16201 instruct CallRuntimeDirect(method meth)
16202 %{
16203   match(CallRuntime);
16204 
16205   effect(USE meth);
16206 
16207   ins_cost(CALL_COST);
16208 
16209   format %{ "CALL, runtime $meth" %}
16210 
16211   ins_encode( aarch64_enc_java_to_runtime(meth) );
16212 
16213   ins_pipe(pipe_class_call);
16214 %}
16215 
16216 // Call Runtime Instruction
16217 
16218 instruct CallLeafDirect(method meth)
16219 %{
16220   match(CallLeaf);
16221 
16222   effect(USE meth);
16223 
16224   ins_cost(CALL_COST);
16225 
16226   format %{ "CALL, runtime leaf $meth" %}
16227 
16228   ins_encode( aarch64_enc_java_to_runtime(meth) );
16229 
16230   ins_pipe(pipe_class_call);
16231 %}
16232 
16233 // Call Runtime Instruction
16234 
16235 instruct CallLeafNoFPDirect(method meth)
16236 %{
16237   match(CallLeafNoFP);
16238 
16239   effect(USE meth);
16240 
16241   ins_cost(CALL_COST);
16242 
16243   format %{ "CALL, runtime leaf nofp $meth" %}
16244 
16245   ins_encode( aarch64_enc_java_to_runtime(meth) );
16246 
16247   ins_pipe(pipe_class_call);
16248 %}
16249 
16250 // Tail Call; Jump from runtime stub to Java code.
16251 // Also known as an 'interprocedural jump'.
16252 // Target of jump will eventually return to caller.
16253 // TailJump below removes the return address.
16254 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16255 %{
16256   match(TailCall jump_target method_ptr);
16257 
16258   ins_cost(CALL_COST);
16259 
16260   format %{ "br $jump_target\t# $method_ptr holds method" %}
16261 
16262   ins_encode(aarch64_enc_tail_call(jump_target));
16263 
16264   ins_pipe(pipe_class_call);
16265 %}
16266 
16267 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16268 %{
16269   match(TailJump jump_target ex_oop);
16270 
16271   ins_cost(CALL_COST);
16272 
16273   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16274 
16275   ins_encode(aarch64_enc_tail_jmp(jump_target));
16276 
16277   ins_pipe(pipe_class_call);
16278 %}
16279 
16280 // Create exception oop: created by stack-crawling runtime code.
16281 // Created exception is now available to this handler, and is setup
16282 // just prior to jumping to this handler. No code emitted.
16283 // TODO check
16284 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16285 instruct CreateException(iRegP_R0 ex_oop)
16286 %{
16287   match(Set ex_oop (CreateEx));
16288 
16289   format %{ " -- \t// exception oop; no code emitted" %}
16290 
16291   size(0);
16292 
16293   ins_encode( /*empty*/ );
16294 
16295   ins_pipe(pipe_class_empty);
16296 %}
16297 
16298 // Rethrow exception: The exception oop will come in the first
16299 // argument position. Then JUMP (not call) to the rethrow stub code.
16300 instruct RethrowException() %{
16301   match(Rethrow);
16302   ins_cost(CALL_COST);
16303 
16304   format %{ "b rethrow_stub" %}
16305 
16306   ins_encode( aarch64_enc_rethrow() );
16307 
16308   ins_pipe(pipe_class_call);
16309 %}
16310 
16311 
16312 // Return Instruction
16313 // epilog node loads ret address into lr as part of frame pop
16314 instruct Ret()
16315 %{
16316   match(Return);
16317 
16318   format %{ "ret\t// return register" %}
16319 
16320   ins_encode( aarch64_enc_ret() );
16321 
16322   ins_pipe(pipe_branch);
16323 %}
16324 
16325 // Die now.
16326 instruct ShouldNotReachHere() %{
16327   match(Halt);
16328 
16329   ins_cost(CALL_COST);
16330   format %{ "ShouldNotReachHere" %}
16331 
16332   ins_encode %{
16333     if (is_reachable()) {
16334       __ stop(_halt_reason);
16335     }
16336   %}
16337 
16338   ins_pipe(pipe_class_default);
16339 %}
16340 
16341 // ============================================================================
16342 // Partial Subtype Check
16343 //
16344 // superklass array for an instance of the superklass.  Set a hidden
16345 // internal cache on a hit (cache is checked with exposed code in
16346 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16347 // encoding ALSO sets flags.
16348 
16349 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16350 %{
16351   match(Set result (PartialSubtypeCheck sub super));
16352   effect(KILL cr, KILL temp);
16353 
16354   ins_cost(1100);  // slightly larger than the next version
16355   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16356 
16357   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16358 
16359   opcode(0x1); // Force zero of result reg on hit
16360 
16361   ins_pipe(pipe_class_memory);
16362 %}
16363 
16364 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16365 %{
16366   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16367   effect(KILL temp, KILL result);
16368 
16369   ins_cost(1100);  // slightly larger than the next version
16370   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16371 
16372   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16373 
16374   opcode(0x0); // Don't zero result reg on hit
16375 
16376   ins_pipe(pipe_class_memory);
16377 %}
16378 
16379 // Intrisics for String.compareTo()
16380 
16381 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16382                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16383 %{
16384   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16385   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16386   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16387 
16388   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16389   ins_encode %{
16390     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16391     __ string_compare($str1$$Register, $str2$$Register,
16392                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16393                       $tmp1$$Register, $tmp2$$Register,
16394                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16395   %}
16396   ins_pipe(pipe_class_memory);
16397 %}
16398 
16399 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16400                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16401 %{
16402   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16403   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16404   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16405 
16406   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16407   ins_encode %{
16408     __ string_compare($str1$$Register, $str2$$Register,
16409                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16410                       $tmp1$$Register, $tmp2$$Register,
16411                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16412   %}
16413   ins_pipe(pipe_class_memory);
16414 %}
16415 
16416 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16417                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16418                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16419 %{
16420   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16421   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16422   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16423          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16424 
16425   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16426   ins_encode %{
16427     __ string_compare($str1$$Register, $str2$$Register,
16428                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16429                       $tmp1$$Register, $tmp2$$Register,
16430                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16431                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16432   %}
16433   ins_pipe(pipe_class_memory);
16434 %}
16435 
16436 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16437                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16438                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16439 %{
16440   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16441   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16442   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16443          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16444 
16445   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16446   ins_encode %{
16447     __ string_compare($str1$$Register, $str2$$Register,
16448                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16449                       $tmp1$$Register, $tmp2$$Register,
16450                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16451                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16452   %}
16453   ins_pipe(pipe_class_memory);
16454 %}
16455 
16456 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16457 // these string_compare variants as NEON register type for convenience so that the prototype of
16458 // string_compare can be shared with all variants.
16459 
16460 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16461                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16462                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16463                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16464 %{
16465   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16466   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16467   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16468          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16469 
16470   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16471   ins_encode %{
16472     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16473     __ string_compare($str1$$Register, $str2$$Register,
16474                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16475                       $tmp1$$Register, $tmp2$$Register,
16476                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16477                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16478                       StrIntrinsicNode::LL);
16479   %}
16480   ins_pipe(pipe_class_memory);
16481 %}
16482 
16483 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16484                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16485                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16486                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16487 %{
16488   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16489   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16490   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16491          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16492 
16493   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16494   ins_encode %{
16495     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16496     __ string_compare($str1$$Register, $str2$$Register,
16497                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16498                       $tmp1$$Register, $tmp2$$Register,
16499                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16500                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16501                       StrIntrinsicNode::LU);
16502   %}
16503   ins_pipe(pipe_class_memory);
16504 %}
16505 
16506 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16507                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16508                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16509                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16510 %{
16511   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16512   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16513   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16514          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16515 
16516   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16517   ins_encode %{
16518     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16519     __ string_compare($str1$$Register, $str2$$Register,
16520                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16521                       $tmp1$$Register, $tmp2$$Register,
16522                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16523                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16524                       StrIntrinsicNode::UL);
16525   %}
16526   ins_pipe(pipe_class_memory);
16527 %}
16528 
16529 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16530                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16531                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16532                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16533 %{
16534   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16535   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16536   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16537          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16538 
16539   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16540   ins_encode %{
16541     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16542     __ string_compare($str1$$Register, $str2$$Register,
16543                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16544                       $tmp1$$Register, $tmp2$$Register,
16545                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16546                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16547                       StrIntrinsicNode::UU);
16548   %}
16549   ins_pipe(pipe_class_memory);
16550 %}
16551 
16552 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16553        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16554        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16555 %{
16556   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16557   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16558   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16559          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16560   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16561 
16562   ins_encode %{
16563     __ string_indexof($str1$$Register, $str2$$Register,
16564                       $cnt1$$Register, $cnt2$$Register,
16565                       $tmp1$$Register, $tmp2$$Register,
16566                       $tmp3$$Register, $tmp4$$Register,
16567                       $tmp5$$Register, $tmp6$$Register,
16568                       -1, $result$$Register, StrIntrinsicNode::UU);
16569   %}
16570   ins_pipe(pipe_class_memory);
16571 %}
16572 
16573 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16574        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16575        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16576 %{
16577   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16578   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16579   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16580          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16581   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16582 
16583   ins_encode %{
16584     __ string_indexof($str1$$Register, $str2$$Register,
16585                       $cnt1$$Register, $cnt2$$Register,
16586                       $tmp1$$Register, $tmp2$$Register,
16587                       $tmp3$$Register, $tmp4$$Register,
16588                       $tmp5$$Register, $tmp6$$Register,
16589                       -1, $result$$Register, StrIntrinsicNode::LL);
16590   %}
16591   ins_pipe(pipe_class_memory);
16592 %}
16593 
16594 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16595        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16596        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16597 %{
16598   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16599   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16600   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16601          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16602   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
16603 
16604   ins_encode %{
16605     __ string_indexof($str1$$Register, $str2$$Register,
16606                       $cnt1$$Register, $cnt2$$Register,
16607                       $tmp1$$Register, $tmp2$$Register,
16608                       $tmp3$$Register, $tmp4$$Register,
16609                       $tmp5$$Register, $tmp6$$Register,
16610                       -1, $result$$Register, StrIntrinsicNode::UL);
16611   %}
16612   ins_pipe(pipe_class_memory);
16613 %}
16614 
16615 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16616                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16617                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16618 %{
16619   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16620   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16621   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16622          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16623   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
16624 
16625   ins_encode %{
16626     int icnt2 = (int)$int_cnt2$$constant;
16627     __ string_indexof($str1$$Register, $str2$$Register,
16628                       $cnt1$$Register, zr,
16629                       $tmp1$$Register, $tmp2$$Register,
16630                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16631                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16632   %}
16633   ins_pipe(pipe_class_memory);
16634 %}
16635 
16636 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16637                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16638                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16639 %{
16640   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16641   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16642   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16643          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16644   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
16645 
16646   ins_encode %{
16647     int icnt2 = (int)$int_cnt2$$constant;
16648     __ string_indexof($str1$$Register, $str2$$Register,
16649                       $cnt1$$Register, zr,
16650                       $tmp1$$Register, $tmp2$$Register,
16651                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16652                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16653   %}
16654   ins_pipe(pipe_class_memory);
16655 %}
16656 
16657 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16658                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16659                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16660 %{
16661   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16662   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16663   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16664          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16665   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
16666 
16667   ins_encode %{
16668     int icnt2 = (int)$int_cnt2$$constant;
16669     __ string_indexof($str1$$Register, $str2$$Register,
16670                       $cnt1$$Register, zr,
16671                       $tmp1$$Register, $tmp2$$Register,
16672                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16673                       icnt2, $result$$Register, StrIntrinsicNode::UL);
16674   %}
16675   ins_pipe(pipe_class_memory);
16676 %}
16677 
16678 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16679                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16680                              iRegINoSp tmp3, rFlagsReg cr)
16681 %{
16682   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16683   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16684   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16685          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16686 
16687   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16688 
16689   ins_encode %{
16690     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16691                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16692                            $tmp3$$Register);
16693   %}
16694   ins_pipe(pipe_class_memory);
16695 %}
16696 
16697 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16698                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16699                               iRegINoSp tmp3, rFlagsReg cr)
16700 %{
16701   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16702   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16703   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16704          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16705 
16706   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16707 
16708   ins_encode %{
16709     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16710                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
16711                             $tmp3$$Register);
16712   %}
16713   ins_pipe(pipe_class_memory);
16714 %}
16715 
16716 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16717                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16718                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16719   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16720   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16721   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16722   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16723   ins_encode %{
16724     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16725                                $result$$Register, $ztmp1$$FloatRegister,
16726                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16727                                $ptmp$$PRegister, true /* isL */);
16728   %}
16729   ins_pipe(pipe_class_memory);
16730 %}
16731 
16732 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16733                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16734                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16735   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16736   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16737   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16738   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16739   ins_encode %{
16740     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16741                                $result$$Register, $ztmp1$$FloatRegister,
16742                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16743                                $ptmp$$PRegister, false /* isL */);
16744   %}
16745   ins_pipe(pipe_class_memory);
16746 %}
16747 
16748 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16749                         iRegI_R0 result, rFlagsReg cr)
16750 %{
16751   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16752   match(Set result (StrEquals (Binary str1 str2) cnt));
16753   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16754 
16755   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16756   ins_encode %{
16757     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16758     __ string_equals($str1$$Register, $str2$$Register,
16759                      $result$$Register, $cnt$$Register, 1);
16760   %}
16761   ins_pipe(pipe_class_memory);
16762 %}
16763 
16764 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16765                         iRegI_R0 result, rFlagsReg cr)
16766 %{
16767   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
16768   match(Set result (StrEquals (Binary str1 str2) cnt));
16769   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16770 
16771   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16772   ins_encode %{
16773     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16774     __ string_equals($str1$$Register, $str2$$Register,
16775                      $result$$Register, $cnt$$Register, 2);
16776   %}
16777   ins_pipe(pipe_class_memory);
16778 %}
16779 
16780 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16781                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16782                        iRegP_R10 tmp, rFlagsReg cr)
16783 %{
16784   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16785   match(Set result (AryEq ary1 ary2));
16786   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16787 
16788   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16789   ins_encode %{
16790     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16791                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16792                                    $result$$Register, $tmp$$Register, 1);
16793     if (tpc == NULL) {
16794       ciEnv::current()->record_failure("CodeCache is full");
16795       return;
16796     }
16797   %}
16798   ins_pipe(pipe_class_memory);
16799 %}
16800 
16801 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16802                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16803                        iRegP_R10 tmp, rFlagsReg cr)
16804 %{
16805   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16806   match(Set result (AryEq ary1 ary2));
16807   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16808 
16809   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16810   ins_encode %{
16811     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16812                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16813                                    $result$$Register, $tmp$$Register, 2);
16814     if (tpc == NULL) {
16815       ciEnv::current()->record_failure("CodeCache is full");
16816       return;
16817     }
16818   %}
16819   ins_pipe(pipe_class_memory);
16820 %}
16821 
16822 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16823 %{
16824   match(Set result (CountPositives ary1 len));
16825   effect(USE_KILL ary1, USE_KILL len, KILL cr);
16826   format %{ "count positives byte[] $ary1,$len -> $result" %}
16827   ins_encode %{
16828     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16829     if (tpc == NULL) {
16830       ciEnv::current()->record_failure("CodeCache is full");
16831       return;
16832     }
16833   %}
16834   ins_pipe( pipe_slow );
16835 %}
16836 
16837 // fast char[] to byte[] compression
16838 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16839                          vRegD_V0 tmp1, vRegD_V1 tmp2,
16840                          vRegD_V2 tmp3, vRegD_V3 tmp4,
16841                          iRegI_R0 result, rFlagsReg cr)
16842 %{
16843   match(Set result (StrCompressedCopy src (Binary dst len)));
16844   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4,
16845          USE_KILL src, USE_KILL dst, USE len, KILL cr);
16846 
16847   format %{ "String Compress $src,$dst,$len -> $result  // KILL $src,$dst" %}
16848   ins_encode %{
16849     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16850                            $result$$Register,
16851                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16852                            $tmp3$$FloatRegister, $tmp4$$FloatRegister);
16853   %}
16854   ins_pipe(pipe_slow);
16855 %}
16856 
16857 // fast byte[] to char[] inflation
16858 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
16859                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
16860 %{
16861   match(Set dummy (StrInflatedCopy src (Binary dst len)));
16862   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16863 
16864   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
16865   ins_encode %{
16866     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16867                                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16868                                         $tmp3$$FloatRegister, $tmp4$$Register);
16869     if (tpc == NULL) {
16870       ciEnv::current()->record_failure("CodeCache is full");
16871       return;
16872     }
16873   %}
16874   ins_pipe(pipe_class_memory);
16875 %}
16876 
16877 // encode char[] to byte[] in ISO_8859_1
16878 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16879                           vRegD_V0 vtmp0, vRegD_V1 vtmp1,
16880                           vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16881                           iRegI_R0 result, rFlagsReg cr)
16882 %{
16883   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16884   match(Set result (EncodeISOArray src (Binary dst len)));
16885   effect(USE_KILL src, USE_KILL dst, USE len,
16886          KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
16887 
16888   format %{ "Encode ISO array $src,$dst,$len -> $result" %}
16889   ins_encode %{
16890     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16891                         $result$$Register, false,
16892                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16893                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
16894   %}
16895   ins_pipe(pipe_class_memory);
16896 %}
16897 
16898 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16899                             vRegD_V0 vtmp0, vRegD_V1 vtmp1,
16900                             vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16901                             iRegI_R0 result, rFlagsReg cr)
16902 %{
16903   predicate(((EncodeISOArrayNode*)n)->is_ascii());
16904   match(Set result (EncodeISOArray src (Binary dst len)));
16905   effect(USE_KILL src, USE_KILL dst, USE len,
16906          KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
16907 
16908   format %{ "Encode ASCII array $src,$dst,$len -> $result" %}
16909   ins_encode %{
16910     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16911                         $result$$Register, true,
16912                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16913                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
16914   %}
16915   ins_pipe(pipe_class_memory);
16916 %}
16917 
16918 // ============================================================================
16919 // This name is KNOWN by the ADLC and cannot be changed.
16920 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16921 // for this guy.
16922 instruct tlsLoadP(thread_RegP dst)
16923 %{
16924   match(Set dst (ThreadLocal));
16925 
16926   ins_cost(0);
16927 
16928   format %{ " -- \t// $dst=Thread::current(), empty" %}
16929 
16930   size(0);
16931 
16932   ins_encode( /*empty*/ );
16933 
16934   ins_pipe(pipe_class_empty);
16935 %}
16936 
16937 //----------PEEPHOLE RULES-----------------------------------------------------
16938 // These must follow all instruction definitions as they use the names
16939 // defined in the instructions definitions.
16940 //
16941 // peepmatch ( root_instr_name [preceding_instruction]* );
16942 //
16943 // peepconstraint %{
16944 // (instruction_number.operand_name relational_op instruction_number.operand_name
16945 //  [, ...] );
16946 // // instruction numbers are zero-based using left to right order in peepmatch
16947 //
16948 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
16949 // // provide an instruction_number.operand_name for each operand that appears
16950 // // in the replacement instruction's match rule
16951 //
16952 // ---------VM FLAGS---------------------------------------------------------
16953 //
16954 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16955 //
16956 // Each peephole rule is given an identifying number starting with zero and
16957 // increasing by one in the order seen by the parser.  An individual peephole
16958 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16959 // on the command-line.
16960 //
16961 // ---------CURRENT LIMITATIONS----------------------------------------------
16962 //
16963 // Only match adjacent instructions in same basic block
16964 // Only equality constraints
16965 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16966 // Only one replacement instruction
16967 //
16968 // ---------EXAMPLE----------------------------------------------------------
16969 //
16970 // // pertinent parts of existing instructions in architecture description
16971 // instruct movI(iRegINoSp dst, iRegI src)
16972 // %{
16973 //   match(Set dst (CopyI src));
16974 // %}
16975 //
16976 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16977 // %{
16978 //   match(Set dst (AddI dst src));
16979 //   effect(KILL cr);
16980 // %}
16981 //
16982 // // Change (inc mov) to lea
16983 // peephole %{
16984 //   // increment preceded by register-register move
16985 //   peepmatch ( incI_iReg movI );
16986 //   // require that the destination register of the increment
16987 //   // match the destination register of the move
16988 //   peepconstraint ( 0.dst == 1.dst );
16989 //   // construct a replacement instruction that sets
16990 //   // the destination to ( move's source register + one )
16991 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16992 // %}
16993 //
16994 
16995 // Implementation no longer uses movX instructions since
16996 // machine-independent system no longer uses CopyX nodes.
16997 //
16998 // peephole
16999 // %{
17000 //   peepmatch (incI_iReg movI);
17001 //   peepconstraint (0.dst == 1.dst);
17002 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17003 // %}
17004 
17005 // peephole
17006 // %{
17007 //   peepmatch (decI_iReg movI);
17008 //   peepconstraint (0.dst == 1.dst);
17009 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17010 // %}
17011 
17012 // peephole
17013 // %{
17014 //   peepmatch (addI_iReg_imm movI);
17015 //   peepconstraint (0.dst == 1.dst);
17016 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17017 // %}
17018 
17019 // peephole
17020 // %{
17021 //   peepmatch (incL_iReg movL);
17022 //   peepconstraint (0.dst == 1.dst);
17023 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17024 // %}
17025 
17026 // peephole
17027 // %{
17028 //   peepmatch (decL_iReg movL);
17029 //   peepconstraint (0.dst == 1.dst);
17030 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17031 // %}
17032 
17033 // peephole
17034 // %{
17035 //   peepmatch (addL_iReg_imm movL);
17036 //   peepconstraint (0.dst == 1.dst);
17037 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17038 // %}
17039 
17040 // peephole
17041 // %{
17042 //   peepmatch (addP_iReg_imm movP);
17043 //   peepconstraint (0.dst == 1.dst);
17044 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17045 // %}
17046 
17047 // // Change load of spilled value to only a spill
17048 // instruct storeI(memory mem, iRegI src)
17049 // %{
17050 //   match(Set mem (StoreI mem src));
17051 // %}
17052 //
17053 // instruct loadI(iRegINoSp dst, memory mem)
17054 // %{
17055 //   match(Set dst (LoadI mem));
17056 // %}
17057 //
17058 
17059 //----------SMARTSPILL RULES---------------------------------------------------
17060 // These must follow all instruction definitions as they use the names
17061 // defined in the instructions definitions.
17062 
17063 // Local Variables:
17064 // mode: c++
17065 // End: