1 //
    2 // Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // architecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182 
  183   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  184   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  185   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  186   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  187 
  188   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  189   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  190   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  191   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  192 
  193   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  194   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  195   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  196   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  197 
  198   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  199   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  200   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  201   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  202 
  203   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  204   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  205   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  206   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  207 
  208   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  209   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  210   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  211   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  212 
  213   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  214   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  215   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  216   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  217 
  218   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  219   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  220   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  221   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  222 
  223   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  224   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  225   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  226   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  227 
  228   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  229   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  230   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  231   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  232 
  233   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  234   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  235   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  236   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  237 
  238   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  239   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  240   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  241   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  242 
  243   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  244   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  245   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  246   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  247 
  248   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  249   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  250   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  251   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  252 
  253   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  254   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  255   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  256   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  257 
  258   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  259   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  260   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  261   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  262 
  263   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  264   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  265   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  266   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  267 
  268   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  269   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  270   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  271   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  272 
  273   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  274   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  275   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  276   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  277 
  278   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  279   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  280   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  281   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  282 
  283   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  284   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  285   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  286   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  287 
  288   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  289   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  290   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  291   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  292 
  293   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  294   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  295   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  296   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  297 
  298   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  299   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  300   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  301   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  302 
  303   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  304   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  305   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  306   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  307 
  308   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  309   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  310   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  311   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  312 
  313   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  314   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  315   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  316   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  317 
  318   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  319   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  320   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  321   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  322 
  323   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  324   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  325   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  326   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  327 
  328   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  329   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  330   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  331   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  332 
  333   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  334   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  335   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  336   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  337 
  338 // ----------------------------
  339 // SVE Predicate Registers
  340 // ----------------------------
  341   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  342   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  343   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  344   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  345   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  346   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  347   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  348   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  349   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  350   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  351   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  352   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  353   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  354   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  355   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  356   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  357 
  358 // ----------------------------
  359 // Special Registers
  360 // ----------------------------
  361 
  362 // the AArch64 CSPR status flag register is not directly accessible as
  363 // instruction operand. the FPSR status flag register is a system
  364 // register which can be written/read using MSR/MRS but again does not
  365 // appear as an operand (a code identifying the FSPR occurs as an
  366 // immediate value in the instruction).
  367 
  368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  369 
  370 // Specify priority of register selection within phases of register
  371 // allocation.  Highest priority is first.  A useful heuristic is to
  372 // give registers a low priority when they are required by machine
  373 // instructions, like EAX and EDX on I486, and choose no-save registers
  374 // before save-on-call, & save-on-call before save-on-entry.  Registers
  375 // which participate in fixed calling sequences should come last.
  376 // Registers which are used as pairs must fall on an even boundary.
  377 
  378 alloc_class chunk0(
  379     // volatiles
  380     R10, R10_H,
  381     R11, R11_H,
  382     R12, R12_H,
  383     R13, R13_H,
  384     R14, R14_H,
  385     R15, R15_H,
  386     R16, R16_H,
  387     R17, R17_H,
  388     R18, R18_H,
  389 
  390     // arg registers
  391     R0, R0_H,
  392     R1, R1_H,
  393     R2, R2_H,
  394     R3, R3_H,
  395     R4, R4_H,
  396     R5, R5_H,
  397     R6, R6_H,
  398     R7, R7_H,
  399 
  400     // non-volatiles
  401     R19, R19_H,
  402     R20, R20_H,
  403     R21, R21_H,
  404     R22, R22_H,
  405     R23, R23_H,
  406     R24, R24_H,
  407     R25, R25_H,
  408     R26, R26_H,
  409 
  410     // non-allocatable registers
  411 
  412     R27, R27_H, // heapbase
  413     R28, R28_H, // thread
  414     R29, R29_H, // fp
  415     R30, R30_H, // lr
  416     R31, R31_H, // sp
  417     R8, R8_H,   // rscratch1
  418     R9, R9_H,   // rscratch2
  419 );
  420 
  421 alloc_class chunk1(
  422 
  423     // no save
  424     V16, V16_H, V16_J, V16_K,
  425     V17, V17_H, V17_J, V17_K,
  426     V18, V18_H, V18_J, V18_K,
  427     V19, V19_H, V19_J, V19_K,
  428     V20, V20_H, V20_J, V20_K,
  429     V21, V21_H, V21_J, V21_K,
  430     V22, V22_H, V22_J, V22_K,
  431     V23, V23_H, V23_J, V23_K,
  432     V24, V24_H, V24_J, V24_K,
  433     V25, V25_H, V25_J, V25_K,
  434     V26, V26_H, V26_J, V26_K,
  435     V27, V27_H, V27_J, V27_K,
  436     V28, V28_H, V28_J, V28_K,
  437     V29, V29_H, V29_J, V29_K,
  438     V30, V30_H, V30_J, V30_K,
  439     V31, V31_H, V31_J, V31_K,
  440 
  441     // arg registers
  442     V0, V0_H, V0_J, V0_K,
  443     V1, V1_H, V1_J, V1_K,
  444     V2, V2_H, V2_J, V2_K,
  445     V3, V3_H, V3_J, V3_K,
  446     V4, V4_H, V4_J, V4_K,
  447     V5, V5_H, V5_J, V5_K,
  448     V6, V6_H, V6_J, V6_K,
  449     V7, V7_H, V7_J, V7_K,
  450 
  451     // non-volatiles
  452     V8, V8_H, V8_J, V8_K,
  453     V9, V9_H, V9_J, V9_K,
  454     V10, V10_H, V10_J, V10_K,
  455     V11, V11_H, V11_J, V11_K,
  456     V12, V12_H, V12_J, V12_K,
  457     V13, V13_H, V13_J, V13_K,
  458     V14, V14_H, V14_J, V14_K,
  459     V15, V15_H, V15_J, V15_K,
  460 );
  461 
  462 alloc_class chunk2 (
  463     // Governing predicates for load/store and arithmetic
  464     P0,
  465     P1,
  466     P2,
  467     P3,
  468     P4,
  469     P5,
  470     P6,
  471 
  472     // Extra predicates
  473     P8,
  474     P9,
  475     P10,
  476     P11,
  477     P12,
  478     P13,
  479     P14,
  480     P15,
  481 
  482     // Preserved for all-true predicate
  483     P7,
  484 );
  485 
  486 alloc_class chunk3(RFLAGS);
  487 
  488 //----------Architecture Description Register Classes--------------------------
  489 // Several register classes are automatically defined based upon information in
  490 // this architecture description.
  491 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  493 //
  494 
  495 // Class for all 32 bit general purpose registers
  496 reg_class all_reg32(
  497     R0,
  498     R1,
  499     R2,
  500     R3,
  501     R4,
  502     R5,
  503     R6,
  504     R7,
  505     R10,
  506     R11,
  507     R12,
  508     R13,
  509     R14,
  510     R15,
  511     R16,
  512     R17,
  513     R18,
  514     R19,
  515     R20,
  516     R21,
  517     R22,
  518     R23,
  519     R24,
  520     R25,
  521     R26,
  522     R27,
  523     R28,
  524     R29,
  525     R30,
  526     R31
  527 );
  528 
  529 
  530 // Class for all 32 bit integer registers (excluding SP which
  531 // will never be used as an integer register)
  532 reg_class any_reg32 %{
  533   return _ANY_REG32_mask;
  534 %}
  535 
  536 // Singleton class for R0 int register
  537 reg_class int_r0_reg(R0);
  538 
  539 // Singleton class for R2 int register
  540 reg_class int_r2_reg(R2);
  541 
  542 // Singleton class for R3 int register
  543 reg_class int_r3_reg(R3);
  544 
  545 // Singleton class for R4 int register
  546 reg_class int_r4_reg(R4);
  547 
  548 // Singleton class for R31 int register
  549 reg_class int_r31_reg(R31);
  550 
  551 // Class for all 64 bit general purpose registers
  552 reg_class all_reg(
  553     R0, R0_H,
  554     R1, R1_H,
  555     R2, R2_H,
  556     R3, R3_H,
  557     R4, R4_H,
  558     R5, R5_H,
  559     R6, R6_H,
  560     R7, R7_H,
  561     R10, R10_H,
  562     R11, R11_H,
  563     R12, R12_H,
  564     R13, R13_H,
  565     R14, R14_H,
  566     R15, R15_H,
  567     R16, R16_H,
  568     R17, R17_H,
  569     R18, R18_H,
  570     R19, R19_H,
  571     R20, R20_H,
  572     R21, R21_H,
  573     R22, R22_H,
  574     R23, R23_H,
  575     R24, R24_H,
  576     R25, R25_H,
  577     R26, R26_H,
  578     R27, R27_H,
  579     R28, R28_H,
  580     R29, R29_H,
  581     R30, R30_H,
  582     R31, R31_H
  583 );
  584 
  585 // Class for all long integer registers (including SP)
  586 reg_class any_reg %{
  587   return _ANY_REG_mask;
  588 %}
  589 
  590 // Class for non-allocatable 32 bit registers
  591 reg_class non_allocatable_reg32(
  592 #ifdef R18_RESERVED
  593     // See comment in register_aarch64.hpp
  594     R18,                        // tls on Windows
  595 #endif
  596     R28,                        // thread
  597     R30,                        // lr
  598     R31                         // sp
  599 );
  600 
  601 // Class for non-allocatable 64 bit registers
  602 reg_class non_allocatable_reg(
  603 #ifdef R18_RESERVED
  604     // See comment in register_aarch64.hpp
  605     R18, R18_H,                 // tls on Windows, platform register on macOS
  606 #endif
  607     R28, R28_H,                 // thread
  608     R30, R30_H,                 // lr
  609     R31, R31_H                  // sp
  610 );
  611 
  612 // Class for all non-special integer registers
  613 reg_class no_special_reg32 %{
  614   return _NO_SPECIAL_REG32_mask;
  615 %}
  616 
  617 // Class for all non-special long integer registers
  618 reg_class no_special_reg %{
  619   return _NO_SPECIAL_REG_mask;
  620 %}
  621 
  622 // Class for 64 bit register r0
  623 reg_class r0_reg(
  624     R0, R0_H
  625 );
  626 
  627 // Class for 64 bit register r1
  628 reg_class r1_reg(
  629     R1, R1_H
  630 );
  631 
  632 // Class for 64 bit register r2
  633 reg_class r2_reg(
  634     R2, R2_H
  635 );
  636 
  637 // Class for 64 bit register r3
  638 reg_class r3_reg(
  639     R3, R3_H
  640 );
  641 
  642 // Class for 64 bit register r4
  643 reg_class r4_reg(
  644     R4, R4_H
  645 );
  646 
  647 // Class for 64 bit register r5
  648 reg_class r5_reg(
  649     R5, R5_H
  650 );
  651 
  652 // Class for 64 bit register r10
  653 reg_class r10_reg(
  654     R10, R10_H
  655 );
  656 
  657 // Class for 64 bit register r11
  658 reg_class r11_reg(
  659     R11, R11_H
  660 );
  661 
  662 // Class for method register
  663 reg_class method_reg(
  664     R12, R12_H
  665 );
  666 
  667 // Class for heapbase register
  668 reg_class heapbase_reg(
  669     R27, R27_H
  670 );
  671 
  672 // Class for thread register
  673 reg_class thread_reg(
  674     R28, R28_H
  675 );
  676 
  677 // Class for frame pointer register
  678 reg_class fp_reg(
  679     R29, R29_H
  680 );
  681 
  682 // Class for link register
  683 reg_class lr_reg(
  684     R30, R30_H
  685 );
  686 
  687 // Class for long sp register
  688 reg_class sp_reg(
  689   R31, R31_H
  690 );
  691 
  692 // Class for all pointer registers
  693 reg_class ptr_reg %{
  694   return _PTR_REG_mask;
  695 %}
  696 
  697 // Class for all non_special pointer registers
  698 reg_class no_special_ptr_reg %{
  699   return _NO_SPECIAL_PTR_REG_mask;
  700 %}
  701 
  702 // Class for all float registers
  703 reg_class float_reg(
  704     V0,
  705     V1,
  706     V2,
  707     V3,
  708     V4,
  709     V5,
  710     V6,
  711     V7,
  712     V8,
  713     V9,
  714     V10,
  715     V11,
  716     V12,
  717     V13,
  718     V14,
  719     V15,
  720     V16,
  721     V17,
  722     V18,
  723     V19,
  724     V20,
  725     V21,
  726     V22,
  727     V23,
  728     V24,
  729     V25,
  730     V26,
  731     V27,
  732     V28,
  733     V29,
  734     V30,
  735     V31
  736 );
  737 
  738 // Double precision float registers have virtual `high halves' that
  739 // are needed by the allocator.
  740 // Class for all double registers
  741 reg_class double_reg(
  742     V0, V0_H,
  743     V1, V1_H,
  744     V2, V2_H,
  745     V3, V3_H,
  746     V4, V4_H,
  747     V5, V5_H,
  748     V6, V6_H,
  749     V7, V7_H,
  750     V8, V8_H,
  751     V9, V9_H,
  752     V10, V10_H,
  753     V11, V11_H,
  754     V12, V12_H,
  755     V13, V13_H,
  756     V14, V14_H,
  757     V15, V15_H,
  758     V16, V16_H,
  759     V17, V17_H,
  760     V18, V18_H,
  761     V19, V19_H,
  762     V20, V20_H,
  763     V21, V21_H,
  764     V22, V22_H,
  765     V23, V23_H,
  766     V24, V24_H,
  767     V25, V25_H,
  768     V26, V26_H,
  769     V27, V27_H,
  770     V28, V28_H,
  771     V29, V29_H,
  772     V30, V30_H,
  773     V31, V31_H
  774 );
  775 
  776 // Class for all SVE vector registers.
  777 reg_class vectora_reg (
  778     V0, V0_H, V0_J, V0_K,
  779     V1, V1_H, V1_J, V1_K,
  780     V2, V2_H, V2_J, V2_K,
  781     V3, V3_H, V3_J, V3_K,
  782     V4, V4_H, V4_J, V4_K,
  783     V5, V5_H, V5_J, V5_K,
  784     V6, V6_H, V6_J, V6_K,
  785     V7, V7_H, V7_J, V7_K,
  786     V8, V8_H, V8_J, V8_K,
  787     V9, V9_H, V9_J, V9_K,
  788     V10, V10_H, V10_J, V10_K,
  789     V11, V11_H, V11_J, V11_K,
  790     V12, V12_H, V12_J, V12_K,
  791     V13, V13_H, V13_J, V13_K,
  792     V14, V14_H, V14_J, V14_K,
  793     V15, V15_H, V15_J, V15_K,
  794     V16, V16_H, V16_J, V16_K,
  795     V17, V17_H, V17_J, V17_K,
  796     V18, V18_H, V18_J, V18_K,
  797     V19, V19_H, V19_J, V19_K,
  798     V20, V20_H, V20_J, V20_K,
  799     V21, V21_H, V21_J, V21_K,
  800     V22, V22_H, V22_J, V22_K,
  801     V23, V23_H, V23_J, V23_K,
  802     V24, V24_H, V24_J, V24_K,
  803     V25, V25_H, V25_J, V25_K,
  804     V26, V26_H, V26_J, V26_K,
  805     V27, V27_H, V27_J, V27_K,
  806     V28, V28_H, V28_J, V28_K,
  807     V29, V29_H, V29_J, V29_K,
  808     V30, V30_H, V30_J, V30_K,
  809     V31, V31_H, V31_J, V31_K,
  810 );
  811 
  812 // Class for all 64bit vector registers
  813 reg_class vectord_reg(
  814     V0, V0_H,
  815     V1, V1_H,
  816     V2, V2_H,
  817     V3, V3_H,
  818     V4, V4_H,
  819     V5, V5_H,
  820     V6, V6_H,
  821     V7, V7_H,
  822     V8, V8_H,
  823     V9, V9_H,
  824     V10, V10_H,
  825     V11, V11_H,
  826     V12, V12_H,
  827     V13, V13_H,
  828     V14, V14_H,
  829     V15, V15_H,
  830     V16, V16_H,
  831     V17, V17_H,
  832     V18, V18_H,
  833     V19, V19_H,
  834     V20, V20_H,
  835     V21, V21_H,
  836     V22, V22_H,
  837     V23, V23_H,
  838     V24, V24_H,
  839     V25, V25_H,
  840     V26, V26_H,
  841     V27, V27_H,
  842     V28, V28_H,
  843     V29, V29_H,
  844     V30, V30_H,
  845     V31, V31_H
  846 );
  847 
  848 // Class for all 128bit vector registers
  849 reg_class vectorx_reg(
  850     V0, V0_H, V0_J, V0_K,
  851     V1, V1_H, V1_J, V1_K,
  852     V2, V2_H, V2_J, V2_K,
  853     V3, V3_H, V3_J, V3_K,
  854     V4, V4_H, V4_J, V4_K,
  855     V5, V5_H, V5_J, V5_K,
  856     V6, V6_H, V6_J, V6_K,
  857     V7, V7_H, V7_J, V7_K,
  858     V8, V8_H, V8_J, V8_K,
  859     V9, V9_H, V9_J, V9_K,
  860     V10, V10_H, V10_J, V10_K,
  861     V11, V11_H, V11_J, V11_K,
  862     V12, V12_H, V12_J, V12_K,
  863     V13, V13_H, V13_J, V13_K,
  864     V14, V14_H, V14_J, V14_K,
  865     V15, V15_H, V15_J, V15_K,
  866     V16, V16_H, V16_J, V16_K,
  867     V17, V17_H, V17_J, V17_K,
  868     V18, V18_H, V18_J, V18_K,
  869     V19, V19_H, V19_J, V19_K,
  870     V20, V20_H, V20_J, V20_K,
  871     V21, V21_H, V21_J, V21_K,
  872     V22, V22_H, V22_J, V22_K,
  873     V23, V23_H, V23_J, V23_K,
  874     V24, V24_H, V24_J, V24_K,
  875     V25, V25_H, V25_J, V25_K,
  876     V26, V26_H, V26_J, V26_K,
  877     V27, V27_H, V27_J, V27_K,
  878     V28, V28_H, V28_J, V28_K,
  879     V29, V29_H, V29_J, V29_K,
  880     V30, V30_H, V30_J, V30_K,
  881     V31, V31_H, V31_J, V31_K
  882 );
  883 
  884 // Class for 128 bit register v0
  885 reg_class v0_reg(
  886     V0, V0_H
  887 );
  888 
  889 // Class for 128 bit register v1
  890 reg_class v1_reg(
  891     V1, V1_H
  892 );
  893 
  894 // Class for 128 bit register v2
  895 reg_class v2_reg(
  896     V2, V2_H
  897 );
  898 
  899 // Class for 128 bit register v3
  900 reg_class v3_reg(
  901     V3, V3_H
  902 );
  903 
  904 // Class for 128 bit register v4
  905 reg_class v4_reg(
  906     V4, V4_H
  907 );
  908 
  909 // Class for 128 bit register v5
  910 reg_class v5_reg(
  911     V5, V5_H
  912 );
  913 
  914 // Class for 128 bit register v6
  915 reg_class v6_reg(
  916     V6, V6_H
  917 );
  918 
  919 // Class for 128 bit register v7
  920 reg_class v7_reg(
  921     V7, V7_H
  922 );
  923 
  924 // Class for 128 bit register v8
  925 reg_class v8_reg(
  926     V8, V8_H
  927 );
  928 
  929 // Class for 128 bit register v9
  930 reg_class v9_reg(
  931     V9, V9_H
  932 );
  933 
  934 // Class for 128 bit register v10
  935 reg_class v10_reg(
  936     V10, V10_H
  937 );
  938 
  939 // Class for 128 bit register v11
  940 reg_class v11_reg(
  941     V11, V11_H
  942 );
  943 
  944 // Class for 128 bit register v12
  945 reg_class v12_reg(
  946     V12, V12_H
  947 );
  948 
  949 // Class for 128 bit register v13
  950 reg_class v13_reg(
  951     V13, V13_H
  952 );
  953 
  954 // Class for 128 bit register v14
  955 reg_class v14_reg(
  956     V14, V14_H
  957 );
  958 
  959 // Class for 128 bit register v15
  960 reg_class v15_reg(
  961     V15, V15_H
  962 );
  963 
  964 // Class for 128 bit register v16
  965 reg_class v16_reg(
  966     V16, V16_H
  967 );
  968 
  969 // Class for 128 bit register v17
  970 reg_class v17_reg(
  971     V17, V17_H
  972 );
  973 
  974 // Class for 128 bit register v18
  975 reg_class v18_reg(
  976     V18, V18_H
  977 );
  978 
  979 // Class for 128 bit register v19
  980 reg_class v19_reg(
  981     V19, V19_H
  982 );
  983 
  984 // Class for 128 bit register v20
  985 reg_class v20_reg(
  986     V20, V20_H
  987 );
  988 
  989 // Class for 128 bit register v21
  990 reg_class v21_reg(
  991     V21, V21_H
  992 );
  993 
  994 // Class for 128 bit register v22
  995 reg_class v22_reg(
  996     V22, V22_H
  997 );
  998 
  999 // Class for 128 bit register v23
 1000 reg_class v23_reg(
 1001     V23, V23_H
 1002 );
 1003 
 1004 // Class for 128 bit register v24
 1005 reg_class v24_reg(
 1006     V24, V24_H
 1007 );
 1008 
 1009 // Class for 128 bit register v25
 1010 reg_class v25_reg(
 1011     V25, V25_H
 1012 );
 1013 
 1014 // Class for 128 bit register v26
 1015 reg_class v26_reg(
 1016     V26, V26_H
 1017 );
 1018 
 1019 // Class for 128 bit register v27
 1020 reg_class v27_reg(
 1021     V27, V27_H
 1022 );
 1023 
 1024 // Class for 128 bit register v28
 1025 reg_class v28_reg(
 1026     V28, V28_H
 1027 );
 1028 
 1029 // Class for 128 bit register v29
 1030 reg_class v29_reg(
 1031     V29, V29_H
 1032 );
 1033 
 1034 // Class for 128 bit register v30
 1035 reg_class v30_reg(
 1036     V30, V30_H
 1037 );
 1038 
 1039 // Class for 128 bit register v31
 1040 reg_class v31_reg(
 1041     V31, V31_H
 1042 );
 1043 
 1044 // Class for all SVE predicate registers.
 1045 reg_class pr_reg (
 1046     P0,
 1047     P1,
 1048     P2,
 1049     P3,
 1050     P4,
 1051     P5,
 1052     P6,
 1053     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1054     P8,
 1055     P9,
 1056     P10,
 1057     P11,
 1058     P12,
 1059     P13,
 1060     P14,
 1061     P15
 1062 );
 1063 
 1064 // Class for SVE governing predicate registers, which are used
 1065 // to determine the active elements of a predicated instruction.
 1066 reg_class gov_pr (
 1067     P0,
 1068     P1,
 1069     P2,
 1070     P3,
 1071     P4,
 1072     P5,
 1073     P6,
 1074     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1075 );
 1076 
 1077 reg_class p0_reg(P0);
 1078 reg_class p1_reg(P1);
 1079 
 1080 // Singleton class for condition codes
 1081 reg_class int_flags(RFLAGS);
 1082 
 1083 %}
 1084 
 1085 //----------DEFINITION BLOCK---------------------------------------------------
 1086 // Define name --> value mappings to inform the ADLC of an integer valued name
 1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1088 // Format:
 1089 //        int_def  <name>         ( <int_value>, <expression>);
 1090 // Generated Code in ad_<arch>.hpp
 1091 //        #define  <name>   (<expression>)
 1092 //        // value == <int_value>
 1093 // Generated code in ad_<arch>.cpp adlc_verification()
 1094 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1095 //
 1096 
 1097 // we follow the ppc-aix port in using a simple cost model which ranks
 1098 // register operations as cheap, memory ops as more expensive and
 1099 // branches as most expensive. the first two have a low as well as a
 1100 // normal cost. huge cost appears to be a way of saying don't do
 1101 // something
 1102 
 1103 definitions %{
 1104   // The default cost (of a register move instruction).
 1105   int_def INSN_COST            (    100,     100);
 1106   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1107   int_def CALL_COST            (    200,     2 * INSN_COST);
 1108   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1109 %}
 1110 
 1111 
 1112 //----------SOURCE BLOCK-------------------------------------------------------
 1113 // This is a block of C++ code which provides values, functions, and
 1114 // definitions necessary in the rest of the architecture description
 1115 
 1116 source_hpp %{
 1117 
 1118 #include "asm/macroAssembler.hpp"
 1119 #include "gc/shared/barrierSetAssembler.hpp"
 1120 #include "gc/shared/cardTable.hpp"
 1121 #include "gc/shared/cardTableBarrierSet.hpp"
 1122 #include "gc/shared/collectedHeap.hpp"
 1123 #include "opto/addnode.hpp"
 1124 #include "opto/convertnode.hpp"
 1125 #include "runtime/objectMonitor.hpp"
 1126 
 1127 extern RegMask _ANY_REG32_mask;
 1128 extern RegMask _ANY_REG_mask;
 1129 extern RegMask _PTR_REG_mask;
 1130 extern RegMask _NO_SPECIAL_REG32_mask;
 1131 extern RegMask _NO_SPECIAL_REG_mask;
 1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1133 
 1134 class CallStubImpl {
 1135 
 1136   //--------------------------------------------------------------
 1137   //---<  Used for optimization in Compile::shorten_branches  >---
 1138   //--------------------------------------------------------------
 1139 
 1140  public:
 1141   // Size of call trampoline stub.
 1142   static uint size_call_trampoline() {
 1143     return 0; // no call trampolines on this platform
 1144   }
 1145 
 1146   // number of relocations needed by a call trampoline stub
 1147   static uint reloc_call_trampoline() {
 1148     return 0; // no call trampolines on this platform
 1149   }
 1150 };
 1151 
 1152 class HandlerImpl {
 1153 
 1154  public:
 1155 
 1156   static int emit_exception_handler(CodeBuffer &cbuf);
 1157   static int emit_deopt_handler(CodeBuffer& cbuf);
 1158 
 1159   static uint size_exception_handler() {
 1160     return MacroAssembler::far_codestub_branch_size();
 1161   }
 1162 
 1163   static uint size_deopt_handler() {
 1164     // count one adr and one far branch instruction
 1165     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1166   }
 1167 };
 1168 
 1169 class Node::PD {
 1170 public:
 1171   enum NodeFlags {
 1172     _last_flag = Node::_last_flag
 1173   };
 1174 };
 1175 
 1176   bool is_CAS(int opcode, bool maybe_volatile);
 1177 
 1178   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1179 
 1180   bool unnecessary_acquire(const Node *barrier);
 1181   bool needs_acquiring_load(const Node *load);
 1182 
 1183   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1184 
 1185   bool unnecessary_release(const Node *barrier);
 1186   bool unnecessary_volatile(const Node *barrier);
 1187   bool needs_releasing_store(const Node *store);
 1188 
 1189   // predicate controlling translation of CompareAndSwapX
 1190   bool needs_acquiring_load_exclusive(const Node *load);
 1191 
 1192   // predicate controlling addressing modes
 1193   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1194 %}
 1195 
 1196 source %{
 1197 
 1198   // Derived RegMask with conditionally allocatable registers
 1199 
 1200   void PhaseOutput::pd_perform_mach_node_analysis() {
 1201   }
 1202 
 1203   int MachNode::pd_alignment_required() const {
 1204     return 1;
 1205   }
 1206 
 1207   int MachNode::compute_padding(int current_offset) const {
 1208     return 0;
 1209   }
 1210 
 1211   RegMask _ANY_REG32_mask;
 1212   RegMask _ANY_REG_mask;
 1213   RegMask _PTR_REG_mask;
 1214   RegMask _NO_SPECIAL_REG32_mask;
 1215   RegMask _NO_SPECIAL_REG_mask;
 1216   RegMask _NO_SPECIAL_PTR_REG_mask;
 1217 
 1218   void reg_mask_init() {
 1219     // We derive below RegMask(s) from the ones which are auto-generated from
 1220     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1221     // registers conditionally reserved.
 1222 
 1223     _ANY_REG32_mask = _ALL_REG32_mask;
 1224     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1225 
 1226     _ANY_REG_mask = _ALL_REG_mask;
 1227 
 1228     _PTR_REG_mask = _ALL_REG_mask;
 1229 
 1230     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1231     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1232 
 1233     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1234     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1235 
 1236     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1237     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1238 
 1239     // r27 is not allocatable when compressed oops is on and heapbase is not
 1240     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1241     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1242       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1243       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1244       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1245     }
 1246 
 1247     // r29 is not allocatable when PreserveFramePointer is on
 1248     if (PreserveFramePointer) {
 1249       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1250       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1251       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1252     }
 1253   }
 1254 
 1255   // Optimizaton of volatile gets and puts
 1256   // -------------------------------------
 1257   //
 1258   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1259   // use to implement volatile reads and writes. For a volatile read
 1260   // we simply need
 1261   //
 1262   //   ldar<x>
 1263   //
 1264   // and for a volatile write we need
 1265   //
 1266   //   stlr<x>
 1267   //
 1268   // Alternatively, we can implement them by pairing a normal
 1269   // load/store with a memory barrier. For a volatile read we need
 1270   //
 1271   //   ldr<x>
 1272   //   dmb ishld
 1273   //
 1274   // for a volatile write
 1275   //
 1276   //   dmb ish
 1277   //   str<x>
 1278   //   dmb ish
 1279   //
 1280   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1281   // sequences. These are normally translated to an instruction
 1282   // sequence like the following
 1283   //
 1284   //   dmb      ish
 1285   // retry:
 1286   //   ldxr<x>   rval raddr
 1287   //   cmp       rval rold
 1288   //   b.ne done
 1289   //   stlxr<x>  rval, rnew, rold
 1290   //   cbnz      rval retry
 1291   // done:
 1292   //   cset      r0, eq
 1293   //   dmb ishld
 1294   //
 1295   // Note that the exclusive store is already using an stlxr
 1296   // instruction. That is required to ensure visibility to other
 1297   // threads of the exclusive write (assuming it succeeds) before that
 1298   // of any subsequent writes.
 1299   //
 1300   // The following instruction sequence is an improvement on the above
 1301   //
 1302   // retry:
 1303   //   ldaxr<x>  rval raddr
 1304   //   cmp       rval rold
 1305   //   b.ne done
 1306   //   stlxr<x>  rval, rnew, rold
 1307   //   cbnz      rval retry
 1308   // done:
 1309   //   cset      r0, eq
 1310   //
 1311   // We don't need the leading dmb ish since the stlxr guarantees
 1312   // visibility of prior writes in the case that the swap is
 1313   // successful. Crucially we don't have to worry about the case where
 1314   // the swap is not successful since no valid program should be
 1315   // relying on visibility of prior changes by the attempting thread
 1316   // in the case where the CAS fails.
 1317   //
 1318   // Similarly, we don't need the trailing dmb ishld if we substitute
 1319   // an ldaxr instruction since that will provide all the guarantees we
 1320   // require regarding observation of changes made by other threads
 1321   // before any change to the CAS address observed by the load.
 1322   //
 1323   // In order to generate the desired instruction sequence we need to
 1324   // be able to identify specific 'signature' ideal graph node
 1325   // sequences which i) occur as a translation of a volatile reads or
 1326   // writes or CAS operations and ii) do not occur through any other
 1327   // translation or graph transformation. We can then provide
 1328   // alternative aldc matching rules which translate these node
 1329   // sequences to the desired machine code sequences. Selection of the
 1330   // alternative rules can be implemented by predicates which identify
 1331   // the relevant node sequences.
 1332   //
 1333   // The ideal graph generator translates a volatile read to the node
 1334   // sequence
 1335   //
 1336   //   LoadX[mo_acquire]
 1337   //   MemBarAcquire
 1338   //
 1339   // As a special case when using the compressed oops optimization we
 1340   // may also see this variant
 1341   //
 1342   //   LoadN[mo_acquire]
 1343   //   DecodeN
 1344   //   MemBarAcquire
 1345   //
 1346   // A volatile write is translated to the node sequence
 1347   //
 1348   //   MemBarRelease
 1349   //   StoreX[mo_release] {CardMark}-optional
 1350   //   MemBarVolatile
 1351   //
 1352   // n.b. the above node patterns are generated with a strict
 1353   // 'signature' configuration of input and output dependencies (see
 1354   // the predicates below for exact details). The card mark may be as
 1355   // simple as a few extra nodes or, in a few GC configurations, may
 1356   // include more complex control flow between the leading and
 1357   // trailing memory barriers. However, whatever the card mark
 1358   // configuration these signatures are unique to translated volatile
 1359   // reads/stores -- they will not appear as a result of any other
 1360   // bytecode translation or inlining nor as a consequence of
 1361   // optimizing transforms.
 1362   //
 1363   // We also want to catch inlined unsafe volatile gets and puts and
 1364   // be able to implement them using either ldar<x>/stlr<x> or some
 1365   // combination of ldr<x>/stlr<x> and dmb instructions.
 1366   //
 1367   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1368   // normal volatile put node sequence containing an extra cpuorder
 1369   // membar
 1370   //
 1371   //   MemBarRelease
 1372   //   MemBarCPUOrder
 1373   //   StoreX[mo_release] {CardMark}-optional
 1374   //   MemBarCPUOrder
 1375   //   MemBarVolatile
 1376   //
 1377   // n.b. as an aside, a cpuorder membar is not itself subject to
 1378   // matching and translation by adlc rules.  However, the rule
 1379   // predicates need to detect its presence in order to correctly
 1380   // select the desired adlc rules.
 1381   //
 1382   // Inlined unsafe volatile gets manifest as a slightly different
 1383   // node sequence to a normal volatile get because of the
 1384   // introduction of some CPUOrder memory barriers to bracket the
 1385   // Load. However, but the same basic skeleton of a LoadX feeding a
 1386   // MemBarAcquire, possibly through an optional DecodeN, is still
 1387   // present
 1388   //
 1389   //   MemBarCPUOrder
 1390   //        ||       \\
 1391   //   MemBarCPUOrder LoadX[mo_acquire]
 1392   //        ||            |
 1393   //        ||       {DecodeN} optional
 1394   //        ||       /
 1395   //     MemBarAcquire
 1396   //
 1397   // In this case the acquire membar does not directly depend on the
 1398   // load. However, we can be sure that the load is generated from an
 1399   // inlined unsafe volatile get if we see it dependent on this unique
 1400   // sequence of membar nodes. Similarly, given an acquire membar we
 1401   // can know that it was added because of an inlined unsafe volatile
 1402   // get if it is fed and feeds a cpuorder membar and if its feed
 1403   // membar also feeds an acquiring load.
 1404   //
 1405   // Finally an inlined (Unsafe) CAS operation is translated to the
 1406   // following ideal graph
 1407   //
 1408   //   MemBarRelease
 1409   //   MemBarCPUOrder
 1410   //   CompareAndSwapX {CardMark}-optional
 1411   //   MemBarCPUOrder
 1412   //   MemBarAcquire
 1413   //
 1414   // So, where we can identify these volatile read and write
 1415   // signatures we can choose to plant either of the above two code
 1416   // sequences. For a volatile read we can simply plant a normal
 1417   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1418   // also choose to inhibit translation of the MemBarAcquire and
 1419   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1420   //
 1421   // When we recognise a volatile store signature we can choose to
 1422   // plant at a dmb ish as a translation for the MemBarRelease, a
 1423   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1424   // Alternatively, we can inhibit translation of the MemBarRelease
 1425   // and MemBarVolatile and instead plant a simple stlr<x>
 1426   // instruction.
 1427   //
 1428   // when we recognise a CAS signature we can choose to plant a dmb
 1429   // ish as a translation for the MemBarRelease, the conventional
 1430   // macro-instruction sequence for the CompareAndSwap node (which
 1431   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1432   // Alternatively, we can elide generation of the dmb instructions
 1433   // and plant the alternative CompareAndSwap macro-instruction
 1434   // sequence (which uses ldaxr<x>).
 1435   //
 1436   // Of course, the above only applies when we see these signature
 1437   // configurations. We still want to plant dmb instructions in any
 1438   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1439   // MemBarVolatile. For example, at the end of a constructor which
 1440   // writes final/volatile fields we will see a MemBarRelease
 1441   // instruction and this needs a 'dmb ish' lest we risk the
 1442   // constructed object being visible without making the
 1443   // final/volatile field writes visible.
 1444   //
 1445   // n.b. the translation rules below which rely on detection of the
 1446   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1447   // If we see anything other than the signature configurations we
 1448   // always just translate the loads and stores to ldr<x> and str<x>
 1449   // and translate acquire, release and volatile membars to the
 1450   // relevant dmb instructions.
 1451   //
 1452 
 1453   // is_CAS(int opcode, bool maybe_volatile)
 1454   //
 1455   // return true if opcode is one of the possible CompareAndSwapX
 1456   // values otherwise false.
 1457 
 1458   bool is_CAS(int opcode, bool maybe_volatile)
 1459   {
 1460     switch(opcode) {
 1461       // We handle these
 1462     case Op_CompareAndSwapI:
 1463     case Op_CompareAndSwapL:
 1464     case Op_CompareAndSwapP:
 1465     case Op_CompareAndSwapN:
 1466     case Op_ShenandoahCompareAndSwapP:
 1467     case Op_ShenandoahCompareAndSwapN:
 1468     case Op_CompareAndSwapB:
 1469     case Op_CompareAndSwapS:
 1470     case Op_GetAndSetI:
 1471     case Op_GetAndSetL:
 1472     case Op_GetAndSetP:
 1473     case Op_GetAndSetN:
 1474     case Op_GetAndAddI:
 1475     case Op_GetAndAddL:
 1476       return true;
 1477     case Op_CompareAndExchangeI:
 1478     case Op_CompareAndExchangeN:
 1479     case Op_CompareAndExchangeB:
 1480     case Op_CompareAndExchangeS:
 1481     case Op_CompareAndExchangeL:
 1482     case Op_CompareAndExchangeP:
 1483     case Op_WeakCompareAndSwapB:
 1484     case Op_WeakCompareAndSwapS:
 1485     case Op_WeakCompareAndSwapI:
 1486     case Op_WeakCompareAndSwapL:
 1487     case Op_WeakCompareAndSwapP:
 1488     case Op_WeakCompareAndSwapN:
 1489     case Op_ShenandoahWeakCompareAndSwapP:
 1490     case Op_ShenandoahWeakCompareAndSwapN:
 1491     case Op_ShenandoahCompareAndExchangeP:
 1492     case Op_ShenandoahCompareAndExchangeN:
 1493       return maybe_volatile;
 1494     default:
 1495       return false;
 1496     }
 1497   }
 1498 
 1499   // helper to determine the maximum number of Phi nodes we may need to
 1500   // traverse when searching from a card mark membar for the merge mem
 1501   // feeding a trailing membar or vice versa
 1502 
 1503 // predicates controlling emit of ldr<x>/ldar<x>
 1504 
 1505 bool unnecessary_acquire(const Node *barrier)
 1506 {
 1507   assert(barrier->is_MemBar(), "expecting a membar");
 1508 
 1509   MemBarNode* mb = barrier->as_MemBar();
 1510 
 1511   if (mb->trailing_load()) {
 1512     return true;
 1513   }
 1514 
 1515   if (mb->trailing_load_store()) {
 1516     Node* load_store = mb->in(MemBarNode::Precedent);
 1517     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1518     return is_CAS(load_store->Opcode(), true);
 1519   }
 1520 
 1521   return false;
 1522 }
 1523 
 1524 bool needs_acquiring_load(const Node *n)
 1525 {
 1526   assert(n->is_Load(), "expecting a load");
 1527   LoadNode *ld = n->as_Load();
 1528   return ld->is_acquire();
 1529 }
 1530 
 1531 bool unnecessary_release(const Node *n)
 1532 {
 1533   assert((n->is_MemBar() &&
 1534           n->Opcode() == Op_MemBarRelease),
 1535          "expecting a release membar");
 1536 
 1537   MemBarNode *barrier = n->as_MemBar();
 1538   if (!barrier->leading()) {
 1539     return false;
 1540   } else {
 1541     Node* trailing = barrier->trailing_membar();
 1542     MemBarNode* trailing_mb = trailing->as_MemBar();
 1543     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1544     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1545 
 1546     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1547     if (mem->is_Store()) {
 1548       assert(mem->as_Store()->is_release(), "");
 1549       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1550       return true;
 1551     } else {
 1552       assert(mem->is_LoadStore(), "");
 1553       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1554       return is_CAS(mem->Opcode(), true);
 1555     }
 1556   }
 1557   return false;
 1558 }
 1559 
 1560 bool unnecessary_volatile(const Node *n)
 1561 {
 1562   // assert n->is_MemBar();
 1563   MemBarNode *mbvol = n->as_MemBar();
 1564 
 1565   bool release = mbvol->trailing_store();
 1566   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1567 #ifdef ASSERT
 1568   if (release) {
 1569     Node* leading = mbvol->leading_membar();
 1570     assert(leading->Opcode() == Op_MemBarRelease, "");
 1571     assert(leading->as_MemBar()->leading_store(), "");
 1572     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1573   }
 1574 #endif
 1575 
 1576   return release;
 1577 }
 1578 
 1579 // predicates controlling emit of str<x>/stlr<x>
 1580 
 1581 bool needs_releasing_store(const Node *n)
 1582 {
 1583   // assert n->is_Store();
 1584   StoreNode *st = n->as_Store();
 1585   return st->trailing_membar() != NULL;
 1586 }
 1587 
 1588 // predicate controlling translation of CAS
 1589 //
 1590 // returns true if CAS needs to use an acquiring load otherwise false
 1591 
 1592 bool needs_acquiring_load_exclusive(const Node *n)
 1593 {
 1594   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1595   LoadStoreNode* ldst = n->as_LoadStore();
 1596   if (is_CAS(n->Opcode(), false)) {
 1597     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1598   } else {
 1599     return ldst->trailing_membar() != NULL;
 1600   }
 1601 
 1602   // so we can just return true here
 1603   return true;
 1604 }
 1605 
 1606 #define __ _masm.
 1607 
 1608 // advance declarations for helper functions to convert register
 1609 // indices to register objects
 1610 
 1611 // the ad file has to provide implementations of certain methods
 1612 // expected by the generic code
 1613 //
 1614 // REQUIRED FUNCTIONALITY
 1615 
 1616 //=============================================================================
 1617 
 1618 // !!!!! Special hack to get all types of calls to specify the byte offset
 1619 //       from the start of the call to the point where the return address
 1620 //       will point.
 1621 
 1622 int MachCallStaticJavaNode::ret_addr_offset()
 1623 {
 1624   // call should be a simple bl
 1625   int off = 4;
 1626   return off;
 1627 }
 1628 
 1629 int MachCallDynamicJavaNode::ret_addr_offset()
 1630 {
 1631   return 16; // movz, movk, movk, bl
 1632 }
 1633 
 1634 int MachCallRuntimeNode::ret_addr_offset() {
 1635   // for generated stubs the call will be
 1636   //   bl(addr)
 1637   // or with far branches
 1638   //   bl(trampoline_stub)
 1639   // for real runtime callouts it will be six instructions
 1640   // see aarch64_enc_java_to_runtime
 1641   //   adr(rscratch2, retaddr)
 1642   //   lea(rscratch1, RuntimeAddress(addr)
 1643   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1644   //   blr(rscratch1)
 1645   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1646   if (cb) {
 1647     return 1 * NativeInstruction::instruction_size;
 1648   } else {
 1649     return 6 * NativeInstruction::instruction_size;
 1650   }
 1651 }
 1652 
 1653 //=============================================================================
 1654 
 1655 #ifndef PRODUCT
 1656 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1657   st->print("BREAKPOINT");
 1658 }
 1659 #endif
 1660 
 1661 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1662   C2_MacroAssembler _masm(&cbuf);
 1663   __ brk(0);
 1664 }
 1665 
 1666 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1667   return MachNode::size(ra_);
 1668 }
 1669 
 1670 //=============================================================================
 1671 
 1672 #ifndef PRODUCT
 1673   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1674     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1675   }
 1676 #endif
 1677 
 1678   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1679     C2_MacroAssembler _masm(&cbuf);
 1680     for (int i = 0; i < _count; i++) {
 1681       __ nop();
 1682     }
 1683   }
 1684 
 1685   uint MachNopNode::size(PhaseRegAlloc*) const {
 1686     return _count * NativeInstruction::instruction_size;
 1687   }
 1688 
 1689 //=============================================================================
 1690 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1691 
 1692 int ConstantTable::calculate_table_base_offset() const {
 1693   return 0;  // absolute addressing, no offset
 1694 }
 1695 
 1696 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1697 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1698   ShouldNotReachHere();
 1699 }
 1700 
 1701 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1702   // Empty encoding
 1703 }
 1704 
 1705 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1706   return 0;
 1707 }
 1708 
 1709 #ifndef PRODUCT
 1710 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1711   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1712 }
 1713 #endif
 1714 
 1715 #ifndef PRODUCT
 1716 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1717   Compile* C = ra_->C;
 1718 
 1719   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1720 
 1721   if (C->output()->need_stack_bang(framesize))
 1722     st->print("# stack bang size=%d\n\t", framesize);
 1723 
 1724   if (VM_Version::use_rop_protection()) {
 1725     st->print("ldr zr, [lr]\n\t");
 1726     st->print("pacia  lr, rfp\n\t");
 1727   }
 1728   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1729     st->print("sub  sp, sp, #%d\n\t", framesize);
 1730     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1731     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1732   } else {
 1733     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1734     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1735     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1736     st->print("sub  sp, sp, rscratch1");
 1737   }
 1738   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1739     st->print("\n\t");
 1740     st->print("ldr  rscratch1, [guard]\n\t");
 1741     st->print("dmb ishld\n\t");
 1742     st->print("ldr  rscratch2, [rthread, #thread_disarmed_offset]\n\t");
 1743     st->print("cmp  rscratch1, rscratch2\n\t");
 1744     st->print("b.eq skip");
 1745     st->print("\n\t");
 1746     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1747     st->print("b skip\n\t");
 1748     st->print("guard: int\n\t");
 1749     st->print("\n\t");
 1750     st->print("skip:\n\t");
 1751   }
 1752 }
 1753 #endif
 1754 
 1755 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1756   Compile* C = ra_->C;
 1757   C2_MacroAssembler _masm(&cbuf);
 1758 
 1759   // n.b. frame size includes space for return pc and rfp
 1760   const int framesize = C->output()->frame_size_in_bytes();
 1761 
 1762   // insert a nop at the start of the prolog so we can patch in a
 1763   // branch if we need to invalidate the method later
 1764   __ nop();
 1765 
 1766   if (C->clinit_barrier_on_entry()) {
 1767     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1768 
 1769     Label L_skip_barrier;
 1770 
 1771     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1772     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1773     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1774     __ bind(L_skip_barrier);
 1775   }
 1776 
 1777   if (C->max_vector_size() > 0) {
 1778     __ reinitialize_ptrue();
 1779   }
 1780 
 1781   int bangsize = C->output()->bang_size_in_bytes();
 1782   if (C->output()->need_stack_bang(bangsize))
 1783     __ generate_stack_overflow_check(bangsize);
 1784 
 1785   __ build_frame(framesize);
 1786 
 1787   if (C->stub_function() == NULL) {
 1788     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1789     if (BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1790       // Dummy labels for just measuring the code size
 1791       Label dummy_slow_path;
 1792       Label dummy_continuation;
 1793       Label dummy_guard;
 1794       Label* slow_path = &dummy_slow_path;
 1795       Label* continuation = &dummy_continuation;
 1796       Label* guard = &dummy_guard;
 1797       if (!Compile::current()->output()->in_scratch_emit_size()) {
 1798         // Use real labels from actual stub when not emitting code for the purpose of measuring its size
 1799         C2EntryBarrierStub* stub = Compile::current()->output()->entry_barrier_table()->add_entry_barrier();
 1800         slow_path = &stub->slow_path();
 1801         continuation = &stub->continuation();
 1802         guard = &stub->guard();
 1803       }
 1804       // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
 1805       bs->nmethod_entry_barrier(&_masm, slow_path, continuation, guard);
 1806     }
 1807   }
 1808 
 1809   if (VerifyStackAtCalls) {
 1810     Unimplemented();
 1811   }
 1812 
 1813   C->output()->set_frame_complete(cbuf.insts_size());
 1814 
 1815   if (C->has_mach_constant_base_node()) {
 1816     // NOTE: We set the table base offset here because users might be
 1817     // emitted before MachConstantBaseNode.
 1818     ConstantTable& constant_table = C->output()->constant_table();
 1819     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1820   }
 1821 }
 1822 
 1823 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1824 {
 1825   return MachNode::size(ra_); // too many variables; just compute it
 1826                               // the hard way
 1827 }
 1828 
 1829 int MachPrologNode::reloc() const
 1830 {
 1831   return 0;
 1832 }
 1833 
 1834 //=============================================================================
 1835 
 1836 #ifndef PRODUCT
 1837 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1838   Compile* C = ra_->C;
 1839   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1840 
 1841   st->print("# pop frame %d\n\t",framesize);
 1842 
 1843   if (framesize == 0) {
 1844     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1845   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1846     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1847     st->print("add  sp, sp, #%d\n\t", framesize);
 1848   } else {
 1849     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1850     st->print("add  sp, sp, rscratch1\n\t");
 1851     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1852   }
 1853   if (VM_Version::use_rop_protection()) {
 1854     st->print("autia lr, rfp\n\t");
 1855     st->print("ldr zr, [lr]\n\t");
 1856   }
 1857 
 1858   if (do_polling() && C->is_method_compilation()) {
 1859     st->print("# test polling word\n\t");
 1860     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1861     st->print("cmp  sp, rscratch1\n\t");
 1862     st->print("bhi #slow_path");
 1863   }
 1864 }
 1865 #endif
 1866 
 1867 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1868   Compile* C = ra_->C;
 1869   C2_MacroAssembler _masm(&cbuf);
 1870   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1871 
 1872   __ remove_frame(framesize);
 1873 
 1874   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1875     __ reserved_stack_check();
 1876   }
 1877 
 1878   if (do_polling() && C->is_method_compilation()) {
 1879     Label dummy_label;
 1880     Label* code_stub = &dummy_label;
 1881     if (!C->output()->in_scratch_emit_size()) {
 1882       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1883     }
 1884     __ relocate(relocInfo::poll_return_type);
 1885     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1886   }
 1887 }
 1888 
 1889 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 1890   // Variable size. Determine dynamically.
 1891   return MachNode::size(ra_);
 1892 }
 1893 
 1894 int MachEpilogNode::reloc() const {
 1895   // Return number of relocatable values contained in this instruction.
 1896   return 1; // 1 for polling page.
 1897 }
 1898 
 1899 const Pipeline * MachEpilogNode::pipeline() const {
 1900   return MachNode::pipeline_class();
 1901 }
 1902 
 1903 //=============================================================================
 1904 
 1905 // Figure out which register class each belongs in: rc_int, rc_float or
 1906 // rc_stack.
 1907 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 1908 
 1909 static enum RC rc_class(OptoReg::Name reg) {
 1910 
 1911   if (reg == OptoReg::Bad) {
 1912     return rc_bad;
 1913   }
 1914 
 1915   // we have 32 int registers * 2 halves
 1916   int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
 1917 
 1918   if (reg < slots_of_int_registers) {
 1919     return rc_int;
 1920   }
 1921 
 1922   // we have 32 float register * 8 halves
 1923   int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
 1924   if (reg < slots_of_int_registers + slots_of_float_registers) {
 1925     return rc_float;
 1926   }
 1927 
 1928   int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
 1929   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 1930     return rc_predicate;
 1931   }
 1932 
 1933   // Between predicate regs & stack is the flags.
 1934   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 1935 
 1936   return rc_stack;
 1937 }
 1938 
 1939 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 1940   Compile* C = ra_->C;
 1941 
 1942   // Get registers to move.
 1943   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 1944   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 1945   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 1946   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 1947 
 1948   enum RC src_hi_rc = rc_class(src_hi);
 1949   enum RC src_lo_rc = rc_class(src_lo);
 1950   enum RC dst_hi_rc = rc_class(dst_hi);
 1951   enum RC dst_lo_rc = rc_class(dst_lo);
 1952 
 1953   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 1954 
 1955   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 1956     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 1957            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 1958            "expected aligned-adjacent pairs");
 1959   }
 1960 
 1961   if (src_lo == dst_lo && src_hi == dst_hi) {
 1962     return 0;            // Self copy, no move.
 1963   }
 1964 
 1965   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 1966               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 1967   int src_offset = ra_->reg2offset(src_lo);
 1968   int dst_offset = ra_->reg2offset(dst_lo);
 1969 
 1970   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 1971     uint ireg = ideal_reg();
 1972     if (ireg == Op_VecA && cbuf) {
 1973       C2_MacroAssembler _masm(cbuf);
 1974       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 1975       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1976         // stack->stack
 1977         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 1978                                                 sve_vector_reg_size_in_bytes);
 1979       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 1980         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 1981                             sve_vector_reg_size_in_bytes);
 1982       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 1983         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 1984                               sve_vector_reg_size_in_bytes);
 1985       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 1986         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1987                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 1988                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 1989       } else {
 1990         ShouldNotReachHere();
 1991       }
 1992     } else if (cbuf) {
 1993       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 1994       C2_MacroAssembler _masm(cbuf);
 1995       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 1996       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1997         // stack->stack
 1998         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 1999         if (ireg == Op_VecD) {
 2000           __ unspill(rscratch1, true, src_offset);
 2001           __ spill(rscratch1, true, dst_offset);
 2002         } else {
 2003           __ spill_copy128(src_offset, dst_offset);
 2004         }
 2005       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2006         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2007                ireg == Op_VecD ? __ T8B : __ T16B,
 2008                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2009       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2010         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2011                  ireg == Op_VecD ? __ D : __ Q,
 2012                  ra_->reg2offset(dst_lo));
 2013       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2014         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2015                    ireg == Op_VecD ? __ D : __ Q,
 2016                    ra_->reg2offset(src_lo));
 2017       } else {
 2018         ShouldNotReachHere();
 2019       }
 2020     }
 2021   } else if (cbuf) {
 2022     C2_MacroAssembler _masm(cbuf);
 2023     switch (src_lo_rc) {
 2024     case rc_int:
 2025       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2026         if (is64) {
 2027             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2028                    as_Register(Matcher::_regEncode[src_lo]));
 2029         } else {
 2030             C2_MacroAssembler _masm(cbuf);
 2031             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2032                     as_Register(Matcher::_regEncode[src_lo]));
 2033         }
 2034       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2035         if (is64) {
 2036             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2037                      as_Register(Matcher::_regEncode[src_lo]));
 2038         } else {
 2039             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2040                      as_Register(Matcher::_regEncode[src_lo]));
 2041         }
 2042       } else {                    // gpr --> stack spill
 2043         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2044         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2045       }
 2046       break;
 2047     case rc_float:
 2048       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2049         if (is64) {
 2050             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2051                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2052         } else {
 2053             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2054                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2055         }
 2056       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2057         if (is64) {
 2058             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2059                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2060         } else {
 2061             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2062                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2063         }
 2064       } else {                    // fpr --> stack spill
 2065         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2066         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2067                  is64 ? __ D : __ S, dst_offset);
 2068       }
 2069       break;
 2070     case rc_stack:
 2071       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2072         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2073       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2074         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2075                    is64 ? __ D : __ S, src_offset);
 2076       } else if (dst_lo_rc == rc_predicate) {
 2077         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2078                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2079       } else {                    // stack --> stack copy
 2080         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2081         if (ideal_reg() == Op_RegVectMask) {
 2082           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2083                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2084         } else {
 2085           __ unspill(rscratch1, is64, src_offset);
 2086           __ spill(rscratch1, is64, dst_offset);
 2087         }
 2088       }
 2089       break;
 2090     case rc_predicate:
 2091       if (dst_lo_rc == rc_predicate) {
 2092         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2093       } else if (dst_lo_rc == rc_stack) {
 2094         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2095                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2096       } else {
 2097         assert(false, "bad src and dst rc_class combination.");
 2098         ShouldNotReachHere();
 2099       }
 2100       break;
 2101     default:
 2102       assert(false, "bad rc_class for spill");
 2103       ShouldNotReachHere();
 2104     }
 2105   }
 2106 
 2107   if (st) {
 2108     st->print("spill ");
 2109     if (src_lo_rc == rc_stack) {
 2110       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2111     } else {
 2112       st->print("%s -> ", Matcher::regName[src_lo]);
 2113     }
 2114     if (dst_lo_rc == rc_stack) {
 2115       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2116     } else {
 2117       st->print("%s", Matcher::regName[dst_lo]);
 2118     }
 2119     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2120       int vsize = 0;
 2121       switch (ideal_reg()) {
 2122       case Op_VecD:
 2123         vsize = 64;
 2124         break;
 2125       case Op_VecX:
 2126         vsize = 128;
 2127         break;
 2128       case Op_VecA:
 2129         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2130         break;
 2131       default:
 2132         assert(false, "bad register type for spill");
 2133         ShouldNotReachHere();
 2134       }
 2135       st->print("\t# vector spill size = %d", vsize);
 2136     } else if (ideal_reg() == Op_RegVectMask) {
 2137       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2138       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2139       st->print("\t# predicate spill size = %d", vsize);
 2140     } else {
 2141       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2142     }
 2143   }
 2144 
 2145   return 0;
 2146 
 2147 }
 2148 
 2149 #ifndef PRODUCT
 2150 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2151   if (!ra_)
 2152     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2153   else
 2154     implementation(NULL, ra_, false, st);
 2155 }
 2156 #endif
 2157 
 2158 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2159   implementation(&cbuf, ra_, false, NULL);
 2160 }
 2161 
 2162 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2163   return MachNode::size(ra_);
 2164 }
 2165 
 2166 //=============================================================================
 2167 
 2168 #ifndef PRODUCT
 2169 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2170   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2171   int reg = ra_->get_reg_first(this);
 2172   st->print("add %s, rsp, #%d]\t# box lock",
 2173             Matcher::regName[reg], offset);
 2174 }
 2175 #endif
 2176 
 2177 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2178   C2_MacroAssembler _masm(&cbuf);
 2179 
 2180   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2181   int reg    = ra_->get_encode(this);
 2182 
 2183   // This add will handle any 24-bit signed offset. 24 bits allows an
 2184   // 8 megabyte stack frame.
 2185   __ add(as_Register(reg), sp, offset);
 2186 }
 2187 
 2188 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2189   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2190   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2191 
 2192   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2193     return NativeInstruction::instruction_size;
 2194   } else {
 2195     return 2 * NativeInstruction::instruction_size;
 2196   }
 2197 }
 2198 
 2199 //=============================================================================
 2200 
 2201 #ifndef PRODUCT
 2202 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2203 {
 2204   st->print_cr("# MachUEPNode");
 2205   if (UseCompressedClassPointers) {
 2206     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2207     if (CompressedKlassPointers::shift() != 0) {
 2208       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2209     }
 2210   } else {
 2211    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2212   }
 2213   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2214   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2215 }
 2216 #endif
 2217 
 2218 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2219 {
 2220   // This is the unverified entry point.
 2221   C2_MacroAssembler _masm(&cbuf);
 2222 
 2223   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2224   Label skip;
 2225   // TODO
 2226   // can we avoid this skip and still use a reloc?
 2227   __ br(Assembler::EQ, skip);
 2228   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2229   __ bind(skip);
 2230 }
 2231 
 2232 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2233 {
 2234   return MachNode::size(ra_);
 2235 }
 2236 
 2237 // REQUIRED EMIT CODE
 2238 
 2239 //=============================================================================
 2240 
 2241 // Emit exception handler code.
 2242 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2243 {
 2244   // mov rscratch1 #exception_blob_entry_point
 2245   // br rscratch1
 2246   // Note that the code buffer's insts_mark is always relative to insts.
 2247   // That's why we must use the macroassembler to generate a handler.
 2248   C2_MacroAssembler _masm(&cbuf);
 2249   address base = __ start_a_stub(size_exception_handler());
 2250   if (base == NULL) {
 2251     ciEnv::current()->record_failure("CodeCache is full");
 2252     return 0;  // CodeBuffer::expand failed
 2253   }
 2254   int offset = __ offset();
 2255   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2256   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2257   __ end_a_stub();
 2258   return offset;
 2259 }
 2260 
 2261 // Emit deopt handler code.
 2262 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2263 {
 2264   // Note that the code buffer's insts_mark is always relative to insts.
 2265   // That's why we must use the macroassembler to generate a handler.
 2266   C2_MacroAssembler _masm(&cbuf);
 2267   address base = __ start_a_stub(size_deopt_handler());
 2268   if (base == NULL) {
 2269     ciEnv::current()->record_failure("CodeCache is full");
 2270     return 0;  // CodeBuffer::expand failed
 2271   }
 2272   int offset = __ offset();
 2273 
 2274   __ adr(lr, __ pc());
 2275   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2276 
 2277   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2278   __ end_a_stub();
 2279   return offset;
 2280 }
 2281 
 2282 // REQUIRED MATCHER CODE
 2283 
 2284 //=============================================================================
 2285 
 2286 const bool Matcher::match_rule_supported(int opcode) {
 2287   if (!has_match_rule(opcode))
 2288     return false;
 2289 
 2290   bool ret_value = true;
 2291   switch (opcode) {
 2292     case Op_OnSpinWait:
 2293       return VM_Version::supports_on_spin_wait();
 2294     case Op_CacheWB:
 2295     case Op_CacheWBPreSync:
 2296     case Op_CacheWBPostSync:
 2297       if (!VM_Version::supports_data_cache_line_flush()) {
 2298         ret_value = false;
 2299       }
 2300       break;
 2301   }
 2302 
 2303   return ret_value; // Per default match rules are supported.
 2304 }
 2305 
 2306 const RegMask* Matcher::predicate_reg_mask(void) {
 2307   return &_PR_REG_mask;
 2308 }
 2309 
 2310 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2311   return new TypeVectMask(elemTy, length);
 2312 }
 2313 
 2314 // Vector calling convention not yet implemented.
 2315 const bool Matcher::supports_vector_calling_convention(void) {
 2316   return false;
 2317 }
 2318 
 2319 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2320   Unimplemented();
 2321   return OptoRegPair(0, 0);
 2322 }
 2323 
 2324 // Is this branch offset short enough that a short branch can be used?
 2325 //
 2326 // NOTE: If the platform does not provide any short branch variants, then
 2327 //       this method should return false for offset 0.
 2328 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2329   // The passed offset is relative to address of the branch.
 2330 
 2331   return (-32768 <= offset && offset < 32768);
 2332 }
 2333 
 2334 // Vector width in bytes.
 2335 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2336   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2337   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2338   // Minimum 2 values in vector
 2339   if (size < 2*type2aelembytes(bt)) size = 0;
 2340   // But never < 4
 2341   if (size < 4) size = 0;
 2342   return size;
 2343 }
 2344 
 2345 // Limits on vector size (number of elements) loaded into vector.
 2346 const int Matcher::max_vector_size(const BasicType bt) {
 2347   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2348 }
 2349 
 2350 const int Matcher::min_vector_size(const BasicType bt) {
 2351   int max_size = max_vector_size(bt);
 2352   // Limit the min vector size to 8 bytes.
 2353   int size = 8 / type2aelembytes(bt);
 2354   if (bt == T_BYTE) {
 2355     // To support vector api shuffle/rearrange.
 2356     size = 4;
 2357   } else if (bt == T_BOOLEAN) {
 2358     // To support vector api load/store mask.
 2359     size = 2;
 2360   }
 2361   if (size < 2) size = 2;
 2362   return MIN2(size, max_size);
 2363 }
 2364 
 2365 // Actual max scalable vector register length.
 2366 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2367   return Matcher::max_vector_size(bt);
 2368 }
 2369 
 2370 // Vector ideal reg.
 2371 const uint Matcher::vector_ideal_reg(int len) {
 2372   if (UseSVE > 0 && 16 < len && len <= 256) {
 2373     return Op_VecA;
 2374   }
 2375   switch(len) {
 2376     // For 16-bit/32-bit mask vector, reuse VecD.
 2377     case  2:
 2378     case  4:
 2379     case  8: return Op_VecD;
 2380     case 16: return Op_VecX;
 2381   }
 2382   ShouldNotReachHere();
 2383   return 0;
 2384 }
 2385 
 2386 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
 2387   assert(Matcher::is_generic_vector(generic_opnd), "not generic");
 2388   switch (ideal_reg) {
 2389     case Op_VecA: return new vecAOper();
 2390     case Op_VecD: return new vecDOper();
 2391     case Op_VecX: return new vecXOper();
 2392   }
 2393   ShouldNotReachHere();
 2394   return NULL;
 2395 }
 2396 
 2397 bool Matcher::is_reg2reg_move(MachNode* m) {
 2398   return false;
 2399 }
 2400 
 2401 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2402   return opnd->opcode() == VREG;
 2403 }
 2404 
 2405 // Return whether or not this register is ever used as an argument.
 2406 // This function is used on startup to build the trampoline stubs in
 2407 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2408 // call in the trampoline, and arguments in those registers not be
 2409 // available to the callee.
 2410 bool Matcher::can_be_java_arg(int reg)
 2411 {
 2412   return
 2413     reg ==  R0_num || reg == R0_H_num ||
 2414     reg ==  R1_num || reg == R1_H_num ||
 2415     reg ==  R2_num || reg == R2_H_num ||
 2416     reg ==  R3_num || reg == R3_H_num ||
 2417     reg ==  R4_num || reg == R4_H_num ||
 2418     reg ==  R5_num || reg == R5_H_num ||
 2419     reg ==  R6_num || reg == R6_H_num ||
 2420     reg ==  R7_num || reg == R7_H_num ||
 2421     reg ==  V0_num || reg == V0_H_num ||
 2422     reg ==  V1_num || reg == V1_H_num ||
 2423     reg ==  V2_num || reg == V2_H_num ||
 2424     reg ==  V3_num || reg == V3_H_num ||
 2425     reg ==  V4_num || reg == V4_H_num ||
 2426     reg ==  V5_num || reg == V5_H_num ||
 2427     reg ==  V6_num || reg == V6_H_num ||
 2428     reg ==  V7_num || reg == V7_H_num;
 2429 }
 2430 
 2431 bool Matcher::is_spillable_arg(int reg)
 2432 {
 2433   return can_be_java_arg(reg);
 2434 }
 2435 
 2436 uint Matcher::int_pressure_limit()
 2437 {
 2438   // JDK-8183543: When taking the number of available registers as int
 2439   // register pressure threshold, the jtreg test:
 2440   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2441   // failed due to C2 compilation failure with
 2442   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2443   //
 2444   // A derived pointer is live at CallNode and then is flagged by RA
 2445   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2446   // derived pointers and lastly fail to spill after reaching maximum
 2447   // number of iterations. Lowering the default pressure threshold to
 2448   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2449   // a high register pressure area of the code so that split_DEF can
 2450   // generate DefinitionSpillCopy for the derived pointer.
 2451   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2452   if (!PreserveFramePointer) {
 2453     // When PreserveFramePointer is off, frame pointer is allocatable,
 2454     // but different from other SOC registers, it is excluded from
 2455     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2456     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2457     // See check_pressure_at_fatproj().
 2458     default_int_pressure_threshold--;
 2459   }
 2460   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2461 }
 2462 
 2463 uint Matcher::float_pressure_limit()
 2464 {
 2465   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2466   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2467 }
 2468 
 2469 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2470   return false;
 2471 }
 2472 
 2473 RegMask Matcher::divI_proj_mask() {
 2474   ShouldNotReachHere();
 2475   return RegMask();
 2476 }
 2477 
 2478 // Register for MODI projection of divmodI.
 2479 RegMask Matcher::modI_proj_mask() {
 2480   ShouldNotReachHere();
 2481   return RegMask();
 2482 }
 2483 
 2484 // Register for DIVL projection of divmodL.
 2485 RegMask Matcher::divL_proj_mask() {
 2486   ShouldNotReachHere();
 2487   return RegMask();
 2488 }
 2489 
 2490 // Register for MODL projection of divmodL.
 2491 RegMask Matcher::modL_proj_mask() {
 2492   ShouldNotReachHere();
 2493   return RegMask();
 2494 }
 2495 
 2496 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2497   return FP_REG_mask();
 2498 }
 2499 
 2500 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2501   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2502     Node* u = addp->fast_out(i);
 2503     if (u->is_LoadStore()) {
 2504       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2505       // instructions) only take register indirect as an operand, so
 2506       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2507       // must fail.
 2508       return false;
 2509     }
 2510     if (u->is_Mem()) {
 2511       int opsize = u->as_Mem()->memory_size();
 2512       assert(opsize > 0, "unexpected memory operand size");
 2513       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2514         return false;
 2515       }
 2516     }
 2517   }
 2518   return true;
 2519 }
 2520 
 2521 // Binary src (Replicate con)
 2522 bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
 2523   if (n == NULL || m == NULL) {
 2524     return false;
 2525   }
 2526 
 2527   if (UseSVE == 0 || !VectorNode::is_invariant_vector(m)) {
 2528     return false;
 2529   }
 2530 
 2531   Node* imm_node = m->in(1);
 2532   if (!imm_node->is_Con()) {
 2533     return false;
 2534   }
 2535 
 2536   const Type* t = imm_node->bottom_type();
 2537   if (!(t->isa_int() || t->isa_long())) {
 2538     return false;
 2539   }
 2540 
 2541   switch (n->Opcode()) {
 2542   case Op_AndV:
 2543   case Op_OrV:
 2544   case Op_XorV: {
 2545     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
 2546     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2547     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2548   }
 2549   case Op_AddVB:
 2550     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2551   case Op_AddVS:
 2552   case Op_AddVI:
 2553     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2554   case Op_AddVL:
 2555     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2556   default:
 2557     return false;
 2558   }
 2559 }
 2560 
 2561 // (XorV src (Replicate m1))
 2562 // (XorVMask src (MaskAll m1))
 2563 bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
 2564   if (n != NULL && m != NULL) {
 2565     return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
 2566            VectorNode::is_all_ones_vector(m);
 2567   }
 2568   return false;
 2569 }
 2570 
 2571 // Should the matcher clone input 'm' of node 'n'?
 2572 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2573   if (is_vshift_con_pattern(n, m) ||
 2574       is_vector_bitwise_not_pattern(n, m) ||
 2575       is_valid_sve_arith_imm_pattern(n, m)) {
 2576     mstack.push(m, Visit);
 2577     return true;
 2578   }
 2579   return false;
 2580 }
 2581 
 2582 // Should the Matcher clone shifts on addressing modes, expecting them
 2583 // to be subsumed into complex addressing expressions or compute them
 2584 // into registers?
 2585 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2586   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2587     return true;
 2588   }
 2589 
 2590   Node *off = m->in(AddPNode::Offset);
 2591   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2592       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2593       // Are there other uses besides address expressions?
 2594       !is_visited(off)) {
 2595     address_visited.set(off->_idx); // Flag as address_visited
 2596     mstack.push(off->in(2), Visit);
 2597     Node *conv = off->in(1);
 2598     if (conv->Opcode() == Op_ConvI2L &&
 2599         // Are there other uses besides address expressions?
 2600         !is_visited(conv)) {
 2601       address_visited.set(conv->_idx); // Flag as address_visited
 2602       mstack.push(conv->in(1), Pre_Visit);
 2603     } else {
 2604       mstack.push(conv, Pre_Visit);
 2605     }
 2606     address_visited.test_set(m->_idx); // Flag as address_visited
 2607     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2608     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2609     return true;
 2610   } else if (off->Opcode() == Op_ConvI2L &&
 2611              // Are there other uses besides address expressions?
 2612              !is_visited(off)) {
 2613     address_visited.test_set(m->_idx); // Flag as address_visited
 2614     address_visited.set(off->_idx); // Flag as address_visited
 2615     mstack.push(off->in(1), Pre_Visit);
 2616     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2617     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2618     return true;
 2619   }
 2620   return false;
 2621 }
 2622 
 2623 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2624   C2_MacroAssembler _masm(&cbuf);                                       \
 2625   {                                                                     \
 2626     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2627     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2628     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2629     __ INSN(REG, as_Register(BASE));                                    \
 2630   }
 2631 
 2632 
 2633 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2634   {
 2635     Address::extend scale;
 2636 
 2637     // Hooboy, this is fugly.  We need a way to communicate to the
 2638     // encoder that the index needs to be sign extended, so we have to
 2639     // enumerate all the cases.
 2640     switch (opcode) {
 2641     case INDINDEXSCALEDI2L:
 2642     case INDINDEXSCALEDI2LN:
 2643     case INDINDEXI2L:
 2644     case INDINDEXI2LN:
 2645       scale = Address::sxtw(size);
 2646       break;
 2647     default:
 2648       scale = Address::lsl(size);
 2649     }
 2650 
 2651     if (index == -1) {
 2652       return Address(base, disp);
 2653     } else {
 2654       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2655       return Address(base, as_Register(index), scale);
 2656     }
 2657   }
 2658 
 2659 
 2660 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2661 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2662 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2663 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2664                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2665 
 2666   // Used for all non-volatile memory accesses.  The use of
 2667   // $mem->opcode() to discover whether this pattern uses sign-extended
 2668   // offsets is something of a kludge.
 2669   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2670                         Register reg, int opcode,
 2671                         Register base, int index, int scale, int disp,
 2672                         int size_in_memory)
 2673   {
 2674     Address addr = mem2address(opcode, base, index, scale, disp);
 2675     if (addr.getMode() == Address::base_plus_offset) {
 2676       /* If we get an out-of-range offset it is a bug in the compiler,
 2677          so we assert here. */
 2678       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2679              "c2 compiler bug");
 2680       /* Fix up any out-of-range offsets. */
 2681       assert_different_registers(rscratch1, base);
 2682       assert_different_registers(rscratch1, reg);
 2683       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2684     }
 2685     (masm.*insn)(reg, addr);
 2686   }
 2687 
 2688   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2689                         FloatRegister reg, int opcode,
 2690                         Register base, int index, int size, int disp,
 2691                         int size_in_memory)
 2692   {
 2693     Address::extend scale;
 2694 
 2695     switch (opcode) {
 2696     case INDINDEXSCALEDI2L:
 2697     case INDINDEXSCALEDI2LN:
 2698       scale = Address::sxtw(size);
 2699       break;
 2700     default:
 2701       scale = Address::lsl(size);
 2702     }
 2703 
 2704     if (index == -1) {
 2705       /* If we get an out-of-range offset it is a bug in the compiler,
 2706          so we assert here. */
 2707       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2708       /* Fix up any out-of-range offsets. */
 2709       assert_different_registers(rscratch1, base);
 2710       Address addr = Address(base, disp);
 2711       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2712       (masm.*insn)(reg, addr);
 2713     } else {
 2714       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2715       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2716     }
 2717   }
 2718 
 2719   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2720                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2721                         int opcode, Register base, int index, int size, int disp)
 2722   {
 2723     if (index == -1) {
 2724       (masm.*insn)(reg, T, Address(base, disp));
 2725     } else {
 2726       assert(disp == 0, "unsupported address mode");
 2727       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2728     }
 2729   }
 2730 
 2731 %}
 2732 
 2733 
 2734 
 2735 //----------ENCODING BLOCK-----------------------------------------------------
 2736 // This block specifies the encoding classes used by the compiler to
 2737 // output byte streams.  Encoding classes are parameterized macros
 2738 // used by Machine Instruction Nodes in order to generate the bit
 2739 // encoding of the instruction.  Operands specify their base encoding
 2740 // interface with the interface keyword.  There are currently
 2741 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2742 // COND_INTER.  REG_INTER causes an operand to generate a function
 2743 // which returns its register number when queried.  CONST_INTER causes
 2744 // an operand to generate a function which returns the value of the
 2745 // constant when queried.  MEMORY_INTER causes an operand to generate
 2746 // four functions which return the Base Register, the Index Register,
 2747 // the Scale Value, and the Offset Value of the operand when queried.
 2748 // COND_INTER causes an operand to generate six functions which return
 2749 // the encoding code (ie - encoding bits for the instruction)
 2750 // associated with each basic boolean condition for a conditional
 2751 // instruction.
 2752 //
 2753 // Instructions specify two basic values for encoding.  Again, a
 2754 // function is available to check if the constant displacement is an
 2755 // oop. They use the ins_encode keyword to specify their encoding
 2756 // classes (which must be a sequence of enc_class names, and their
 2757 // parameters, specified in the encoding block), and they use the
 2758 // opcode keyword to specify, in order, their primary, secondary, and
 2759 // tertiary opcode.  Only the opcode sections which a particular
 2760 // instruction needs for encoding need to be specified.
 2761 encode %{
 2762   // Build emit functions for each basic byte or larger field in the
 2763   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2764   // from C++ code in the enc_class source block.  Emit functions will
 2765   // live in the main source block for now.  In future, we can
 2766   // generalize this by adding a syntax that specifies the sizes of
 2767   // fields in an order, so that the adlc can build the emit functions
 2768   // automagically
 2769 
 2770   // catch all for unimplemented encodings
 2771   enc_class enc_unimplemented %{
 2772     C2_MacroAssembler _masm(&cbuf);
 2773     __ unimplemented("C2 catch all");
 2774   %}
 2775 
 2776   // BEGIN Non-volatile memory access
 2777 
 2778   // This encoding class is generated automatically from ad_encode.m4.
 2779   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2780   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2781     Register dst_reg = as_Register($dst$$reg);
 2782     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2783                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2784   %}
 2785 
 2786   // This encoding class is generated automatically from ad_encode.m4.
 2787   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2788   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2789     Register dst_reg = as_Register($dst$$reg);
 2790     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2791                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2792   %}
 2793 
 2794   // This encoding class is generated automatically from ad_encode.m4.
 2795   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2796   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2797     Register dst_reg = as_Register($dst$$reg);
 2798     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2799                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2800   %}
 2801 
 2802   // This encoding class is generated automatically from ad_encode.m4.
 2803   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2804   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2805     Register dst_reg = as_Register($dst$$reg);
 2806     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2807                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2808   %}
 2809 
 2810   // This encoding class is generated automatically from ad_encode.m4.
 2811   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2812   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2813     Register dst_reg = as_Register($dst$$reg);
 2814     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2815                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2816   %}
 2817 
 2818   // This encoding class is generated automatically from ad_encode.m4.
 2819   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2820   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2821     Register dst_reg = as_Register($dst$$reg);
 2822     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2823                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2824   %}
 2825 
 2826   // This encoding class is generated automatically from ad_encode.m4.
 2827   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2828   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2829     Register dst_reg = as_Register($dst$$reg);
 2830     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2831                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2832   %}
 2833 
 2834   // This encoding class is generated automatically from ad_encode.m4.
 2835   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2836   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2837     Register dst_reg = as_Register($dst$$reg);
 2838     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2839                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2840   %}
 2841 
 2842   // This encoding class is generated automatically from ad_encode.m4.
 2843   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2844   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2845     Register dst_reg = as_Register($dst$$reg);
 2846     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2847                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2848   %}
 2849 
 2850   // This encoding class is generated automatically from ad_encode.m4.
 2851   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2852   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2853     Register dst_reg = as_Register($dst$$reg);
 2854     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2855                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2856   %}
 2857 
 2858   // This encoding class is generated automatically from ad_encode.m4.
 2859   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2860   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2861     Register dst_reg = as_Register($dst$$reg);
 2862     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2863                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2864   %}
 2865 
 2866   // This encoding class is generated automatically from ad_encode.m4.
 2867   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2868   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2869     Register dst_reg = as_Register($dst$$reg);
 2870     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2871                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2872   %}
 2873 
 2874   // This encoding class is generated automatically from ad_encode.m4.
 2875   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2876   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2877     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2878     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2879                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2880   %}
 2881 
 2882   // This encoding class is generated automatically from ad_encode.m4.
 2883   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2884   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 2885     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2886     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2887                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2888   %}
 2889 
 2890   // This encoding class is generated automatically from ad_encode.m4.
 2891   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2892   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 2893     Register src_reg = as_Register($src$$reg);
 2894     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 2895                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2896   %}
 2897 
 2898   // This encoding class is generated automatically from ad_encode.m4.
 2899   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2900   enc_class aarch64_enc_strb0(memory1 mem) %{
 2901     C2_MacroAssembler _masm(&cbuf);
 2902     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2903                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2904   %}
 2905 
 2906   // This encoding class is generated automatically from ad_encode.m4.
 2907   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2908   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 2909     Register src_reg = as_Register($src$$reg);
 2910     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 2911                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2912   %}
 2913 
 2914   // This encoding class is generated automatically from ad_encode.m4.
 2915   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2916   enc_class aarch64_enc_strh0(memory2 mem) %{
 2917     C2_MacroAssembler _masm(&cbuf);
 2918     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 2919                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2920   %}
 2921 
 2922   // This encoding class is generated automatically from ad_encode.m4.
 2923   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2924   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 2925     Register src_reg = as_Register($src$$reg);
 2926     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 2927                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2928   %}
 2929 
 2930   // This encoding class is generated automatically from ad_encode.m4.
 2931   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2932   enc_class aarch64_enc_strw0(memory4 mem) %{
 2933     C2_MacroAssembler _masm(&cbuf);
 2934     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 2935                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2936   %}
 2937 
 2938   // This encoding class is generated automatically from ad_encode.m4.
 2939   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2940   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 2941     Register src_reg = as_Register($src$$reg);
 2942     // we sometimes get asked to store the stack pointer into the
 2943     // current thread -- we cannot do that directly on AArch64
 2944     if (src_reg == r31_sp) {
 2945       C2_MacroAssembler _masm(&cbuf);
 2946       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 2947       __ mov(rscratch2, sp);
 2948       src_reg = rscratch2;
 2949     }
 2950     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 2951                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2952   %}
 2953 
 2954   // This encoding class is generated automatically from ad_encode.m4.
 2955   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2956   enc_class aarch64_enc_str0(memory8 mem) %{
 2957     C2_MacroAssembler _masm(&cbuf);
 2958     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 2959                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2960   %}
 2961 
 2962   // This encoding class is generated automatically from ad_encode.m4.
 2963   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2964   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 2965     FloatRegister src_reg = as_FloatRegister($src$$reg);
 2966     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 2967                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2968   %}
 2969 
 2970   // This encoding class is generated automatically from ad_encode.m4.
 2971   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2972   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 2973     FloatRegister src_reg = as_FloatRegister($src$$reg);
 2974     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 2975                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2976   %}
 2977 
 2978   // This encoding class is generated automatically from ad_encode.m4.
 2979   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2980   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 2981       C2_MacroAssembler _masm(&cbuf);
 2982       __ membar(Assembler::StoreStore);
 2983       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2984                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2985   %}
 2986 
 2987   // END Non-volatile memory access
 2988 
 2989   // Vector loads and stores
 2990   enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
 2991     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2992     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 2993        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 2994   %}
 2995 
 2996   enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
 2997     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2998     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 2999        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3000   %}
 3001 
 3002   enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
 3003     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3004     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3005        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3006   %}
 3007 
 3008   enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
 3009     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3010     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3011        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3012   %}
 3013 
 3014   enc_class aarch64_enc_strvH(vReg src, memory mem) %{
 3015     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3016     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3017        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3018   %}
 3019 
 3020   enc_class aarch64_enc_strvS(vReg src, memory mem) %{
 3021     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3022     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3023        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3024   %}
 3025 
 3026   enc_class aarch64_enc_strvD(vReg src, memory mem) %{
 3027     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3028     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3029        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3030   %}
 3031 
 3032   enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
 3033     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3034     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3035        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3036   %}
 3037 
 3038   // volatile loads and stores
 3039 
 3040   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3041     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3042                  rscratch1, stlrb);
 3043   %}
 3044 
 3045   enc_class aarch64_enc_stlrb0(memory mem) %{
 3046     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3047                  rscratch1, stlrb);
 3048   %}
 3049 
 3050   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3051     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3052                  rscratch1, stlrh);
 3053   %}
 3054 
 3055   enc_class aarch64_enc_stlrh0(memory mem) %{
 3056     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3057                  rscratch1, stlrh);
 3058   %}
 3059 
 3060   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3061     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3062                  rscratch1, stlrw);
 3063   %}
 3064 
 3065   enc_class aarch64_enc_stlrw0(memory mem) %{
 3066     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3067                  rscratch1, stlrw);
 3068   %}
 3069 
 3070   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3071     Register dst_reg = as_Register($dst$$reg);
 3072     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3073              rscratch1, ldarb);
 3074     __ sxtbw(dst_reg, dst_reg);
 3075   %}
 3076 
 3077   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3078     Register dst_reg = as_Register($dst$$reg);
 3079     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3080              rscratch1, ldarb);
 3081     __ sxtb(dst_reg, dst_reg);
 3082   %}
 3083 
 3084   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3085     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3086              rscratch1, ldarb);
 3087   %}
 3088 
 3089   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3090     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3091              rscratch1, ldarb);
 3092   %}
 3093 
 3094   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3095     Register dst_reg = as_Register($dst$$reg);
 3096     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3097              rscratch1, ldarh);
 3098     __ sxthw(dst_reg, dst_reg);
 3099   %}
 3100 
 3101   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3102     Register dst_reg = as_Register($dst$$reg);
 3103     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3104              rscratch1, ldarh);
 3105     __ sxth(dst_reg, dst_reg);
 3106   %}
 3107 
 3108   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3109     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3110              rscratch1, ldarh);
 3111   %}
 3112 
 3113   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3114     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3115              rscratch1, ldarh);
 3116   %}
 3117 
 3118   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3119     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3120              rscratch1, ldarw);
 3121   %}
 3122 
 3123   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3124     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3125              rscratch1, ldarw);
 3126   %}
 3127 
 3128   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3129     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3130              rscratch1, ldar);
 3131   %}
 3132 
 3133   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3134     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3135              rscratch1, ldarw);
 3136     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3137   %}
 3138 
 3139   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3140     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3141              rscratch1, ldar);
 3142     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3143   %}
 3144 
 3145   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3146     Register src_reg = as_Register($src$$reg);
 3147     // we sometimes get asked to store the stack pointer into the
 3148     // current thread -- we cannot do that directly on AArch64
 3149     if (src_reg == r31_sp) {
 3150       C2_MacroAssembler _masm(&cbuf);
 3151       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3152       __ mov(rscratch2, sp);
 3153       src_reg = rscratch2;
 3154     }
 3155     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3156                  rscratch1, stlr);
 3157   %}
 3158 
 3159   enc_class aarch64_enc_stlr0(memory mem) %{
 3160     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3161                  rscratch1, stlr);
 3162   %}
 3163 
 3164   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3165     {
 3166       C2_MacroAssembler _masm(&cbuf);
 3167       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3168       __ fmovs(rscratch2, src_reg);
 3169     }
 3170     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3171                  rscratch1, stlrw);
 3172   %}
 3173 
 3174   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3175     {
 3176       C2_MacroAssembler _masm(&cbuf);
 3177       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3178       __ fmovd(rscratch2, src_reg);
 3179     }
 3180     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3181                  rscratch1, stlr);
 3182   %}
 3183 
 3184   // synchronized read/update encodings
 3185 
 3186   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3187     C2_MacroAssembler _masm(&cbuf);
 3188     Register dst_reg = as_Register($dst$$reg);
 3189     Register base = as_Register($mem$$base);
 3190     int index = $mem$$index;
 3191     int scale = $mem$$scale;
 3192     int disp = $mem$$disp;
 3193     if (index == -1) {
 3194        if (disp != 0) {
 3195         __ lea(rscratch1, Address(base, disp));
 3196         __ ldaxr(dst_reg, rscratch1);
 3197       } else {
 3198         // TODO
 3199         // should we ever get anything other than this case?
 3200         __ ldaxr(dst_reg, base);
 3201       }
 3202     } else {
 3203       Register index_reg = as_Register(index);
 3204       if (disp == 0) {
 3205         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3206         __ ldaxr(dst_reg, rscratch1);
 3207       } else {
 3208         __ lea(rscratch1, Address(base, disp));
 3209         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3210         __ ldaxr(dst_reg, rscratch1);
 3211       }
 3212     }
 3213   %}
 3214 
 3215   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3216     C2_MacroAssembler _masm(&cbuf);
 3217     Register src_reg = as_Register($src$$reg);
 3218     Register base = as_Register($mem$$base);
 3219     int index = $mem$$index;
 3220     int scale = $mem$$scale;
 3221     int disp = $mem$$disp;
 3222     if (index == -1) {
 3223        if (disp != 0) {
 3224         __ lea(rscratch2, Address(base, disp));
 3225         __ stlxr(rscratch1, src_reg, rscratch2);
 3226       } else {
 3227         // TODO
 3228         // should we ever get anything other than this case?
 3229         __ stlxr(rscratch1, src_reg, base);
 3230       }
 3231     } else {
 3232       Register index_reg = as_Register(index);
 3233       if (disp == 0) {
 3234         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3235         __ stlxr(rscratch1, src_reg, rscratch2);
 3236       } else {
 3237         __ lea(rscratch2, Address(base, disp));
 3238         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3239         __ stlxr(rscratch1, src_reg, rscratch2);
 3240       }
 3241     }
 3242     __ cmpw(rscratch1, zr);
 3243   %}
 3244 
 3245   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3246     C2_MacroAssembler _masm(&cbuf);
 3247     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3248     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3249                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3250                /*weak*/ false, noreg);
 3251   %}
 3252 
 3253   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3254     C2_MacroAssembler _masm(&cbuf);
 3255     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3256     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3257                Assembler::word, /*acquire*/ false, /*release*/ true,
 3258                /*weak*/ false, noreg);
 3259   %}
 3260 
 3261   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3262     C2_MacroAssembler _masm(&cbuf);
 3263     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3264     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3265                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3266                /*weak*/ false, noreg);
 3267   %}
 3268 
 3269   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3270     C2_MacroAssembler _masm(&cbuf);
 3271     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3272     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3273                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3274                /*weak*/ false, noreg);
 3275   %}
 3276 
 3277 
 3278   // The only difference between aarch64_enc_cmpxchg and
 3279   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3280   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3281   // lock.
 3282   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3283     C2_MacroAssembler _masm(&cbuf);
 3284     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3285     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3286                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3287                /*weak*/ false, noreg);
 3288   %}
 3289 
 3290   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3291     C2_MacroAssembler _masm(&cbuf);
 3292     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3293     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3294                Assembler::word, /*acquire*/ true, /*release*/ true,
 3295                /*weak*/ false, noreg);
 3296   %}
 3297 
 3298   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3299     C2_MacroAssembler _masm(&cbuf);
 3300     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3301     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3302                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3303                /*weak*/ false, noreg);
 3304   %}
 3305 
 3306   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3307     C2_MacroAssembler _masm(&cbuf);
 3308     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3309     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3310                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3311                /*weak*/ false, noreg);
 3312   %}
 3313 
 3314   // auxiliary used for CompareAndSwapX to set result register
 3315   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3316     C2_MacroAssembler _masm(&cbuf);
 3317     Register res_reg = as_Register($res$$reg);
 3318     __ cset(res_reg, Assembler::EQ);
 3319   %}
 3320 
 3321   // prefetch encodings
 3322 
 3323   enc_class aarch64_enc_prefetchw(memory mem) %{
 3324     C2_MacroAssembler _masm(&cbuf);
 3325     Register base = as_Register($mem$$base);
 3326     int index = $mem$$index;
 3327     int scale = $mem$$scale;
 3328     int disp = $mem$$disp;
 3329     if (index == -1) {
 3330       __ prfm(Address(base, disp), PSTL1KEEP);
 3331     } else {
 3332       Register index_reg = as_Register(index);
 3333       if (disp == 0) {
 3334         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3335       } else {
 3336         __ lea(rscratch1, Address(base, disp));
 3337 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3338       }
 3339     }
 3340   %}
 3341 
 3342   /// mov envcodings
 3343 
 3344   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3345     C2_MacroAssembler _masm(&cbuf);
 3346     uint32_t con = (uint32_t)$src$$constant;
 3347     Register dst_reg = as_Register($dst$$reg);
 3348     if (con == 0) {
 3349       __ movw(dst_reg, zr);
 3350     } else {
 3351       __ movw(dst_reg, con);
 3352     }
 3353   %}
 3354 
 3355   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3356     C2_MacroAssembler _masm(&cbuf);
 3357     Register dst_reg = as_Register($dst$$reg);
 3358     uint64_t con = (uint64_t)$src$$constant;
 3359     if (con == 0) {
 3360       __ mov(dst_reg, zr);
 3361     } else {
 3362       __ mov(dst_reg, con);
 3363     }
 3364   %}
 3365 
 3366   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3367     C2_MacroAssembler _masm(&cbuf);
 3368     Register dst_reg = as_Register($dst$$reg);
 3369     address con = (address)$src$$constant;
 3370     if (con == NULL || con == (address)1) {
 3371       ShouldNotReachHere();
 3372     } else {
 3373       relocInfo::relocType rtype = $src->constant_reloc();
 3374       if (rtype == relocInfo::oop_type) {
 3375         __ movoop(dst_reg, (jobject)con);
 3376       } else if (rtype == relocInfo::metadata_type) {
 3377         __ mov_metadata(dst_reg, (Metadata*)con);
 3378       } else {
 3379         assert(rtype == relocInfo::none, "unexpected reloc type");
 3380         if (con < (address)(uintptr_t)os::vm_page_size()) {
 3381           __ mov(dst_reg, con);
 3382         } else {
 3383           uint64_t offset;
 3384           __ adrp(dst_reg, con, offset);
 3385           __ add(dst_reg, dst_reg, offset);
 3386         }
 3387       }
 3388     }
 3389   %}
 3390 
 3391   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3392     C2_MacroAssembler _masm(&cbuf);
 3393     Register dst_reg = as_Register($dst$$reg);
 3394     __ mov(dst_reg, zr);
 3395   %}
 3396 
 3397   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3398     C2_MacroAssembler _masm(&cbuf);
 3399     Register dst_reg = as_Register($dst$$reg);
 3400     __ mov(dst_reg, (uint64_t)1);
 3401   %}
 3402 
 3403   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3404     C2_MacroAssembler _masm(&cbuf);
 3405     __ load_byte_map_base($dst$$Register);
 3406   %}
 3407 
 3408   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3409     C2_MacroAssembler _masm(&cbuf);
 3410     Register dst_reg = as_Register($dst$$reg);
 3411     address con = (address)$src$$constant;
 3412     if (con == NULL) {
 3413       ShouldNotReachHere();
 3414     } else {
 3415       relocInfo::relocType rtype = $src->constant_reloc();
 3416       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3417       __ set_narrow_oop(dst_reg, (jobject)con);
 3418     }
 3419   %}
 3420 
 3421   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3422     C2_MacroAssembler _masm(&cbuf);
 3423     Register dst_reg = as_Register($dst$$reg);
 3424     __ mov(dst_reg, zr);
 3425   %}
 3426 
 3427   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3428     C2_MacroAssembler _masm(&cbuf);
 3429     Register dst_reg = as_Register($dst$$reg);
 3430     address con = (address)$src$$constant;
 3431     if (con == NULL) {
 3432       ShouldNotReachHere();
 3433     } else {
 3434       relocInfo::relocType rtype = $src->constant_reloc();
 3435       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3436       __ set_narrow_klass(dst_reg, (Klass *)con);
 3437     }
 3438   %}
 3439 
 3440   // arithmetic encodings
 3441 
 3442   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3443     C2_MacroAssembler _masm(&cbuf);
 3444     Register dst_reg = as_Register($dst$$reg);
 3445     Register src_reg = as_Register($src1$$reg);
 3446     int32_t con = (int32_t)$src2$$constant;
 3447     // add has primary == 0, subtract has primary == 1
 3448     if ($primary) { con = -con; }
 3449     if (con < 0) {
 3450       __ subw(dst_reg, src_reg, -con);
 3451     } else {
 3452       __ addw(dst_reg, src_reg, con);
 3453     }
 3454   %}
 3455 
 3456   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3457     C2_MacroAssembler _masm(&cbuf);
 3458     Register dst_reg = as_Register($dst$$reg);
 3459     Register src_reg = as_Register($src1$$reg);
 3460     int32_t con = (int32_t)$src2$$constant;
 3461     // add has primary == 0, subtract has primary == 1
 3462     if ($primary) { con = -con; }
 3463     if (con < 0) {
 3464       __ sub(dst_reg, src_reg, -con);
 3465     } else {
 3466       __ add(dst_reg, src_reg, con);
 3467     }
 3468   %}
 3469 
 3470   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3471     C2_MacroAssembler _masm(&cbuf);
 3472    Register dst_reg = as_Register($dst$$reg);
 3473    Register src1_reg = as_Register($src1$$reg);
 3474    Register src2_reg = as_Register($src2$$reg);
 3475     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3476   %}
 3477 
 3478   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3479     C2_MacroAssembler _masm(&cbuf);
 3480    Register dst_reg = as_Register($dst$$reg);
 3481    Register src1_reg = as_Register($src1$$reg);
 3482    Register src2_reg = as_Register($src2$$reg);
 3483     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3484   %}
 3485 
 3486   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3487     C2_MacroAssembler _masm(&cbuf);
 3488    Register dst_reg = as_Register($dst$$reg);
 3489    Register src1_reg = as_Register($src1$$reg);
 3490    Register src2_reg = as_Register($src2$$reg);
 3491     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3492   %}
 3493 
 3494   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3495     C2_MacroAssembler _masm(&cbuf);
 3496    Register dst_reg = as_Register($dst$$reg);
 3497    Register src1_reg = as_Register($src1$$reg);
 3498    Register src2_reg = as_Register($src2$$reg);
 3499     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3500   %}
 3501 
 3502   // compare instruction encodings
 3503 
 3504   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3505     C2_MacroAssembler _masm(&cbuf);
 3506     Register reg1 = as_Register($src1$$reg);
 3507     Register reg2 = as_Register($src2$$reg);
 3508     __ cmpw(reg1, reg2);
 3509   %}
 3510 
 3511   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3512     C2_MacroAssembler _masm(&cbuf);
 3513     Register reg = as_Register($src1$$reg);
 3514     int32_t val = $src2$$constant;
 3515     if (val >= 0) {
 3516       __ subsw(zr, reg, val);
 3517     } else {
 3518       __ addsw(zr, reg, -val);
 3519     }
 3520   %}
 3521 
 3522   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3523     C2_MacroAssembler _masm(&cbuf);
 3524     Register reg1 = as_Register($src1$$reg);
 3525     uint32_t val = (uint32_t)$src2$$constant;
 3526     __ movw(rscratch1, val);
 3527     __ cmpw(reg1, rscratch1);
 3528   %}
 3529 
 3530   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3531     C2_MacroAssembler _masm(&cbuf);
 3532     Register reg1 = as_Register($src1$$reg);
 3533     Register reg2 = as_Register($src2$$reg);
 3534     __ cmp(reg1, reg2);
 3535   %}
 3536 
 3537   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3538     C2_MacroAssembler _masm(&cbuf);
 3539     Register reg = as_Register($src1$$reg);
 3540     int64_t val = $src2$$constant;
 3541     if (val >= 0) {
 3542       __ subs(zr, reg, val);
 3543     } else if (val != -val) {
 3544       __ adds(zr, reg, -val);
 3545     } else {
 3546     // aargh, Long.MIN_VALUE is a special case
 3547       __ orr(rscratch1, zr, (uint64_t)val);
 3548       __ subs(zr, reg, rscratch1);
 3549     }
 3550   %}
 3551 
 3552   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3553     C2_MacroAssembler _masm(&cbuf);
 3554     Register reg1 = as_Register($src1$$reg);
 3555     uint64_t val = (uint64_t)$src2$$constant;
 3556     __ mov(rscratch1, val);
 3557     __ cmp(reg1, rscratch1);
 3558   %}
 3559 
 3560   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3561     C2_MacroAssembler _masm(&cbuf);
 3562     Register reg1 = as_Register($src1$$reg);
 3563     Register reg2 = as_Register($src2$$reg);
 3564     __ cmp(reg1, reg2);
 3565   %}
 3566 
 3567   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3568     C2_MacroAssembler _masm(&cbuf);
 3569     Register reg1 = as_Register($src1$$reg);
 3570     Register reg2 = as_Register($src2$$reg);
 3571     __ cmpw(reg1, reg2);
 3572   %}
 3573 
 3574   enc_class aarch64_enc_testp(iRegP src) %{
 3575     C2_MacroAssembler _masm(&cbuf);
 3576     Register reg = as_Register($src$$reg);
 3577     __ cmp(reg, zr);
 3578   %}
 3579 
 3580   enc_class aarch64_enc_testn(iRegN src) %{
 3581     C2_MacroAssembler _masm(&cbuf);
 3582     Register reg = as_Register($src$$reg);
 3583     __ cmpw(reg, zr);
 3584   %}
 3585 
 3586   enc_class aarch64_enc_b(label lbl) %{
 3587     C2_MacroAssembler _masm(&cbuf);
 3588     Label *L = $lbl$$label;
 3589     __ b(*L);
 3590   %}
 3591 
 3592   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3593     C2_MacroAssembler _masm(&cbuf);
 3594     Label *L = $lbl$$label;
 3595     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3596   %}
 3597 
 3598   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3599     C2_MacroAssembler _masm(&cbuf);
 3600     Label *L = $lbl$$label;
 3601     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3602   %}
 3603 
 3604   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3605   %{
 3606      Register sub_reg = as_Register($sub$$reg);
 3607      Register super_reg = as_Register($super$$reg);
 3608      Register temp_reg = as_Register($temp$$reg);
 3609      Register result_reg = as_Register($result$$reg);
 3610 
 3611      Label miss;
 3612      C2_MacroAssembler _masm(&cbuf);
 3613      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3614                                      NULL, &miss,
 3615                                      /*set_cond_codes:*/ true);
 3616      if ($primary) {
 3617        __ mov(result_reg, zr);
 3618      }
 3619      __ bind(miss);
 3620   %}
 3621 
 3622   enc_class aarch64_enc_java_static_call(method meth) %{
 3623     C2_MacroAssembler _masm(&cbuf);
 3624 
 3625     address addr = (address)$meth$$method;
 3626     address call;
 3627     if (!_method) {
 3628       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3629       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
 3630       if (call == NULL) {
 3631         ciEnv::current()->record_failure("CodeCache is full");
 3632         return;
 3633       }
 3634     } else {
 3635       int method_index = resolved_method_index(cbuf);
 3636       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3637                                                   : static_call_Relocation::spec(method_index);
 3638       call = __ trampoline_call(Address(addr, rspec));
 3639       if (call == NULL) {
 3640         ciEnv::current()->record_failure("CodeCache is full");
 3641         return;
 3642       }
 3643       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 3644         // Calls of the same statically bound method can share
 3645         // a stub to the interpreter.
 3646         cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
 3647       } else {
 3648         // Emit stub for static call
 3649         address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
 3650         if (stub == NULL) {
 3651           ciEnv::current()->record_failure("CodeCache is full");
 3652           return;
 3653         }
 3654       }
 3655     }
 3656 
 3657     __ post_call_nop();
 3658 
 3659     // Only non uncommon_trap calls need to reinitialize ptrue.
 3660     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3661       __ reinitialize_ptrue();
 3662     }
 3663   %}
 3664 
 3665   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3666     C2_MacroAssembler _masm(&cbuf);
 3667     int method_index = resolved_method_index(cbuf);
 3668     address call = __ ic_call((address)$meth$$method, method_index);
 3669     if (call == NULL) {
 3670       ciEnv::current()->record_failure("CodeCache is full");
 3671       return;
 3672     }
 3673     __ post_call_nop();
 3674     if (Compile::current()->max_vector_size() > 0) {
 3675       __ reinitialize_ptrue();
 3676     }
 3677   %}
 3678 
 3679   enc_class aarch64_enc_call_epilog() %{
 3680     C2_MacroAssembler _masm(&cbuf);
 3681     if (VerifyStackAtCalls) {
 3682       // Check that stack depth is unchanged: find majik cookie on stack
 3683       __ call_Unimplemented();
 3684     }
 3685   %}
 3686 
 3687   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3688     C2_MacroAssembler _masm(&cbuf);
 3689 
 3690     // some calls to generated routines (arraycopy code) are scheduled
 3691     // by C2 as runtime calls. if so we can call them using a br (they
 3692     // will be in a reachable segment) otherwise we have to use a blr
 3693     // which loads the absolute address into a register.
 3694     address entry = (address)$meth$$method;
 3695     CodeBlob *cb = CodeCache::find_blob(entry);
 3696     if (cb) {
 3697       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3698       if (call == NULL) {
 3699         ciEnv::current()->record_failure("CodeCache is full");
 3700         return;
 3701       }
 3702       __ post_call_nop();
 3703     } else {
 3704       Label retaddr;
 3705       __ adr(rscratch2, retaddr);
 3706       __ lea(rscratch1, RuntimeAddress(entry));
 3707       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3708       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3709       __ blr(rscratch1);
 3710       __ bind(retaddr);
 3711       __ post_call_nop();
 3712       __ add(sp, sp, 2 * wordSize);
 3713     }
 3714     if (Compile::current()->max_vector_size() > 0) {
 3715       __ reinitialize_ptrue();
 3716     }
 3717   %}
 3718 
 3719   enc_class aarch64_enc_rethrow() %{
 3720     C2_MacroAssembler _masm(&cbuf);
 3721     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3722   %}
 3723 
 3724   enc_class aarch64_enc_ret() %{
 3725     C2_MacroAssembler _masm(&cbuf);
 3726 #ifdef ASSERT
 3727     if (Compile::current()->max_vector_size() > 0) {
 3728       __ verify_ptrue();
 3729     }
 3730 #endif
 3731     __ ret(lr);
 3732   %}
 3733 
 3734   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3735     C2_MacroAssembler _masm(&cbuf);
 3736     Register target_reg = as_Register($jump_target$$reg);
 3737     __ br(target_reg);
 3738   %}
 3739 
 3740   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3741     C2_MacroAssembler _masm(&cbuf);
 3742     Register target_reg = as_Register($jump_target$$reg);
 3743     // exception oop should be in r0
 3744     // ret addr has been popped into lr
 3745     // callee expects it in r3
 3746     __ mov(r3, lr);
 3747     __ br(target_reg);
 3748   %}
 3749 
 3750   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3751     C2_MacroAssembler _masm(&cbuf);
 3752     Register oop = as_Register($object$$reg);
 3753     Register box = as_Register($box$$reg);
 3754     Register disp_hdr = as_Register($tmp$$reg);
 3755     Register tmp = as_Register($tmp2$$reg);
 3756     Label cont;
 3757     Label object_has_monitor;
 3758     Label no_count;
 3759 
 3760     assert_different_registers(oop, box, tmp, disp_hdr);
 3761 
 3762     // Load markWord from object into displaced_header.
 3763     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3764 
 3765     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3766       __ load_klass(tmp, oop);
 3767       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3768       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3769       __ br(Assembler::NE, cont);
 3770     }
 3771 
 3772     // Check for existing monitor
 3773     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3774 
 3775     if (!UseHeavyMonitors) {
 3776       // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3777       __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3778 
 3779       // Initialize the box. (Must happen before we update the object mark!)
 3780       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3781 
 3782       // Compare object markWord with an unlocked value (tmp) and if
 3783       // equal exchange the stack address of our box with object markWord.
 3784       // On failure disp_hdr contains the possibly locked markWord.
 3785       __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3786                  /*release*/ true, /*weak*/ false, disp_hdr);
 3787       __ br(Assembler::EQ, cont);
 3788 
 3789       assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3790 
 3791       // If the compare-and-exchange succeeded, then we found an unlocked
 3792       // object, will have now locked it will continue at label cont
 3793 
 3794       // Check if the owner is self by comparing the value in the
 3795       // markWord of object (disp_hdr) with the stack pointer.
 3796       __ mov(rscratch1, sp);
 3797       __ sub(disp_hdr, disp_hdr, rscratch1);
 3798       __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3799       // If condition is true we are cont and hence we can store 0 as the
 3800       // displaced header in the box, which indicates that it is a recursive lock.
 3801       __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3802       __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3803     } else {
 3804       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3805     }
 3806     __ b(cont);
 3807 
 3808     // Handle existing monitor.
 3809     __ bind(object_has_monitor);
 3810 
 3811     // The object's monitor m is unlocked iff m->owner == NULL,
 3812     // otherwise m->owner may contain a thread or a stack address.
 3813     //
 3814     // Try to CAS m->owner from NULL to current thread.
 3815     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3816     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3817                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 3818 
 3819     // Store a non-null value into the box to avoid looking like a re-entrant
 3820     // lock. The fast-path monitor unlock code checks for
 3821     // markWord::monitor_value so use markWord::unused_mark which has the
 3822     // relevant bit set, and also matches ObjectSynchronizer::enter.
 3823     __ mov(tmp, (address)markWord::unused_mark().value());
 3824     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3825 
 3826     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 3827 
 3828     __ cmp(rscratch1, rthread);
 3829     __ br(Assembler::NE, cont); // Check for recursive locking
 3830 
 3831     // Recursive lock case
 3832     __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
 3833     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 3834 
 3835     __ bind(cont);
 3836     // flag == EQ indicates success
 3837     // flag == NE indicates failure
 3838     __ br(Assembler::NE, no_count);
 3839 
 3840     __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
 3841 
 3842     __ bind(no_count);
 3843   %}
 3844 
 3845   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3846     C2_MacroAssembler _masm(&cbuf);
 3847     Register oop = as_Register($object$$reg);
 3848     Register box = as_Register($box$$reg);
 3849     Register disp_hdr = as_Register($tmp$$reg);
 3850     Register tmp = as_Register($tmp2$$reg);
 3851     Label cont;
 3852     Label object_has_monitor;
 3853     Label no_count;
 3854 
 3855     assert_different_registers(oop, box, tmp, disp_hdr);
 3856 
 3857     if (!UseHeavyMonitors) {
 3858       // Find the lock address and load the displaced header from the stack.
 3859       __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3860 
 3861       // If the displaced header is 0, we have a recursive unlock.
 3862       __ cmp(disp_hdr, zr);
 3863       __ br(Assembler::EQ, cont);
 3864     }
 3865 
 3866     // Handle existing monitor.
 3867     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 3868     __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
 3869 
 3870     if (!UseHeavyMonitors) {
 3871       // Check if it is still a light weight lock, this is is true if we
 3872       // see the stack address of the basicLock in the markWord of the
 3873       // object.
 3874 
 3875       __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 3876                  /*release*/ true, /*weak*/ false, tmp);
 3877     } else {
 3878       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3879     }
 3880     __ b(cont);
 3881 
 3882     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3883 
 3884     // Handle existing monitor.
 3885     __ bind(object_has_monitor);
 3886     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 3887     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 3888     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3889 
 3890     Label notRecursive;
 3891     __ cbz(disp_hdr, notRecursive);
 3892 
 3893     // Recursive lock
 3894     __ sub(disp_hdr, disp_hdr, 1u);
 3895     __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3896     __ cmp(disp_hdr, disp_hdr); // Sets flags for result
 3897     __ b(cont);
 3898 
 3899     __ bind(notRecursive);
 3900     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 3901     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 3902     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 3903     __ cmp(rscratch1, zr); // Sets flags for result
 3904     __ cbnz(rscratch1, cont);
 3905     // need a release store here
 3906     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3907     __ stlr(zr, tmp); // set unowned
 3908 
 3909     __ bind(cont);
 3910     // flag == EQ indicates success
 3911     // flag == NE indicates failure
 3912     __ br(Assembler::NE, no_count);
 3913 
 3914     __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
 3915 
 3916     __ bind(no_count);
 3917   %}
 3918 
 3919 %}
 3920 
 3921 //----------FRAME--------------------------------------------------------------
 3922 // Definition of frame structure and management information.
 3923 //
 3924 //  S T A C K   L A Y O U T    Allocators stack-slot number
 3925 //                             |   (to get allocators register number
 3926 //  G  Owned by    |        |  v    add OptoReg::stack0())
 3927 //  r   CALLER     |        |
 3928 //  o     |        +--------+      pad to even-align allocators stack-slot
 3929 //  w     V        |  pad0  |        numbers; owned by CALLER
 3930 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 3931 //  h     ^        |   in   |  5
 3932 //        |        |  args  |  4   Holes in incoming args owned by SELF
 3933 //  |     |        |        |  3
 3934 //  |     |        +--------+
 3935 //  V     |        | old out|      Empty on Intel, window on Sparc
 3936 //        |    old |preserve|      Must be even aligned.
 3937 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 3938 //        |        |   in   |  3   area for Intel ret address
 3939 //     Owned by    |preserve|      Empty on Sparc.
 3940 //       SELF      +--------+
 3941 //        |        |  pad2  |  2   pad to align old SP
 3942 //        |        +--------+  1
 3943 //        |        | locks  |  0
 3944 //        |        +--------+----> OptoReg::stack0(), even aligned
 3945 //        |        |  pad1  | 11   pad to align new SP
 3946 //        |        +--------+
 3947 //        |        |        | 10
 3948 //        |        | spills |  9   spills
 3949 //        V        |        |  8   (pad0 slot for callee)
 3950 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 3951 //        ^        |  out   |  7
 3952 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 3953 //     Owned by    +--------+
 3954 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 3955 //        |    new |preserve|      Must be even-aligned.
 3956 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 3957 //        |        |        |
 3958 //
 3959 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 3960 //         known from SELF's arguments and the Java calling convention.
 3961 //         Region 6-7 is determined per call site.
 3962 // Note 2: If the calling convention leaves holes in the incoming argument
 3963 //         area, those holes are owned by SELF.  Holes in the outgoing area
 3964 //         are owned by the CALLEE.  Holes should not be necessary in the
 3965 //         incoming area, as the Java calling convention is completely under
 3966 //         the control of the AD file.  Doubles can be sorted and packed to
 3967 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 3968 //         varargs C calling conventions.
 3969 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 3970 //         even aligned with pad0 as needed.
 3971 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 3972 //           (the latter is true on Intel but is it false on AArch64?)
 3973 //         region 6-11 is even aligned; it may be padded out more so that
 3974 //         the region from SP to FP meets the minimum stack alignment.
 3975 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 3976 //         alignment.  Region 11, pad1, may be dynamically extended so that
 3977 //         SP meets the minimum alignment.
 3978 
 3979 frame %{
 3980   // These three registers define part of the calling convention
 3981   // between compiled code and the interpreter.
 3982 
 3983   // Inline Cache Register or Method for I2C.
 3984   inline_cache_reg(R12);
 3985 
 3986   // Number of stack slots consumed by locking an object
 3987   sync_stack_slots(2);
 3988 
 3989   // Compiled code's Frame Pointer
 3990   frame_pointer(R31);
 3991 
 3992   // Interpreter stores its frame pointer in a register which is
 3993   // stored to the stack by I2CAdaptors.
 3994   // I2CAdaptors convert from interpreted java to compiled java.
 3995   interpreter_frame_pointer(R29);
 3996 
 3997   // Stack alignment requirement
 3998   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 3999 
 4000   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4001   // for calls to C.  Supports the var-args backing area for register parms.
 4002   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4003 
 4004   // The after-PROLOG location of the return address.  Location of
 4005   // return address specifies a type (REG or STACK) and a number
 4006   // representing the register number (i.e. - use a register name) or
 4007   // stack slot.
 4008   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4009   // Otherwise, it is above the locks and verification slot and alignment word
 4010   // TODO this may well be correct but need to check why that - 2 is there
 4011   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4012   // which folds in the space used for monitors
 4013   return_addr(STACK - 2 +
 4014               align_up((Compile::current()->in_preserve_stack_slots() +
 4015                         Compile::current()->fixed_slots()),
 4016                        stack_alignment_in_slots()));
 4017 
 4018   // Location of compiled Java return values.  Same as C for now.
 4019   return_value
 4020   %{
 4021     // TODO do we allow ideal_reg == Op_RegN???
 4022     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4023            "only return normal values");
 4024 
 4025     static const int lo[Op_RegL + 1] = { // enum name
 4026       0,                                 // Op_Node
 4027       0,                                 // Op_Set
 4028       R0_num,                            // Op_RegN
 4029       R0_num,                            // Op_RegI
 4030       R0_num,                            // Op_RegP
 4031       V0_num,                            // Op_RegF
 4032       V0_num,                            // Op_RegD
 4033       R0_num                             // Op_RegL
 4034     };
 4035 
 4036     static const int hi[Op_RegL + 1] = { // enum name
 4037       0,                                 // Op_Node
 4038       0,                                 // Op_Set
 4039       OptoReg::Bad,                      // Op_RegN
 4040       OptoReg::Bad,                      // Op_RegI
 4041       R0_H_num,                          // Op_RegP
 4042       OptoReg::Bad,                      // Op_RegF
 4043       V0_H_num,                          // Op_RegD
 4044       R0_H_num                           // Op_RegL
 4045     };
 4046 
 4047     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4048   %}
 4049 %}
 4050 
 4051 //----------ATTRIBUTES---------------------------------------------------------
 4052 //----------Operand Attributes-------------------------------------------------
 4053 op_attrib op_cost(1);        // Required cost attribute
 4054 
 4055 //----------Instruction Attributes---------------------------------------------
 4056 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4057 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4058 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4059                                 // a non-matching short branch variant
 4060                                 // of some long branch?
 4061 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4062                                 // be a power of 2) specifies the
 4063                                 // alignment that some part of the
 4064                                 // instruction (not necessarily the
 4065                                 // start) requires.  If > 1, a
 4066                                 // compute_padding() function must be
 4067                                 // provided for the instruction
 4068 
 4069 //----------OPERANDS-----------------------------------------------------------
 4070 // Operand definitions must precede instruction definitions for correct parsing
 4071 // in the ADLC because operands constitute user defined types which are used in
 4072 // instruction definitions.
 4073 
 4074 //----------Simple Operands----------------------------------------------------
 4075 
 4076 // Integer operands 32 bit
 4077 // 32 bit immediate
 4078 operand immI()
 4079 %{
 4080   match(ConI);
 4081 
 4082   op_cost(0);
 4083   format %{ %}
 4084   interface(CONST_INTER);
 4085 %}
 4086 
 4087 // 32 bit zero
 4088 operand immI0()
 4089 %{
 4090   predicate(n->get_int() == 0);
 4091   match(ConI);
 4092 
 4093   op_cost(0);
 4094   format %{ %}
 4095   interface(CONST_INTER);
 4096 %}
 4097 
 4098 // 32 bit unit increment
 4099 operand immI_1()
 4100 %{
 4101   predicate(n->get_int() == 1);
 4102   match(ConI);
 4103 
 4104   op_cost(0);
 4105   format %{ %}
 4106   interface(CONST_INTER);
 4107 %}
 4108 
 4109 // 32 bit unit decrement
 4110 operand immI_M1()
 4111 %{
 4112   predicate(n->get_int() == -1);
 4113   match(ConI);
 4114 
 4115   op_cost(0);
 4116   format %{ %}
 4117   interface(CONST_INTER);
 4118 %}
 4119 
 4120 // Shift values for add/sub extension shift
 4121 operand immIExt()
 4122 %{
 4123   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4124   match(ConI);
 4125 
 4126   op_cost(0);
 4127   format %{ %}
 4128   interface(CONST_INTER);
 4129 %}
 4130 
 4131 operand immI_gt_1()
 4132 %{
 4133   predicate(n->get_int() > 1);
 4134   match(ConI);
 4135 
 4136   op_cost(0);
 4137   format %{ %}
 4138   interface(CONST_INTER);
 4139 %}
 4140 
 4141 operand immI_le_4()
 4142 %{
 4143   predicate(n->get_int() <= 4);
 4144   match(ConI);
 4145 
 4146   op_cost(0);
 4147   format %{ %}
 4148   interface(CONST_INTER);
 4149 %}
 4150 
 4151 operand immI_16()
 4152 %{
 4153   predicate(n->get_int() == 16);
 4154   match(ConI);
 4155 
 4156   op_cost(0);
 4157   format %{ %}
 4158   interface(CONST_INTER);
 4159 %}
 4160 
 4161 operand immI_24()
 4162 %{
 4163   predicate(n->get_int() == 24);
 4164   match(ConI);
 4165 
 4166   op_cost(0);
 4167   format %{ %}
 4168   interface(CONST_INTER);
 4169 %}
 4170 
 4171 operand immI_32()
 4172 %{
 4173   predicate(n->get_int() == 32);
 4174   match(ConI);
 4175 
 4176   op_cost(0);
 4177   format %{ %}
 4178   interface(CONST_INTER);
 4179 %}
 4180 
 4181 operand immI_48()
 4182 %{
 4183   predicate(n->get_int() == 48);
 4184   match(ConI);
 4185 
 4186   op_cost(0);
 4187   format %{ %}
 4188   interface(CONST_INTER);
 4189 %}
 4190 
 4191 operand immI_56()
 4192 %{
 4193   predicate(n->get_int() == 56);
 4194   match(ConI);
 4195 
 4196   op_cost(0);
 4197   format %{ %}
 4198   interface(CONST_INTER);
 4199 %}
 4200 
 4201 operand immI_63()
 4202 %{
 4203   predicate(n->get_int() == 63);
 4204   match(ConI);
 4205 
 4206   op_cost(0);
 4207   format %{ %}
 4208   interface(CONST_INTER);
 4209 %}
 4210 
 4211 operand immI_64()
 4212 %{
 4213   predicate(n->get_int() == 64);
 4214   match(ConI);
 4215 
 4216   op_cost(0);
 4217   format %{ %}
 4218   interface(CONST_INTER);
 4219 %}
 4220 
 4221 operand immI_255()
 4222 %{
 4223   predicate(n->get_int() == 255);
 4224   match(ConI);
 4225 
 4226   op_cost(0);
 4227   format %{ %}
 4228   interface(CONST_INTER);
 4229 %}
 4230 
 4231 operand immI_65535()
 4232 %{
 4233   predicate(n->get_int() == 65535);
 4234   match(ConI);
 4235 
 4236   op_cost(0);
 4237   format %{ %}
 4238   interface(CONST_INTER);
 4239 %}
 4240 
 4241 operand immI_positive()
 4242 %{
 4243   predicate(n->get_int() > 0);
 4244   match(ConI);
 4245 
 4246   op_cost(0);
 4247   format %{ %}
 4248   interface(CONST_INTER);
 4249 %}
 4250 
 4251 operand immL_255()
 4252 %{
 4253   predicate(n->get_long() == 255L);
 4254   match(ConL);
 4255 
 4256   op_cost(0);
 4257   format %{ %}
 4258   interface(CONST_INTER);
 4259 %}
 4260 
 4261 operand immL_65535()
 4262 %{
 4263   predicate(n->get_long() == 65535L);
 4264   match(ConL);
 4265 
 4266   op_cost(0);
 4267   format %{ %}
 4268   interface(CONST_INTER);
 4269 %}
 4270 
 4271 operand immL_4294967295()
 4272 %{
 4273   predicate(n->get_long() == 4294967295L);
 4274   match(ConL);
 4275 
 4276   op_cost(0);
 4277   format %{ %}
 4278   interface(CONST_INTER);
 4279 %}
 4280 
 4281 operand immL_bitmask()
 4282 %{
 4283   predicate((n->get_long() != 0)
 4284             && ((n->get_long() & 0xc000000000000000l) == 0)
 4285             && is_power_of_2(n->get_long() + 1));
 4286   match(ConL);
 4287 
 4288   op_cost(0);
 4289   format %{ %}
 4290   interface(CONST_INTER);
 4291 %}
 4292 
 4293 operand immI_bitmask()
 4294 %{
 4295   predicate((n->get_int() != 0)
 4296             && ((n->get_int() & 0xc0000000) == 0)
 4297             && is_power_of_2(n->get_int() + 1));
 4298   match(ConI);
 4299 
 4300   op_cost(0);
 4301   format %{ %}
 4302   interface(CONST_INTER);
 4303 %}
 4304 
 4305 operand immL_positive_bitmaskI()
 4306 %{
 4307   predicate((n->get_long() != 0)
 4308             && ((julong)n->get_long() < 0x80000000ULL)
 4309             && is_power_of_2(n->get_long() + 1));
 4310   match(ConL);
 4311 
 4312   op_cost(0);
 4313   format %{ %}
 4314   interface(CONST_INTER);
 4315 %}
 4316 
 4317 // Scale values for scaled offset addressing modes (up to long but not quad)
 4318 operand immIScale()
 4319 %{
 4320   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4321   match(ConI);
 4322 
 4323   op_cost(0);
 4324   format %{ %}
 4325   interface(CONST_INTER);
 4326 %}
 4327 
 4328 // 26 bit signed offset -- for pc-relative branches
 4329 operand immI26()
 4330 %{
 4331   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4332   match(ConI);
 4333 
 4334   op_cost(0);
 4335   format %{ %}
 4336   interface(CONST_INTER);
 4337 %}
 4338 
 4339 // 19 bit signed offset -- for pc-relative loads
 4340 operand immI19()
 4341 %{
 4342   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4343   match(ConI);
 4344 
 4345   op_cost(0);
 4346   format %{ %}
 4347   interface(CONST_INTER);
 4348 %}
 4349 
 4350 // 12 bit unsigned offset -- for base plus immediate loads
 4351 operand immIU12()
 4352 %{
 4353   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4354   match(ConI);
 4355 
 4356   op_cost(0);
 4357   format %{ %}
 4358   interface(CONST_INTER);
 4359 %}
 4360 
 4361 operand immLU12()
 4362 %{
 4363   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4364   match(ConL);
 4365 
 4366   op_cost(0);
 4367   format %{ %}
 4368   interface(CONST_INTER);
 4369 %}
 4370 
 4371 // Offset for scaled or unscaled immediate loads and stores
 4372 operand immIOffset()
 4373 %{
 4374   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4375   match(ConI);
 4376 
 4377   op_cost(0);
 4378   format %{ %}
 4379   interface(CONST_INTER);
 4380 %}
 4381 
 4382 operand immIOffset1()
 4383 %{
 4384   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4385   match(ConI);
 4386 
 4387   op_cost(0);
 4388   format %{ %}
 4389   interface(CONST_INTER);
 4390 %}
 4391 
 4392 operand immIOffset2()
 4393 %{
 4394   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4395   match(ConI);
 4396 
 4397   op_cost(0);
 4398   format %{ %}
 4399   interface(CONST_INTER);
 4400 %}
 4401 
 4402 operand immIOffset4()
 4403 %{
 4404   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4405   match(ConI);
 4406 
 4407   op_cost(0);
 4408   format %{ %}
 4409   interface(CONST_INTER);
 4410 %}
 4411 
 4412 operand immIOffset8()
 4413 %{
 4414   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4415   match(ConI);
 4416 
 4417   op_cost(0);
 4418   format %{ %}
 4419   interface(CONST_INTER);
 4420 %}
 4421 
 4422 operand immIOffset16()
 4423 %{
 4424   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4425   match(ConI);
 4426 
 4427   op_cost(0);
 4428   format %{ %}
 4429   interface(CONST_INTER);
 4430 %}
 4431 
 4432 operand immLoffset()
 4433 %{
 4434   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4435   match(ConL);
 4436 
 4437   op_cost(0);
 4438   format %{ %}
 4439   interface(CONST_INTER);
 4440 %}
 4441 
 4442 operand immLoffset1()
 4443 %{
 4444   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4445   match(ConL);
 4446 
 4447   op_cost(0);
 4448   format %{ %}
 4449   interface(CONST_INTER);
 4450 %}
 4451 
 4452 operand immLoffset2()
 4453 %{
 4454   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4455   match(ConL);
 4456 
 4457   op_cost(0);
 4458   format %{ %}
 4459   interface(CONST_INTER);
 4460 %}
 4461 
 4462 operand immLoffset4()
 4463 %{
 4464   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4465   match(ConL);
 4466 
 4467   op_cost(0);
 4468   format %{ %}
 4469   interface(CONST_INTER);
 4470 %}
 4471 
 4472 operand immLoffset8()
 4473 %{
 4474   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4475   match(ConL);
 4476 
 4477   op_cost(0);
 4478   format %{ %}
 4479   interface(CONST_INTER);
 4480 %}
 4481 
 4482 operand immLoffset16()
 4483 %{
 4484   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4485   match(ConL);
 4486 
 4487   op_cost(0);
 4488   format %{ %}
 4489   interface(CONST_INTER);
 4490 %}
 4491 
 4492 // 8 bit signed value.
 4493 operand immI8()
 4494 %{
 4495   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4496   match(ConI);
 4497 
 4498   op_cost(0);
 4499   format %{ %}
 4500   interface(CONST_INTER);
 4501 %}
 4502 
 4503 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4504 operand immI8_shift8()
 4505 %{
 4506   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4507             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4508   match(ConI);
 4509 
 4510   op_cost(0);
 4511   format %{ %}
 4512   interface(CONST_INTER);
 4513 %}
 4514 
 4515 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4516 operand immL8_shift8()
 4517 %{
 4518   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4519             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4520   match(ConL);
 4521 
 4522   op_cost(0);
 4523   format %{ %}
 4524   interface(CONST_INTER);
 4525 %}
 4526 
 4527 // 8 bit integer valid for vector add sub immediate
 4528 operand immBAddSubV()
 4529 %{
 4530   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4531   match(ConI);
 4532 
 4533   op_cost(0);
 4534   format %{ %}
 4535   interface(CONST_INTER);
 4536 %}
 4537 
 4538 // 32 bit integer valid for add sub immediate
 4539 operand immIAddSub()
 4540 %{
 4541   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4542   match(ConI);
 4543   op_cost(0);
 4544   format %{ %}
 4545   interface(CONST_INTER);
 4546 %}
 4547 
 4548 // 32 bit integer valid for vector add sub immediate
 4549 operand immIAddSubV()
 4550 %{
 4551   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4552   match(ConI);
 4553 
 4554   op_cost(0);
 4555   format %{ %}
 4556   interface(CONST_INTER);
 4557 %}
 4558 
 4559 // 32 bit unsigned integer valid for logical immediate
 4560 
 4561 operand immBLog()
 4562 %{
 4563   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4564   match(ConI);
 4565 
 4566   op_cost(0);
 4567   format %{ %}
 4568   interface(CONST_INTER);
 4569 %}
 4570 
 4571 operand immSLog()
 4572 %{
 4573   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4574   match(ConI);
 4575 
 4576   op_cost(0);
 4577   format %{ %}
 4578   interface(CONST_INTER);
 4579 %}
 4580 
 4581 operand immILog()
 4582 %{
 4583   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4584   match(ConI);
 4585 
 4586   op_cost(0);
 4587   format %{ %}
 4588   interface(CONST_INTER);
 4589 %}
 4590 
 4591 // Integer operands 64 bit
 4592 // 64 bit immediate
 4593 operand immL()
 4594 %{
 4595   match(ConL);
 4596 
 4597   op_cost(0);
 4598   format %{ %}
 4599   interface(CONST_INTER);
 4600 %}
 4601 
 4602 // 64 bit zero
 4603 operand immL0()
 4604 %{
 4605   predicate(n->get_long() == 0);
 4606   match(ConL);
 4607 
 4608   op_cost(0);
 4609   format %{ %}
 4610   interface(CONST_INTER);
 4611 %}
 4612 
 4613 // 64 bit unit increment
 4614 operand immL_1()
 4615 %{
 4616   predicate(n->get_long() == 1);
 4617   match(ConL);
 4618 
 4619   op_cost(0);
 4620   format %{ %}
 4621   interface(CONST_INTER);
 4622 %}
 4623 
 4624 // 64 bit unit decrement
 4625 operand immL_M1()
 4626 %{
 4627   predicate(n->get_long() == -1);
 4628   match(ConL);
 4629 
 4630   op_cost(0);
 4631   format %{ %}
 4632   interface(CONST_INTER);
 4633 %}
 4634 
 4635 // 32 bit offset of pc in thread anchor
 4636 
 4637 operand immL_pc_off()
 4638 %{
 4639   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4640                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4641   match(ConL);
 4642 
 4643   op_cost(0);
 4644   format %{ %}
 4645   interface(CONST_INTER);
 4646 %}
 4647 
 4648 // 64 bit integer valid for add sub immediate
 4649 operand immLAddSub()
 4650 %{
 4651   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4652   match(ConL);
 4653   op_cost(0);
 4654   format %{ %}
 4655   interface(CONST_INTER);
 4656 %}
 4657 
 4658 // 64 bit integer valid for addv subv immediate
 4659 operand immLAddSubV()
 4660 %{
 4661   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4662   match(ConL);
 4663 
 4664   op_cost(0);
 4665   format %{ %}
 4666   interface(CONST_INTER);
 4667 %}
 4668 
 4669 // 64 bit integer valid for logical immediate
 4670 operand immLLog()
 4671 %{
 4672   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4673   match(ConL);
 4674   op_cost(0);
 4675   format %{ %}
 4676   interface(CONST_INTER);
 4677 %}
 4678 
 4679 // Long Immediate: low 32-bit mask
 4680 operand immL_32bits()
 4681 %{
 4682   predicate(n->get_long() == 0xFFFFFFFFL);
 4683   match(ConL);
 4684   op_cost(0);
 4685   format %{ %}
 4686   interface(CONST_INTER);
 4687 %}
 4688 
 4689 // Pointer operands
 4690 // Pointer Immediate
 4691 operand immP()
 4692 %{
 4693   match(ConP);
 4694 
 4695   op_cost(0);
 4696   format %{ %}
 4697   interface(CONST_INTER);
 4698 %}
 4699 
 4700 // NULL Pointer Immediate
 4701 operand immP0()
 4702 %{
 4703   predicate(n->get_ptr() == 0);
 4704   match(ConP);
 4705 
 4706   op_cost(0);
 4707   format %{ %}
 4708   interface(CONST_INTER);
 4709 %}
 4710 
 4711 // Pointer Immediate One
 4712 // this is used in object initialization (initial object header)
 4713 operand immP_1()
 4714 %{
 4715   predicate(n->get_ptr() == 1);
 4716   match(ConP);
 4717 
 4718   op_cost(0);
 4719   format %{ %}
 4720   interface(CONST_INTER);
 4721 %}
 4722 
 4723 // Card Table Byte Map Base
 4724 operand immByteMapBase()
 4725 %{
 4726   // Get base of card map
 4727   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4728             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4729   match(ConP);
 4730 
 4731   op_cost(0);
 4732   format %{ %}
 4733   interface(CONST_INTER);
 4734 %}
 4735 
 4736 // Pointer Immediate Minus One
 4737 // this is used when we want to write the current PC to the thread anchor
 4738 operand immP_M1()
 4739 %{
 4740   predicate(n->get_ptr() == -1);
 4741   match(ConP);
 4742 
 4743   op_cost(0);
 4744   format %{ %}
 4745   interface(CONST_INTER);
 4746 %}
 4747 
 4748 // Pointer Immediate Minus Two
 4749 // this is used when we want to write the current PC to the thread anchor
 4750 operand immP_M2()
 4751 %{
 4752   predicate(n->get_ptr() == -2);
 4753   match(ConP);
 4754 
 4755   op_cost(0);
 4756   format %{ %}
 4757   interface(CONST_INTER);
 4758 %}
 4759 
 4760 // Float and Double operands
 4761 // Double Immediate
 4762 operand immD()
 4763 %{
 4764   match(ConD);
 4765   op_cost(0);
 4766   format %{ %}
 4767   interface(CONST_INTER);
 4768 %}
 4769 
 4770 // Double Immediate: +0.0d
 4771 operand immD0()
 4772 %{
 4773   predicate(jlong_cast(n->getd()) == 0);
 4774   match(ConD);
 4775 
 4776   op_cost(0);
 4777   format %{ %}
 4778   interface(CONST_INTER);
 4779 %}
 4780 
 4781 // constant 'double +0.0'.
 4782 operand immDPacked()
 4783 %{
 4784   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4785   match(ConD);
 4786   op_cost(0);
 4787   format %{ %}
 4788   interface(CONST_INTER);
 4789 %}
 4790 
 4791 // Float Immediate
 4792 operand immF()
 4793 %{
 4794   match(ConF);
 4795   op_cost(0);
 4796   format %{ %}
 4797   interface(CONST_INTER);
 4798 %}
 4799 
 4800 // Float Immediate: +0.0f.
 4801 operand immF0()
 4802 %{
 4803   predicate(jint_cast(n->getf()) == 0);
 4804   match(ConF);
 4805 
 4806   op_cost(0);
 4807   format %{ %}
 4808   interface(CONST_INTER);
 4809 %}
 4810 
 4811 //
 4812 operand immFPacked()
 4813 %{
 4814   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4815   match(ConF);
 4816   op_cost(0);
 4817   format %{ %}
 4818   interface(CONST_INTER);
 4819 %}
 4820 
 4821 // Narrow pointer operands
 4822 // Narrow Pointer Immediate
 4823 operand immN()
 4824 %{
 4825   match(ConN);
 4826 
 4827   op_cost(0);
 4828   format %{ %}
 4829   interface(CONST_INTER);
 4830 %}
 4831 
 4832 // Narrow NULL Pointer Immediate
 4833 operand immN0()
 4834 %{
 4835   predicate(n->get_narrowcon() == 0);
 4836   match(ConN);
 4837 
 4838   op_cost(0);
 4839   format %{ %}
 4840   interface(CONST_INTER);
 4841 %}
 4842 
 4843 operand immNKlass()
 4844 %{
 4845   match(ConNKlass);
 4846 
 4847   op_cost(0);
 4848   format %{ %}
 4849   interface(CONST_INTER);
 4850 %}
 4851 
 4852 // Integer 32 bit Register Operands
 4853 // Integer 32 bitRegister (excludes SP)
 4854 operand iRegI()
 4855 %{
 4856   constraint(ALLOC_IN_RC(any_reg32));
 4857   match(RegI);
 4858   match(iRegINoSp);
 4859   op_cost(0);
 4860   format %{ %}
 4861   interface(REG_INTER);
 4862 %}
 4863 
 4864 // Integer 32 bit Register not Special
 4865 operand iRegINoSp()
 4866 %{
 4867   constraint(ALLOC_IN_RC(no_special_reg32));
 4868   match(RegI);
 4869   op_cost(0);
 4870   format %{ %}
 4871   interface(REG_INTER);
 4872 %}
 4873 
 4874 // Integer 64 bit Register Operands
 4875 // Integer 64 bit Register (includes SP)
 4876 operand iRegL()
 4877 %{
 4878   constraint(ALLOC_IN_RC(any_reg));
 4879   match(RegL);
 4880   match(iRegLNoSp);
 4881   op_cost(0);
 4882   format %{ %}
 4883   interface(REG_INTER);
 4884 %}
 4885 
 4886 // Integer 64 bit Register not Special
 4887 operand iRegLNoSp()
 4888 %{
 4889   constraint(ALLOC_IN_RC(no_special_reg));
 4890   match(RegL);
 4891   match(iRegL_R0);
 4892   format %{ %}
 4893   interface(REG_INTER);
 4894 %}
 4895 
 4896 // Pointer Register Operands
 4897 // Pointer Register
 4898 operand iRegP()
 4899 %{
 4900   constraint(ALLOC_IN_RC(ptr_reg));
 4901   match(RegP);
 4902   match(iRegPNoSp);
 4903   match(iRegP_R0);
 4904   //match(iRegP_R2);
 4905   //match(iRegP_R4);
 4906   match(iRegP_R5);
 4907   match(thread_RegP);
 4908   op_cost(0);
 4909   format %{ %}
 4910   interface(REG_INTER);
 4911 %}
 4912 
 4913 // Pointer 64 bit Register not Special
 4914 operand iRegPNoSp()
 4915 %{
 4916   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4917   match(RegP);
 4918   // match(iRegP);
 4919   // match(iRegP_R0);
 4920   // match(iRegP_R2);
 4921   // match(iRegP_R4);
 4922   // match(iRegP_R5);
 4923   // match(thread_RegP);
 4924   op_cost(0);
 4925   format %{ %}
 4926   interface(REG_INTER);
 4927 %}
 4928 
 4929 // Pointer 64 bit Register R0 only
 4930 operand iRegP_R0()
 4931 %{
 4932   constraint(ALLOC_IN_RC(r0_reg));
 4933   match(RegP);
 4934   // match(iRegP);
 4935   match(iRegPNoSp);
 4936   op_cost(0);
 4937   format %{ %}
 4938   interface(REG_INTER);
 4939 %}
 4940 
 4941 // Pointer 64 bit Register R1 only
 4942 operand iRegP_R1()
 4943 %{
 4944   constraint(ALLOC_IN_RC(r1_reg));
 4945   match(RegP);
 4946   // match(iRegP);
 4947   match(iRegPNoSp);
 4948   op_cost(0);
 4949   format %{ %}
 4950   interface(REG_INTER);
 4951 %}
 4952 
 4953 // Pointer 64 bit Register R2 only
 4954 operand iRegP_R2()
 4955 %{
 4956   constraint(ALLOC_IN_RC(r2_reg));
 4957   match(RegP);
 4958   // match(iRegP);
 4959   match(iRegPNoSp);
 4960   op_cost(0);
 4961   format %{ %}
 4962   interface(REG_INTER);
 4963 %}
 4964 
 4965 // Pointer 64 bit Register R3 only
 4966 operand iRegP_R3()
 4967 %{
 4968   constraint(ALLOC_IN_RC(r3_reg));
 4969   match(RegP);
 4970   // match(iRegP);
 4971   match(iRegPNoSp);
 4972   op_cost(0);
 4973   format %{ %}
 4974   interface(REG_INTER);
 4975 %}
 4976 
 4977 // Pointer 64 bit Register R4 only
 4978 operand iRegP_R4()
 4979 %{
 4980   constraint(ALLOC_IN_RC(r4_reg));
 4981   match(RegP);
 4982   // match(iRegP);
 4983   match(iRegPNoSp);
 4984   op_cost(0);
 4985   format %{ %}
 4986   interface(REG_INTER);
 4987 %}
 4988 
 4989 // Pointer 64 bit Register R5 only
 4990 operand iRegP_R5()
 4991 %{
 4992   constraint(ALLOC_IN_RC(r5_reg));
 4993   match(RegP);
 4994   // match(iRegP);
 4995   match(iRegPNoSp);
 4996   op_cost(0);
 4997   format %{ %}
 4998   interface(REG_INTER);
 4999 %}
 5000 
 5001 // Pointer 64 bit Register R10 only
 5002 operand iRegP_R10()
 5003 %{
 5004   constraint(ALLOC_IN_RC(r10_reg));
 5005   match(RegP);
 5006   // match(iRegP);
 5007   match(iRegPNoSp);
 5008   op_cost(0);
 5009   format %{ %}
 5010   interface(REG_INTER);
 5011 %}
 5012 
 5013 // Long 64 bit Register R0 only
 5014 operand iRegL_R0()
 5015 %{
 5016   constraint(ALLOC_IN_RC(r0_reg));
 5017   match(RegL);
 5018   match(iRegLNoSp);
 5019   op_cost(0);
 5020   format %{ %}
 5021   interface(REG_INTER);
 5022 %}
 5023 
 5024 // Long 64 bit Register R2 only
 5025 operand iRegL_R2()
 5026 %{
 5027   constraint(ALLOC_IN_RC(r2_reg));
 5028   match(RegL);
 5029   match(iRegLNoSp);
 5030   op_cost(0);
 5031   format %{ %}
 5032   interface(REG_INTER);
 5033 %}
 5034 
 5035 // Long 64 bit Register R3 only
 5036 operand iRegL_R3()
 5037 %{
 5038   constraint(ALLOC_IN_RC(r3_reg));
 5039   match(RegL);
 5040   match(iRegLNoSp);
 5041   op_cost(0);
 5042   format %{ %}
 5043   interface(REG_INTER);
 5044 %}
 5045 
 5046 // Long 64 bit Register R11 only
 5047 operand iRegL_R11()
 5048 %{
 5049   constraint(ALLOC_IN_RC(r11_reg));
 5050   match(RegL);
 5051   match(iRegLNoSp);
 5052   op_cost(0);
 5053   format %{ %}
 5054   interface(REG_INTER);
 5055 %}
 5056 
 5057 // Pointer 64 bit Register FP only
 5058 operand iRegP_FP()
 5059 %{
 5060   constraint(ALLOC_IN_RC(fp_reg));
 5061   match(RegP);
 5062   // match(iRegP);
 5063   op_cost(0);
 5064   format %{ %}
 5065   interface(REG_INTER);
 5066 %}
 5067 
 5068 // Register R0 only
 5069 operand iRegI_R0()
 5070 %{
 5071   constraint(ALLOC_IN_RC(int_r0_reg));
 5072   match(RegI);
 5073   match(iRegINoSp);
 5074   op_cost(0);
 5075   format %{ %}
 5076   interface(REG_INTER);
 5077 %}
 5078 
 5079 // Register R2 only
 5080 operand iRegI_R2()
 5081 %{
 5082   constraint(ALLOC_IN_RC(int_r2_reg));
 5083   match(RegI);
 5084   match(iRegINoSp);
 5085   op_cost(0);
 5086   format %{ %}
 5087   interface(REG_INTER);
 5088 %}
 5089 
 5090 // Register R3 only
 5091 operand iRegI_R3()
 5092 %{
 5093   constraint(ALLOC_IN_RC(int_r3_reg));
 5094   match(RegI);
 5095   match(iRegINoSp);
 5096   op_cost(0);
 5097   format %{ %}
 5098   interface(REG_INTER);
 5099 %}
 5100 
 5101 
 5102 // Register R4 only
 5103 operand iRegI_R4()
 5104 %{
 5105   constraint(ALLOC_IN_RC(int_r4_reg));
 5106   match(RegI);
 5107   match(iRegINoSp);
 5108   op_cost(0);
 5109   format %{ %}
 5110   interface(REG_INTER);
 5111 %}
 5112 
 5113 
 5114 // Pointer Register Operands
 5115 // Narrow Pointer Register
 5116 operand iRegN()
 5117 %{
 5118   constraint(ALLOC_IN_RC(any_reg32));
 5119   match(RegN);
 5120   match(iRegNNoSp);
 5121   op_cost(0);
 5122   format %{ %}
 5123   interface(REG_INTER);
 5124 %}
 5125 
 5126 operand iRegN_R0()
 5127 %{
 5128   constraint(ALLOC_IN_RC(r0_reg));
 5129   match(iRegN);
 5130   op_cost(0);
 5131   format %{ %}
 5132   interface(REG_INTER);
 5133 %}
 5134 
 5135 operand iRegN_R2()
 5136 %{
 5137   constraint(ALLOC_IN_RC(r2_reg));
 5138   match(iRegN);
 5139   op_cost(0);
 5140   format %{ %}
 5141   interface(REG_INTER);
 5142 %}
 5143 
 5144 operand iRegN_R3()
 5145 %{
 5146   constraint(ALLOC_IN_RC(r3_reg));
 5147   match(iRegN);
 5148   op_cost(0);
 5149   format %{ %}
 5150   interface(REG_INTER);
 5151 %}
 5152 
 5153 // Integer 64 bit Register not Special
 5154 operand iRegNNoSp()
 5155 %{
 5156   constraint(ALLOC_IN_RC(no_special_reg32));
 5157   match(RegN);
 5158   op_cost(0);
 5159   format %{ %}
 5160   interface(REG_INTER);
 5161 %}
 5162 
 5163 // heap base register -- used for encoding immN0
 5164 
 5165 operand iRegIHeapbase()
 5166 %{
 5167   constraint(ALLOC_IN_RC(heapbase_reg));
 5168   match(RegI);
 5169   op_cost(0);
 5170   format %{ %}
 5171   interface(REG_INTER);
 5172 %}
 5173 
 5174 // Float Register
 5175 // Float register operands
 5176 operand vRegF()
 5177 %{
 5178   constraint(ALLOC_IN_RC(float_reg));
 5179   match(RegF);
 5180 
 5181   op_cost(0);
 5182   format %{ %}
 5183   interface(REG_INTER);
 5184 %}
 5185 
 5186 // Double Register
 5187 // Double register operands
 5188 operand vRegD()
 5189 %{
 5190   constraint(ALLOC_IN_RC(double_reg));
 5191   match(RegD);
 5192 
 5193   op_cost(0);
 5194   format %{ %}
 5195   interface(REG_INTER);
 5196 %}
 5197 
 5198 // Generic vector class. This will be used for
 5199 // all vector operands, including NEON and SVE.
 5200 operand vReg()
 5201 %{
 5202   constraint(ALLOC_IN_RC(dynamic));
 5203   match(VecA);
 5204   match(VecD);
 5205   match(VecX);
 5206 
 5207   op_cost(0);
 5208   format %{ %}
 5209   interface(REG_INTER);
 5210 %}
 5211 
 5212 operand vecA()
 5213 %{
 5214   constraint(ALLOC_IN_RC(vectora_reg));
 5215   match(VecA);
 5216 
 5217   op_cost(0);
 5218   format %{ %}
 5219   interface(REG_INTER);
 5220 %}
 5221 
 5222 operand vecD()
 5223 %{
 5224   constraint(ALLOC_IN_RC(vectord_reg));
 5225   match(VecD);
 5226 
 5227   op_cost(0);
 5228   format %{ %}
 5229   interface(REG_INTER);
 5230 %}
 5231 
 5232 operand vecX()
 5233 %{
 5234   constraint(ALLOC_IN_RC(vectorx_reg));
 5235   match(VecX);
 5236 
 5237   op_cost(0);
 5238   format %{ %}
 5239   interface(REG_INTER);
 5240 %}
 5241 
 5242 operand vRegD_V0()
 5243 %{
 5244   constraint(ALLOC_IN_RC(v0_reg));
 5245   match(RegD);
 5246   op_cost(0);
 5247   format %{ %}
 5248   interface(REG_INTER);
 5249 %}
 5250 
 5251 operand vRegD_V1()
 5252 %{
 5253   constraint(ALLOC_IN_RC(v1_reg));
 5254   match(RegD);
 5255   op_cost(0);
 5256   format %{ %}
 5257   interface(REG_INTER);
 5258 %}
 5259 
 5260 operand vRegD_V2()
 5261 %{
 5262   constraint(ALLOC_IN_RC(v2_reg));
 5263   match(RegD);
 5264   op_cost(0);
 5265   format %{ %}
 5266   interface(REG_INTER);
 5267 %}
 5268 
 5269 operand vRegD_V3()
 5270 %{
 5271   constraint(ALLOC_IN_RC(v3_reg));
 5272   match(RegD);
 5273   op_cost(0);
 5274   format %{ %}
 5275   interface(REG_INTER);
 5276 %}
 5277 
 5278 operand vRegD_V4()
 5279 %{
 5280   constraint(ALLOC_IN_RC(v4_reg));
 5281   match(RegD);
 5282   op_cost(0);
 5283   format %{ %}
 5284   interface(REG_INTER);
 5285 %}
 5286 
 5287 operand vRegD_V5()
 5288 %{
 5289   constraint(ALLOC_IN_RC(v5_reg));
 5290   match(RegD);
 5291   op_cost(0);
 5292   format %{ %}
 5293   interface(REG_INTER);
 5294 %}
 5295 
 5296 operand vRegD_V6()
 5297 %{
 5298   constraint(ALLOC_IN_RC(v6_reg));
 5299   match(RegD);
 5300   op_cost(0);
 5301   format %{ %}
 5302   interface(REG_INTER);
 5303 %}
 5304 
 5305 operand vRegD_V7()
 5306 %{
 5307   constraint(ALLOC_IN_RC(v7_reg));
 5308   match(RegD);
 5309   op_cost(0);
 5310   format %{ %}
 5311   interface(REG_INTER);
 5312 %}
 5313 
 5314 operand vRegD_V8()
 5315 %{
 5316   constraint(ALLOC_IN_RC(v8_reg));
 5317   match(RegD);
 5318   op_cost(0);
 5319   format %{ %}
 5320   interface(REG_INTER);
 5321 %}
 5322 
 5323 operand vRegD_V9()
 5324 %{
 5325   constraint(ALLOC_IN_RC(v9_reg));
 5326   match(RegD);
 5327   op_cost(0);
 5328   format %{ %}
 5329   interface(REG_INTER);
 5330 %}
 5331 
 5332 operand vRegD_V10()
 5333 %{
 5334   constraint(ALLOC_IN_RC(v10_reg));
 5335   match(RegD);
 5336   op_cost(0);
 5337   format %{ %}
 5338   interface(REG_INTER);
 5339 %}
 5340 
 5341 operand vRegD_V11()
 5342 %{
 5343   constraint(ALLOC_IN_RC(v11_reg));
 5344   match(RegD);
 5345   op_cost(0);
 5346   format %{ %}
 5347   interface(REG_INTER);
 5348 %}
 5349 
 5350 operand vRegD_V12()
 5351 %{
 5352   constraint(ALLOC_IN_RC(v12_reg));
 5353   match(RegD);
 5354   op_cost(0);
 5355   format %{ %}
 5356   interface(REG_INTER);
 5357 %}
 5358 
 5359 operand vRegD_V13()
 5360 %{
 5361   constraint(ALLOC_IN_RC(v13_reg));
 5362   match(RegD);
 5363   op_cost(0);
 5364   format %{ %}
 5365   interface(REG_INTER);
 5366 %}
 5367 
 5368 operand vRegD_V14()
 5369 %{
 5370   constraint(ALLOC_IN_RC(v14_reg));
 5371   match(RegD);
 5372   op_cost(0);
 5373   format %{ %}
 5374   interface(REG_INTER);
 5375 %}
 5376 
 5377 operand vRegD_V15()
 5378 %{
 5379   constraint(ALLOC_IN_RC(v15_reg));
 5380   match(RegD);
 5381   op_cost(0);
 5382   format %{ %}
 5383   interface(REG_INTER);
 5384 %}
 5385 
 5386 operand vRegD_V16()
 5387 %{
 5388   constraint(ALLOC_IN_RC(v16_reg));
 5389   match(RegD);
 5390   op_cost(0);
 5391   format %{ %}
 5392   interface(REG_INTER);
 5393 %}
 5394 
 5395 operand vRegD_V17()
 5396 %{
 5397   constraint(ALLOC_IN_RC(v17_reg));
 5398   match(RegD);
 5399   op_cost(0);
 5400   format %{ %}
 5401   interface(REG_INTER);
 5402 %}
 5403 
 5404 operand vRegD_V18()
 5405 %{
 5406   constraint(ALLOC_IN_RC(v18_reg));
 5407   match(RegD);
 5408   op_cost(0);
 5409   format %{ %}
 5410   interface(REG_INTER);
 5411 %}
 5412 
 5413 operand vRegD_V19()
 5414 %{
 5415   constraint(ALLOC_IN_RC(v19_reg));
 5416   match(RegD);
 5417   op_cost(0);
 5418   format %{ %}
 5419   interface(REG_INTER);
 5420 %}
 5421 
 5422 operand vRegD_V20()
 5423 %{
 5424   constraint(ALLOC_IN_RC(v20_reg));
 5425   match(RegD);
 5426   op_cost(0);
 5427   format %{ %}
 5428   interface(REG_INTER);
 5429 %}
 5430 
 5431 operand vRegD_V21()
 5432 %{
 5433   constraint(ALLOC_IN_RC(v21_reg));
 5434   match(RegD);
 5435   op_cost(0);
 5436   format %{ %}
 5437   interface(REG_INTER);
 5438 %}
 5439 
 5440 operand vRegD_V22()
 5441 %{
 5442   constraint(ALLOC_IN_RC(v22_reg));
 5443   match(RegD);
 5444   op_cost(0);
 5445   format %{ %}
 5446   interface(REG_INTER);
 5447 %}
 5448 
 5449 operand vRegD_V23()
 5450 %{
 5451   constraint(ALLOC_IN_RC(v23_reg));
 5452   match(RegD);
 5453   op_cost(0);
 5454   format %{ %}
 5455   interface(REG_INTER);
 5456 %}
 5457 
 5458 operand vRegD_V24()
 5459 %{
 5460   constraint(ALLOC_IN_RC(v24_reg));
 5461   match(RegD);
 5462   op_cost(0);
 5463   format %{ %}
 5464   interface(REG_INTER);
 5465 %}
 5466 
 5467 operand vRegD_V25()
 5468 %{
 5469   constraint(ALLOC_IN_RC(v25_reg));
 5470   match(RegD);
 5471   op_cost(0);
 5472   format %{ %}
 5473   interface(REG_INTER);
 5474 %}
 5475 
 5476 operand vRegD_V26()
 5477 %{
 5478   constraint(ALLOC_IN_RC(v26_reg));
 5479   match(RegD);
 5480   op_cost(0);
 5481   format %{ %}
 5482   interface(REG_INTER);
 5483 %}
 5484 
 5485 operand vRegD_V27()
 5486 %{
 5487   constraint(ALLOC_IN_RC(v27_reg));
 5488   match(RegD);
 5489   op_cost(0);
 5490   format %{ %}
 5491   interface(REG_INTER);
 5492 %}
 5493 
 5494 operand vRegD_V28()
 5495 %{
 5496   constraint(ALLOC_IN_RC(v28_reg));
 5497   match(RegD);
 5498   op_cost(0);
 5499   format %{ %}
 5500   interface(REG_INTER);
 5501 %}
 5502 
 5503 operand vRegD_V29()
 5504 %{
 5505   constraint(ALLOC_IN_RC(v29_reg));
 5506   match(RegD);
 5507   op_cost(0);
 5508   format %{ %}
 5509   interface(REG_INTER);
 5510 %}
 5511 
 5512 operand vRegD_V30()
 5513 %{
 5514   constraint(ALLOC_IN_RC(v30_reg));
 5515   match(RegD);
 5516   op_cost(0);
 5517   format %{ %}
 5518   interface(REG_INTER);
 5519 %}
 5520 
 5521 operand vRegD_V31()
 5522 %{
 5523   constraint(ALLOC_IN_RC(v31_reg));
 5524   match(RegD);
 5525   op_cost(0);
 5526   format %{ %}
 5527   interface(REG_INTER);
 5528 %}
 5529 
 5530 operand pReg()
 5531 %{
 5532   constraint(ALLOC_IN_RC(pr_reg));
 5533   match(RegVectMask);
 5534   match(pRegGov);
 5535   op_cost(0);
 5536   format %{ %}
 5537   interface(REG_INTER);
 5538 %}
 5539 
 5540 operand pRegGov()
 5541 %{
 5542   constraint(ALLOC_IN_RC(gov_pr));
 5543   match(RegVectMask);
 5544   match(pReg);
 5545   op_cost(0);
 5546   format %{ %}
 5547   interface(REG_INTER);
 5548 %}
 5549 
 5550 operand pRegGov_P0()
 5551 %{
 5552   constraint(ALLOC_IN_RC(p0_reg));
 5553   match(RegVectMask);
 5554   op_cost(0);
 5555   format %{ %}
 5556   interface(REG_INTER);
 5557 %}
 5558 
 5559 operand pRegGov_P1()
 5560 %{
 5561   constraint(ALLOC_IN_RC(p1_reg));
 5562   match(RegVectMask);
 5563   op_cost(0);
 5564   format %{ %}
 5565   interface(REG_INTER);
 5566 %}
 5567 
 5568 // Flags register, used as output of signed compare instructions
 5569 
 5570 // note that on AArch64 we also use this register as the output for
 5571 // for floating point compare instructions (CmpF CmpD). this ensures
 5572 // that ordered inequality tests use GT, GE, LT or LE none of which
 5573 // pass through cases where the result is unordered i.e. one or both
 5574 // inputs to the compare is a NaN. this means that the ideal code can
 5575 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5576 // (where the comparison should always fail). EQ and NE tests are
 5577 // always generated in ideal code so that unordered folds into the NE
 5578 // case, matching the behaviour of AArch64 NE.
 5579 //
 5580 // This differs from x86 where the outputs of FP compares use a
 5581 // special FP flags registers and where compares based on this
 5582 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5583 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5584 // to explicitly handle the unordered case in branches. x86 also has
 5585 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5586 
 5587 operand rFlagsReg()
 5588 %{
 5589   constraint(ALLOC_IN_RC(int_flags));
 5590   match(RegFlags);
 5591 
 5592   op_cost(0);
 5593   format %{ "RFLAGS" %}
 5594   interface(REG_INTER);
 5595 %}
 5596 
 5597 // Flags register, used as output of unsigned compare instructions
 5598 operand rFlagsRegU()
 5599 %{
 5600   constraint(ALLOC_IN_RC(int_flags));
 5601   match(RegFlags);
 5602 
 5603   op_cost(0);
 5604   format %{ "RFLAGSU" %}
 5605   interface(REG_INTER);
 5606 %}
 5607 
 5608 // Special Registers
 5609 
 5610 // Method Register
 5611 operand inline_cache_RegP(iRegP reg)
 5612 %{
 5613   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5614   match(reg);
 5615   match(iRegPNoSp);
 5616   op_cost(0);
 5617   format %{ %}
 5618   interface(REG_INTER);
 5619 %}
 5620 
 5621 // Thread Register
 5622 operand thread_RegP(iRegP reg)
 5623 %{
 5624   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5625   match(reg);
 5626   op_cost(0);
 5627   format %{ %}
 5628   interface(REG_INTER);
 5629 %}
 5630 
 5631 operand lr_RegP(iRegP reg)
 5632 %{
 5633   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5634   match(reg);
 5635   op_cost(0);
 5636   format %{ %}
 5637   interface(REG_INTER);
 5638 %}
 5639 
 5640 //----------Memory Operands----------------------------------------------------
 5641 
 5642 operand indirect(iRegP reg)
 5643 %{
 5644   constraint(ALLOC_IN_RC(ptr_reg));
 5645   match(reg);
 5646   op_cost(0);
 5647   format %{ "[$reg]" %}
 5648   interface(MEMORY_INTER) %{
 5649     base($reg);
 5650     index(0xffffffff);
 5651     scale(0x0);
 5652     disp(0x0);
 5653   %}
 5654 %}
 5655 
 5656 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5657 %{
 5658   constraint(ALLOC_IN_RC(ptr_reg));
 5659   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5660   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5661   op_cost(0);
 5662   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5663   interface(MEMORY_INTER) %{
 5664     base($reg);
 5665     index($ireg);
 5666     scale($scale);
 5667     disp(0x0);
 5668   %}
 5669 %}
 5670 
 5671 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5672 %{
 5673   constraint(ALLOC_IN_RC(ptr_reg));
 5674   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5675   match(AddP reg (LShiftL lreg scale));
 5676   op_cost(0);
 5677   format %{ "$reg, $lreg lsl($scale)" %}
 5678   interface(MEMORY_INTER) %{
 5679     base($reg);
 5680     index($lreg);
 5681     scale($scale);
 5682     disp(0x0);
 5683   %}
 5684 %}
 5685 
 5686 operand indIndexI2L(iRegP reg, iRegI ireg)
 5687 %{
 5688   constraint(ALLOC_IN_RC(ptr_reg));
 5689   match(AddP reg (ConvI2L ireg));
 5690   op_cost(0);
 5691   format %{ "$reg, $ireg, 0, I2L" %}
 5692   interface(MEMORY_INTER) %{
 5693     base($reg);
 5694     index($ireg);
 5695     scale(0x0);
 5696     disp(0x0);
 5697   %}
 5698 %}
 5699 
 5700 operand indIndex(iRegP reg, iRegL lreg)
 5701 %{
 5702   constraint(ALLOC_IN_RC(ptr_reg));
 5703   match(AddP reg lreg);
 5704   op_cost(0);
 5705   format %{ "$reg, $lreg" %}
 5706   interface(MEMORY_INTER) %{
 5707     base($reg);
 5708     index($lreg);
 5709     scale(0x0);
 5710     disp(0x0);
 5711   %}
 5712 %}
 5713 
 5714 operand indOffI(iRegP reg, immIOffset off)
 5715 %{
 5716   constraint(ALLOC_IN_RC(ptr_reg));
 5717   match(AddP reg off);
 5718   op_cost(0);
 5719   format %{ "[$reg, $off]" %}
 5720   interface(MEMORY_INTER) %{
 5721     base($reg);
 5722     index(0xffffffff);
 5723     scale(0x0);
 5724     disp($off);
 5725   %}
 5726 %}
 5727 
 5728 operand indOffI1(iRegP reg, immIOffset1 off)
 5729 %{
 5730   constraint(ALLOC_IN_RC(ptr_reg));
 5731   match(AddP reg off);
 5732   op_cost(0);
 5733   format %{ "[$reg, $off]" %}
 5734   interface(MEMORY_INTER) %{
 5735     base($reg);
 5736     index(0xffffffff);
 5737     scale(0x0);
 5738     disp($off);
 5739   %}
 5740 %}
 5741 
 5742 operand indOffI2(iRegP reg, immIOffset2 off)
 5743 %{
 5744   constraint(ALLOC_IN_RC(ptr_reg));
 5745   match(AddP reg off);
 5746   op_cost(0);
 5747   format %{ "[$reg, $off]" %}
 5748   interface(MEMORY_INTER) %{
 5749     base($reg);
 5750     index(0xffffffff);
 5751     scale(0x0);
 5752     disp($off);
 5753   %}
 5754 %}
 5755 
 5756 operand indOffI4(iRegP reg, immIOffset4 off)
 5757 %{
 5758   constraint(ALLOC_IN_RC(ptr_reg));
 5759   match(AddP reg off);
 5760   op_cost(0);
 5761   format %{ "[$reg, $off]" %}
 5762   interface(MEMORY_INTER) %{
 5763     base($reg);
 5764     index(0xffffffff);
 5765     scale(0x0);
 5766     disp($off);
 5767   %}
 5768 %}
 5769 
 5770 operand indOffI8(iRegP reg, immIOffset8 off)
 5771 %{
 5772   constraint(ALLOC_IN_RC(ptr_reg));
 5773   match(AddP reg off);
 5774   op_cost(0);
 5775   format %{ "[$reg, $off]" %}
 5776   interface(MEMORY_INTER) %{
 5777     base($reg);
 5778     index(0xffffffff);
 5779     scale(0x0);
 5780     disp($off);
 5781   %}
 5782 %}
 5783 
 5784 operand indOffI16(iRegP reg, immIOffset16 off)
 5785 %{
 5786   constraint(ALLOC_IN_RC(ptr_reg));
 5787   match(AddP reg off);
 5788   op_cost(0);
 5789   format %{ "[$reg, $off]" %}
 5790   interface(MEMORY_INTER) %{
 5791     base($reg);
 5792     index(0xffffffff);
 5793     scale(0x0);
 5794     disp($off);
 5795   %}
 5796 %}
 5797 
 5798 operand indOffL(iRegP reg, immLoffset off)
 5799 %{
 5800   constraint(ALLOC_IN_RC(ptr_reg));
 5801   match(AddP reg off);
 5802   op_cost(0);
 5803   format %{ "[$reg, $off]" %}
 5804   interface(MEMORY_INTER) %{
 5805     base($reg);
 5806     index(0xffffffff);
 5807     scale(0x0);
 5808     disp($off);
 5809   %}
 5810 %}
 5811 
 5812 operand indOffL1(iRegP reg, immLoffset1 off)
 5813 %{
 5814   constraint(ALLOC_IN_RC(ptr_reg));
 5815   match(AddP reg off);
 5816   op_cost(0);
 5817   format %{ "[$reg, $off]" %}
 5818   interface(MEMORY_INTER) %{
 5819     base($reg);
 5820     index(0xffffffff);
 5821     scale(0x0);
 5822     disp($off);
 5823   %}
 5824 %}
 5825 
 5826 operand indOffL2(iRegP reg, immLoffset2 off)
 5827 %{
 5828   constraint(ALLOC_IN_RC(ptr_reg));
 5829   match(AddP reg off);
 5830   op_cost(0);
 5831   format %{ "[$reg, $off]" %}
 5832   interface(MEMORY_INTER) %{
 5833     base($reg);
 5834     index(0xffffffff);
 5835     scale(0x0);
 5836     disp($off);
 5837   %}
 5838 %}
 5839 
 5840 operand indOffL4(iRegP reg, immLoffset4 off)
 5841 %{
 5842   constraint(ALLOC_IN_RC(ptr_reg));
 5843   match(AddP reg off);
 5844   op_cost(0);
 5845   format %{ "[$reg, $off]" %}
 5846   interface(MEMORY_INTER) %{
 5847     base($reg);
 5848     index(0xffffffff);
 5849     scale(0x0);
 5850     disp($off);
 5851   %}
 5852 %}
 5853 
 5854 operand indOffL8(iRegP reg, immLoffset8 off)
 5855 %{
 5856   constraint(ALLOC_IN_RC(ptr_reg));
 5857   match(AddP reg off);
 5858   op_cost(0);
 5859   format %{ "[$reg, $off]" %}
 5860   interface(MEMORY_INTER) %{
 5861     base($reg);
 5862     index(0xffffffff);
 5863     scale(0x0);
 5864     disp($off);
 5865   %}
 5866 %}
 5867 
 5868 operand indOffL16(iRegP reg, immLoffset16 off)
 5869 %{
 5870   constraint(ALLOC_IN_RC(ptr_reg));
 5871   match(AddP reg off);
 5872   op_cost(0);
 5873   format %{ "[$reg, $off]" %}
 5874   interface(MEMORY_INTER) %{
 5875     base($reg);
 5876     index(0xffffffff);
 5877     scale(0x0);
 5878     disp($off);
 5879   %}
 5880 %}
 5881 
 5882 operand indirectN(iRegN reg)
 5883 %{
 5884   predicate(CompressedOops::shift() == 0);
 5885   constraint(ALLOC_IN_RC(ptr_reg));
 5886   match(DecodeN reg);
 5887   op_cost(0);
 5888   format %{ "[$reg]\t# narrow" %}
 5889   interface(MEMORY_INTER) %{
 5890     base($reg);
 5891     index(0xffffffff);
 5892     scale(0x0);
 5893     disp(0x0);
 5894   %}
 5895 %}
 5896 
 5897 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5898 %{
 5899   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5900   constraint(ALLOC_IN_RC(ptr_reg));
 5901   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5902   op_cost(0);
 5903   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5904   interface(MEMORY_INTER) %{
 5905     base($reg);
 5906     index($ireg);
 5907     scale($scale);
 5908     disp(0x0);
 5909   %}
 5910 %}
 5911 
 5912 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5913 %{
 5914   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5915   constraint(ALLOC_IN_RC(ptr_reg));
 5916   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5917   op_cost(0);
 5918   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5919   interface(MEMORY_INTER) %{
 5920     base($reg);
 5921     index($lreg);
 5922     scale($scale);
 5923     disp(0x0);
 5924   %}
 5925 %}
 5926 
 5927 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5928 %{
 5929   predicate(CompressedOops::shift() == 0);
 5930   constraint(ALLOC_IN_RC(ptr_reg));
 5931   match(AddP (DecodeN reg) (ConvI2L ireg));
 5932   op_cost(0);
 5933   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5934   interface(MEMORY_INTER) %{
 5935     base($reg);
 5936     index($ireg);
 5937     scale(0x0);
 5938     disp(0x0);
 5939   %}
 5940 %}
 5941 
 5942 operand indIndexN(iRegN reg, iRegL lreg)
 5943 %{
 5944   predicate(CompressedOops::shift() == 0);
 5945   constraint(ALLOC_IN_RC(ptr_reg));
 5946   match(AddP (DecodeN reg) lreg);
 5947   op_cost(0);
 5948   format %{ "$reg, $lreg\t# narrow" %}
 5949   interface(MEMORY_INTER) %{
 5950     base($reg);
 5951     index($lreg);
 5952     scale(0x0);
 5953     disp(0x0);
 5954   %}
 5955 %}
 5956 
 5957 operand indOffIN(iRegN reg, immIOffset off)
 5958 %{
 5959   predicate(CompressedOops::shift() == 0);
 5960   constraint(ALLOC_IN_RC(ptr_reg));
 5961   match(AddP (DecodeN reg) off);
 5962   op_cost(0);
 5963   format %{ "[$reg, $off]\t# narrow" %}
 5964   interface(MEMORY_INTER) %{
 5965     base($reg);
 5966     index(0xffffffff);
 5967     scale(0x0);
 5968     disp($off);
 5969   %}
 5970 %}
 5971 
 5972 operand indOffLN(iRegN reg, immLoffset off)
 5973 %{
 5974   predicate(CompressedOops::shift() == 0);
 5975   constraint(ALLOC_IN_RC(ptr_reg));
 5976   match(AddP (DecodeN reg) off);
 5977   op_cost(0);
 5978   format %{ "[$reg, $off]\t# narrow" %}
 5979   interface(MEMORY_INTER) %{
 5980     base($reg);
 5981     index(0xffffffff);
 5982     scale(0x0);
 5983     disp($off);
 5984   %}
 5985 %}
 5986 
 5987 
 5988 
 5989 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 5990 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 5991 %{
 5992   constraint(ALLOC_IN_RC(ptr_reg));
 5993   match(AddP reg off);
 5994   op_cost(0);
 5995   format %{ "[$reg, $off]" %}
 5996   interface(MEMORY_INTER) %{
 5997     base($reg);
 5998     index(0xffffffff);
 5999     scale(0x0);
 6000     disp($off);
 6001   %}
 6002 %}
 6003 
 6004 //----------Special Memory Operands--------------------------------------------
 6005 // Stack Slot Operand - This operand is used for loading and storing temporary
 6006 //                      values on the stack where a match requires a value to
 6007 //                      flow through memory.
 6008 operand stackSlotP(sRegP reg)
 6009 %{
 6010   constraint(ALLOC_IN_RC(stack_slots));
 6011   op_cost(100);
 6012   // No match rule because this operand is only generated in matching
 6013   // match(RegP);
 6014   format %{ "[$reg]" %}
 6015   interface(MEMORY_INTER) %{
 6016     base(0x1e);  // RSP
 6017     index(0x0);  // No Index
 6018     scale(0x0);  // No Scale
 6019     disp($reg);  // Stack Offset
 6020   %}
 6021 %}
 6022 
 6023 operand stackSlotI(sRegI reg)
 6024 %{
 6025   constraint(ALLOC_IN_RC(stack_slots));
 6026   // No match rule because this operand is only generated in matching
 6027   // match(RegI);
 6028   format %{ "[$reg]" %}
 6029   interface(MEMORY_INTER) %{
 6030     base(0x1e);  // RSP
 6031     index(0x0);  // No Index
 6032     scale(0x0);  // No Scale
 6033     disp($reg);  // Stack Offset
 6034   %}
 6035 %}
 6036 
 6037 operand stackSlotF(sRegF reg)
 6038 %{
 6039   constraint(ALLOC_IN_RC(stack_slots));
 6040   // No match rule because this operand is only generated in matching
 6041   // match(RegF);
 6042   format %{ "[$reg]" %}
 6043   interface(MEMORY_INTER) %{
 6044     base(0x1e);  // RSP
 6045     index(0x0);  // No Index
 6046     scale(0x0);  // No Scale
 6047     disp($reg);  // Stack Offset
 6048   %}
 6049 %}
 6050 
 6051 operand stackSlotD(sRegD reg)
 6052 %{
 6053   constraint(ALLOC_IN_RC(stack_slots));
 6054   // No match rule because this operand is only generated in matching
 6055   // match(RegD);
 6056   format %{ "[$reg]" %}
 6057   interface(MEMORY_INTER) %{
 6058     base(0x1e);  // RSP
 6059     index(0x0);  // No Index
 6060     scale(0x0);  // No Scale
 6061     disp($reg);  // Stack Offset
 6062   %}
 6063 %}
 6064 
 6065 operand stackSlotL(sRegL reg)
 6066 %{
 6067   constraint(ALLOC_IN_RC(stack_slots));
 6068   // No match rule because this operand is only generated in matching
 6069   // match(RegL);
 6070   format %{ "[$reg]" %}
 6071   interface(MEMORY_INTER) %{
 6072     base(0x1e);  // RSP
 6073     index(0x0);  // No Index
 6074     scale(0x0);  // No Scale
 6075     disp($reg);  // Stack Offset
 6076   %}
 6077 %}
 6078 
 6079 // Operands for expressing Control Flow
 6080 // NOTE: Label is a predefined operand which should not be redefined in
 6081 //       the AD file. It is generically handled within the ADLC.
 6082 
 6083 //----------Conditional Branch Operands----------------------------------------
 6084 // Comparison Op  - This is the operation of the comparison, and is limited to
 6085 //                  the following set of codes:
 6086 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6087 //
 6088 // Other attributes of the comparison, such as unsignedness, are specified
 6089 // by the comparison instruction that sets a condition code flags register.
 6090 // That result is represented by a flags operand whose subtype is appropriate
 6091 // to the unsignedness (etc.) of the comparison.
 6092 //
 6093 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6094 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6095 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6096 
 6097 // used for signed integral comparisons and fp comparisons
 6098 
 6099 operand cmpOp()
 6100 %{
 6101   match(Bool);
 6102 
 6103   format %{ "" %}
 6104   interface(COND_INTER) %{
 6105     equal(0x0, "eq");
 6106     not_equal(0x1, "ne");
 6107     less(0xb, "lt");
 6108     greater_equal(0xa, "ge");
 6109     less_equal(0xd, "le");
 6110     greater(0xc, "gt");
 6111     overflow(0x6, "vs");
 6112     no_overflow(0x7, "vc");
 6113   %}
 6114 %}
 6115 
 6116 // used for unsigned integral comparisons
 6117 
 6118 operand cmpOpU()
 6119 %{
 6120   match(Bool);
 6121 
 6122   format %{ "" %}
 6123   interface(COND_INTER) %{
 6124     equal(0x0, "eq");
 6125     not_equal(0x1, "ne");
 6126     less(0x3, "lo");
 6127     greater_equal(0x2, "hs");
 6128     less_equal(0x9, "ls");
 6129     greater(0x8, "hi");
 6130     overflow(0x6, "vs");
 6131     no_overflow(0x7, "vc");
 6132   %}
 6133 %}
 6134 
 6135 // used for certain integral comparisons which can be
 6136 // converted to cbxx or tbxx instructions
 6137 
 6138 operand cmpOpEqNe()
 6139 %{
 6140   match(Bool);
 6141   op_cost(0);
 6142   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6143             || n->as_Bool()->_test._test == BoolTest::eq);
 6144 
 6145   format %{ "" %}
 6146   interface(COND_INTER) %{
 6147     equal(0x0, "eq");
 6148     not_equal(0x1, "ne");
 6149     less(0xb, "lt");
 6150     greater_equal(0xa, "ge");
 6151     less_equal(0xd, "le");
 6152     greater(0xc, "gt");
 6153     overflow(0x6, "vs");
 6154     no_overflow(0x7, "vc");
 6155   %}
 6156 %}
 6157 
 6158 // used for certain integral comparisons which can be
 6159 // converted to cbxx or tbxx instructions
 6160 
 6161 operand cmpOpLtGe()
 6162 %{
 6163   match(Bool);
 6164   op_cost(0);
 6165 
 6166   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6167             || n->as_Bool()->_test._test == BoolTest::ge);
 6168 
 6169   format %{ "" %}
 6170   interface(COND_INTER) %{
 6171     equal(0x0, "eq");
 6172     not_equal(0x1, "ne");
 6173     less(0xb, "lt");
 6174     greater_equal(0xa, "ge");
 6175     less_equal(0xd, "le");
 6176     greater(0xc, "gt");
 6177     overflow(0x6, "vs");
 6178     no_overflow(0x7, "vc");
 6179   %}
 6180 %}
 6181 
 6182 // used for certain unsigned integral comparisons which can be
 6183 // converted to cbxx or tbxx instructions
 6184 
 6185 operand cmpOpUEqNeLtGe()
 6186 %{
 6187   match(Bool);
 6188   op_cost(0);
 6189 
 6190   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6191             || n->as_Bool()->_test._test == BoolTest::ne
 6192             || n->as_Bool()->_test._test == BoolTest::lt
 6193             || n->as_Bool()->_test._test == BoolTest::ge);
 6194 
 6195   format %{ "" %}
 6196   interface(COND_INTER) %{
 6197     equal(0x0, "eq");
 6198     not_equal(0x1, "ne");
 6199     less(0xb, "lt");
 6200     greater_equal(0xa, "ge");
 6201     less_equal(0xd, "le");
 6202     greater(0xc, "gt");
 6203     overflow(0x6, "vs");
 6204     no_overflow(0x7, "vc");
 6205   %}
 6206 %}
 6207 
 6208 // Special operand allowing long args to int ops to be truncated for free
 6209 
 6210 operand iRegL2I(iRegL reg) %{
 6211 
 6212   op_cost(0);
 6213 
 6214   match(ConvL2I reg);
 6215 
 6216   format %{ "l2i($reg)" %}
 6217 
 6218   interface(REG_INTER)
 6219 %}
 6220 
 6221 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6222 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6223 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6224 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6225 
 6226 //----------OPERAND CLASSES----------------------------------------------------
 6227 // Operand Classes are groups of operands that are used as to simplify
 6228 // instruction definitions by not requiring the AD writer to specify
 6229 // separate instructions for every form of operand when the
 6230 // instruction accepts multiple operand types with the same basic
 6231 // encoding and format. The classic case of this is memory operands.
 6232 
 6233 // memory is used to define read/write location for load/store
 6234 // instruction defs. we can turn a memory op into an Address
 6235 
 6236 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6237                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6238 
 6239 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6240                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6241 
 6242 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6243                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6244 
 6245 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6246                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6247 
 6248 // All of the memory operands. For the pipeline description.
 6249 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6250                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6251                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6252 
 6253 
 6254 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6255 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6256 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6257 // can be elided because the 32-bit instruction will just employ the
 6258 // lower 32 bits anyway.
 6259 //
 6260 // n.b. this does not elide all L2I conversions. if the truncated
 6261 // value is consumed by more than one operation then the ConvL2I
 6262 // cannot be bundled into the consuming nodes so an l2i gets planted
 6263 // (actually a movw $dst $src) and the downstream instructions consume
 6264 // the result of the l2i as an iRegI input. That's a shame since the
 6265 // movw is actually redundant but its not too costly.
 6266 
 6267 opclass iRegIorL2I(iRegI, iRegL2I);
 6268 
 6269 //----------PIPELINE-----------------------------------------------------------
 6270 // Rules which define the behavior of the target architectures pipeline.
 6271 
 6272 // For specific pipelines, eg A53, define the stages of that pipeline
 6273 //pipe_desc(ISS, EX1, EX2, WR);
 6274 #define ISS S0
 6275 #define EX1 S1
 6276 #define EX2 S2
 6277 #define WR  S3
 6278 
 6279 // Integer ALU reg operation
 6280 pipeline %{
 6281 
 6282 attributes %{
 6283   // ARM instructions are of fixed length
 6284   fixed_size_instructions;        // Fixed size instructions TODO does
 6285   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6286   // ARM instructions come in 32-bit word units
 6287   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6288   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6289   instruction_fetch_units = 1;       // of 64 bytes
 6290 
 6291   // List of nop instructions
 6292   nops( MachNop );
 6293 %}
 6294 
 6295 // We don't use an actual pipeline model so don't care about resources
 6296 // or description. we do use pipeline classes to introduce fixed
 6297 // latencies
 6298 
 6299 //----------RESOURCES----------------------------------------------------------
 6300 // Resources are the functional units available to the machine
 6301 
 6302 resources( INS0, INS1, INS01 = INS0 | INS1,
 6303            ALU0, ALU1, ALU = ALU0 | ALU1,
 6304            MAC,
 6305            DIV,
 6306            BRANCH,
 6307            LDST,
 6308            NEON_FP);
 6309 
 6310 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6311 // Pipeline Description specifies the stages in the machine's pipeline
 6312 
 6313 // Define the pipeline as a generic 6 stage pipeline
 6314 pipe_desc(S0, S1, S2, S3, S4, S5);
 6315 
 6316 //----------PIPELINE CLASSES---------------------------------------------------
 6317 // Pipeline Classes describe the stages in which input and output are
 6318 // referenced by the hardware pipeline.
 6319 
 6320 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6321 %{
 6322   single_instruction;
 6323   src1   : S1(read);
 6324   src2   : S2(read);
 6325   dst    : S5(write);
 6326   INS01  : ISS;
 6327   NEON_FP : S5;
 6328 %}
 6329 
 6330 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6331 %{
 6332   single_instruction;
 6333   src1   : S1(read);
 6334   src2   : S2(read);
 6335   dst    : S5(write);
 6336   INS01  : ISS;
 6337   NEON_FP : S5;
 6338 %}
 6339 
 6340 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6341 %{
 6342   single_instruction;
 6343   src    : S1(read);
 6344   dst    : S5(write);
 6345   INS01  : ISS;
 6346   NEON_FP : S5;
 6347 %}
 6348 
 6349 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6350 %{
 6351   single_instruction;
 6352   src    : S1(read);
 6353   dst    : S5(write);
 6354   INS01  : ISS;
 6355   NEON_FP : S5;
 6356 %}
 6357 
 6358 pipe_class fp_d2f(vRegF dst, vRegD src)
 6359 %{
 6360   single_instruction;
 6361   src    : S1(read);
 6362   dst    : S5(write);
 6363   INS01  : ISS;
 6364   NEON_FP : S5;
 6365 %}
 6366 
 6367 pipe_class fp_f2d(vRegD dst, vRegF src)
 6368 %{
 6369   single_instruction;
 6370   src    : S1(read);
 6371   dst    : S5(write);
 6372   INS01  : ISS;
 6373   NEON_FP : S5;
 6374 %}
 6375 
 6376 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6377 %{
 6378   single_instruction;
 6379   src    : S1(read);
 6380   dst    : S5(write);
 6381   INS01  : ISS;
 6382   NEON_FP : S5;
 6383 %}
 6384 
 6385 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6386 %{
 6387   single_instruction;
 6388   src    : S1(read);
 6389   dst    : S5(write);
 6390   INS01  : ISS;
 6391   NEON_FP : S5;
 6392 %}
 6393 
 6394 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6395 %{
 6396   single_instruction;
 6397   src    : S1(read);
 6398   dst    : S5(write);
 6399   INS01  : ISS;
 6400   NEON_FP : S5;
 6401 %}
 6402 
 6403 pipe_class fp_l2f(vRegF dst, iRegL src)
 6404 %{
 6405   single_instruction;
 6406   src    : S1(read);
 6407   dst    : S5(write);
 6408   INS01  : ISS;
 6409   NEON_FP : S5;
 6410 %}
 6411 
 6412 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6413 %{
 6414   single_instruction;
 6415   src    : S1(read);
 6416   dst    : S5(write);
 6417   INS01  : ISS;
 6418   NEON_FP : S5;
 6419 %}
 6420 
 6421 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6422 %{
 6423   single_instruction;
 6424   src    : S1(read);
 6425   dst    : S5(write);
 6426   INS01  : ISS;
 6427   NEON_FP : S5;
 6428 %}
 6429 
 6430 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6431 %{
 6432   single_instruction;
 6433   src    : S1(read);
 6434   dst    : S5(write);
 6435   INS01  : ISS;
 6436   NEON_FP : S5;
 6437 %}
 6438 
 6439 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6440 %{
 6441   single_instruction;
 6442   src    : S1(read);
 6443   dst    : S5(write);
 6444   INS01  : ISS;
 6445   NEON_FP : S5;
 6446 %}
 6447 
 6448 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6449 %{
 6450   single_instruction;
 6451   src1   : S1(read);
 6452   src2   : S2(read);
 6453   dst    : S5(write);
 6454   INS0   : ISS;
 6455   NEON_FP : S5;
 6456 %}
 6457 
 6458 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6459 %{
 6460   single_instruction;
 6461   src1   : S1(read);
 6462   src2   : S2(read);
 6463   dst    : S5(write);
 6464   INS0   : ISS;
 6465   NEON_FP : S5;
 6466 %}
 6467 
 6468 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6469 %{
 6470   single_instruction;
 6471   cr     : S1(read);
 6472   src1   : S1(read);
 6473   src2   : S1(read);
 6474   dst    : S3(write);
 6475   INS01  : ISS;
 6476   NEON_FP : S3;
 6477 %}
 6478 
 6479 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6480 %{
 6481   single_instruction;
 6482   cr     : S1(read);
 6483   src1   : S1(read);
 6484   src2   : S1(read);
 6485   dst    : S3(write);
 6486   INS01  : ISS;
 6487   NEON_FP : S3;
 6488 %}
 6489 
 6490 pipe_class fp_imm_s(vRegF dst)
 6491 %{
 6492   single_instruction;
 6493   dst    : S3(write);
 6494   INS01  : ISS;
 6495   NEON_FP : S3;
 6496 %}
 6497 
 6498 pipe_class fp_imm_d(vRegD dst)
 6499 %{
 6500   single_instruction;
 6501   dst    : S3(write);
 6502   INS01  : ISS;
 6503   NEON_FP : S3;
 6504 %}
 6505 
 6506 pipe_class fp_load_constant_s(vRegF dst)
 6507 %{
 6508   single_instruction;
 6509   dst    : S4(write);
 6510   INS01  : ISS;
 6511   NEON_FP : S4;
 6512 %}
 6513 
 6514 pipe_class fp_load_constant_d(vRegD dst)
 6515 %{
 6516   single_instruction;
 6517   dst    : S4(write);
 6518   INS01  : ISS;
 6519   NEON_FP : S4;
 6520 %}
 6521 
 6522 //------- Integer ALU operations --------------------------
 6523 
 6524 // Integer ALU reg-reg operation
 6525 // Operands needed in EX1, result generated in EX2
 6526 // Eg.  ADD     x0, x1, x2
 6527 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6528 %{
 6529   single_instruction;
 6530   dst    : EX2(write);
 6531   src1   : EX1(read);
 6532   src2   : EX1(read);
 6533   INS01  : ISS; // Dual issue as instruction 0 or 1
 6534   ALU    : EX2;
 6535 %}
 6536 
 6537 // Integer ALU reg-reg operation with constant shift
 6538 // Shifted register must be available in LATE_ISS instead of EX1
 6539 // Eg.  ADD     x0, x1, x2, LSL #2
 6540 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6541 %{
 6542   single_instruction;
 6543   dst    : EX2(write);
 6544   src1   : EX1(read);
 6545   src2   : ISS(read);
 6546   INS01  : ISS;
 6547   ALU    : EX2;
 6548 %}
 6549 
 6550 // Integer ALU reg operation with constant shift
 6551 // Eg.  LSL     x0, x1, #shift
 6552 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6553 %{
 6554   single_instruction;
 6555   dst    : EX2(write);
 6556   src1   : ISS(read);
 6557   INS01  : ISS;
 6558   ALU    : EX2;
 6559 %}
 6560 
 6561 // Integer ALU reg-reg operation with variable shift
 6562 // Both operands must be available in LATE_ISS instead of EX1
 6563 // Result is available in EX1 instead of EX2
 6564 // Eg.  LSLV    x0, x1, x2
 6565 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6566 %{
 6567   single_instruction;
 6568   dst    : EX1(write);
 6569   src1   : ISS(read);
 6570   src2   : ISS(read);
 6571   INS01  : ISS;
 6572   ALU    : EX1;
 6573 %}
 6574 
 6575 // Integer ALU reg-reg operation with extract
 6576 // As for _vshift above, but result generated in EX2
 6577 // Eg.  EXTR    x0, x1, x2, #N
 6578 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6579 %{
 6580   single_instruction;
 6581   dst    : EX2(write);
 6582   src1   : ISS(read);
 6583   src2   : ISS(read);
 6584   INS1   : ISS; // Can only dual issue as Instruction 1
 6585   ALU    : EX1;
 6586 %}
 6587 
 6588 // Integer ALU reg operation
 6589 // Eg.  NEG     x0, x1
 6590 pipe_class ialu_reg(iRegI dst, iRegI src)
 6591 %{
 6592   single_instruction;
 6593   dst    : EX2(write);
 6594   src    : EX1(read);
 6595   INS01  : ISS;
 6596   ALU    : EX2;
 6597 %}
 6598 
 6599 // Integer ALU reg mmediate operation
 6600 // Eg.  ADD     x0, x1, #N
 6601 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6602 %{
 6603   single_instruction;
 6604   dst    : EX2(write);
 6605   src1   : EX1(read);
 6606   INS01  : ISS;
 6607   ALU    : EX2;
 6608 %}
 6609 
 6610 // Integer ALU immediate operation (no source operands)
 6611 // Eg.  MOV     x0, #N
 6612 pipe_class ialu_imm(iRegI dst)
 6613 %{
 6614   single_instruction;
 6615   dst    : EX1(write);
 6616   INS01  : ISS;
 6617   ALU    : EX1;
 6618 %}
 6619 
 6620 //------- Compare operation -------------------------------
 6621 
 6622 // Compare reg-reg
 6623 // Eg.  CMP     x0, x1
 6624 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6625 %{
 6626   single_instruction;
 6627 //  fixed_latency(16);
 6628   cr     : EX2(write);
 6629   op1    : EX1(read);
 6630   op2    : EX1(read);
 6631   INS01  : ISS;
 6632   ALU    : EX2;
 6633 %}
 6634 
 6635 // Compare reg-reg
 6636 // Eg.  CMP     x0, #N
 6637 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6638 %{
 6639   single_instruction;
 6640 //  fixed_latency(16);
 6641   cr     : EX2(write);
 6642   op1    : EX1(read);
 6643   INS01  : ISS;
 6644   ALU    : EX2;
 6645 %}
 6646 
 6647 //------- Conditional instructions ------------------------
 6648 
 6649 // Conditional no operands
 6650 // Eg.  CSINC   x0, zr, zr, <cond>
 6651 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6652 %{
 6653   single_instruction;
 6654   cr     : EX1(read);
 6655   dst    : EX2(write);
 6656   INS01  : ISS;
 6657   ALU    : EX2;
 6658 %}
 6659 
 6660 // Conditional 2 operand
 6661 // EG.  CSEL    X0, X1, X2, <cond>
 6662 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6663 %{
 6664   single_instruction;
 6665   cr     : EX1(read);
 6666   src1   : EX1(read);
 6667   src2   : EX1(read);
 6668   dst    : EX2(write);
 6669   INS01  : ISS;
 6670   ALU    : EX2;
 6671 %}
 6672 
 6673 // Conditional 2 operand
 6674 // EG.  CSEL    X0, X1, X2, <cond>
 6675 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6676 %{
 6677   single_instruction;
 6678   cr     : EX1(read);
 6679   src    : EX1(read);
 6680   dst    : EX2(write);
 6681   INS01  : ISS;
 6682   ALU    : EX2;
 6683 %}
 6684 
 6685 //------- Multiply pipeline operations --------------------
 6686 
 6687 // Multiply reg-reg
 6688 // Eg.  MUL     w0, w1, w2
 6689 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6690 %{
 6691   single_instruction;
 6692   dst    : WR(write);
 6693   src1   : ISS(read);
 6694   src2   : ISS(read);
 6695   INS01  : ISS;
 6696   MAC    : WR;
 6697 %}
 6698 
 6699 // Multiply accumulate
 6700 // Eg.  MADD    w0, w1, w2, w3
 6701 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6702 %{
 6703   single_instruction;
 6704   dst    : WR(write);
 6705   src1   : ISS(read);
 6706   src2   : ISS(read);
 6707   src3   : ISS(read);
 6708   INS01  : ISS;
 6709   MAC    : WR;
 6710 %}
 6711 
 6712 // Eg.  MUL     w0, w1, w2
 6713 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6714 %{
 6715   single_instruction;
 6716   fixed_latency(3); // Maximum latency for 64 bit mul
 6717   dst    : WR(write);
 6718   src1   : ISS(read);
 6719   src2   : ISS(read);
 6720   INS01  : ISS;
 6721   MAC    : WR;
 6722 %}
 6723 
 6724 // Multiply accumulate
 6725 // Eg.  MADD    w0, w1, w2, w3
 6726 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6727 %{
 6728   single_instruction;
 6729   fixed_latency(3); // Maximum latency for 64 bit mul
 6730   dst    : WR(write);
 6731   src1   : ISS(read);
 6732   src2   : ISS(read);
 6733   src3   : ISS(read);
 6734   INS01  : ISS;
 6735   MAC    : WR;
 6736 %}
 6737 
 6738 //------- Divide pipeline operations --------------------
 6739 
 6740 // Eg.  SDIV    w0, w1, w2
 6741 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6742 %{
 6743   single_instruction;
 6744   fixed_latency(8); // Maximum latency for 32 bit divide
 6745   dst    : WR(write);
 6746   src1   : ISS(read);
 6747   src2   : ISS(read);
 6748   INS0   : ISS; // Can only dual issue as instruction 0
 6749   DIV    : WR;
 6750 %}
 6751 
 6752 // Eg.  SDIV    x0, x1, x2
 6753 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6754 %{
 6755   single_instruction;
 6756   fixed_latency(16); // Maximum latency for 64 bit divide
 6757   dst    : WR(write);
 6758   src1   : ISS(read);
 6759   src2   : ISS(read);
 6760   INS0   : ISS; // Can only dual issue as instruction 0
 6761   DIV    : WR;
 6762 %}
 6763 
 6764 //------- Load pipeline operations ------------------------
 6765 
 6766 // Load - prefetch
 6767 // Eg.  PFRM    <mem>
 6768 pipe_class iload_prefetch(memory mem)
 6769 %{
 6770   single_instruction;
 6771   mem    : ISS(read);
 6772   INS01  : ISS;
 6773   LDST   : WR;
 6774 %}
 6775 
 6776 // Load - reg, mem
 6777 // Eg.  LDR     x0, <mem>
 6778 pipe_class iload_reg_mem(iRegI dst, memory mem)
 6779 %{
 6780   single_instruction;
 6781   dst    : WR(write);
 6782   mem    : ISS(read);
 6783   INS01  : ISS;
 6784   LDST   : WR;
 6785 %}
 6786 
 6787 // Load - reg, reg
 6788 // Eg.  LDR     x0, [sp, x1]
 6789 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 6790 %{
 6791   single_instruction;
 6792   dst    : WR(write);
 6793   src    : ISS(read);
 6794   INS01  : ISS;
 6795   LDST   : WR;
 6796 %}
 6797 
 6798 //------- Store pipeline operations -----------------------
 6799 
 6800 // Store - zr, mem
 6801 // Eg.  STR     zr, <mem>
 6802 pipe_class istore_mem(memory mem)
 6803 %{
 6804   single_instruction;
 6805   mem    : ISS(read);
 6806   INS01  : ISS;
 6807   LDST   : WR;
 6808 %}
 6809 
 6810 // Store - reg, mem
 6811 // Eg.  STR     x0, <mem>
 6812 pipe_class istore_reg_mem(iRegI src, memory mem)
 6813 %{
 6814   single_instruction;
 6815   mem    : ISS(read);
 6816   src    : EX2(read);
 6817   INS01  : ISS;
 6818   LDST   : WR;
 6819 %}
 6820 
 6821 // Store - reg, reg
 6822 // Eg. STR      x0, [sp, x1]
 6823 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 6824 %{
 6825   single_instruction;
 6826   dst    : ISS(read);
 6827   src    : EX2(read);
 6828   INS01  : ISS;
 6829   LDST   : WR;
 6830 %}
 6831 
 6832 //------- Store pipeline operations -----------------------
 6833 
 6834 // Branch
 6835 pipe_class pipe_branch()
 6836 %{
 6837   single_instruction;
 6838   INS01  : ISS;
 6839   BRANCH : EX1;
 6840 %}
 6841 
 6842 // Conditional branch
 6843 pipe_class pipe_branch_cond(rFlagsReg cr)
 6844 %{
 6845   single_instruction;
 6846   cr     : EX1(read);
 6847   INS01  : ISS;
 6848   BRANCH : EX1;
 6849 %}
 6850 
 6851 // Compare & Branch
 6852 // EG.  CBZ/CBNZ
 6853 pipe_class pipe_cmp_branch(iRegI op1)
 6854 %{
 6855   single_instruction;
 6856   op1    : EX1(read);
 6857   INS01  : ISS;
 6858   BRANCH : EX1;
 6859 %}
 6860 
 6861 //------- Synchronisation operations ----------------------
 6862 
 6863 // Any operation requiring serialization.
 6864 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 6865 pipe_class pipe_serial()
 6866 %{
 6867   single_instruction;
 6868   force_serialization;
 6869   fixed_latency(16);
 6870   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6871   LDST   : WR;
 6872 %}
 6873 
 6874 // Generic big/slow expanded idiom - also serialized
 6875 pipe_class pipe_slow()
 6876 %{
 6877   instruction_count(10);
 6878   multiple_bundles;
 6879   force_serialization;
 6880   fixed_latency(16);
 6881   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6882   LDST   : WR;
 6883 %}
 6884 
 6885 // Empty pipeline class
 6886 pipe_class pipe_class_empty()
 6887 %{
 6888   single_instruction;
 6889   fixed_latency(0);
 6890 %}
 6891 
 6892 // Default pipeline class.
 6893 pipe_class pipe_class_default()
 6894 %{
 6895   single_instruction;
 6896   fixed_latency(2);
 6897 %}
 6898 
 6899 // Pipeline class for compares.
 6900 pipe_class pipe_class_compare()
 6901 %{
 6902   single_instruction;
 6903   fixed_latency(16);
 6904 %}
 6905 
 6906 // Pipeline class for memory operations.
 6907 pipe_class pipe_class_memory()
 6908 %{
 6909   single_instruction;
 6910   fixed_latency(16);
 6911 %}
 6912 
 6913 // Pipeline class for call.
 6914 pipe_class pipe_class_call()
 6915 %{
 6916   single_instruction;
 6917   fixed_latency(100);
 6918 %}
 6919 
 6920 // Define the class for the Nop node.
 6921 define %{
 6922    MachNop = pipe_class_empty;
 6923 %}
 6924 
 6925 %}
 6926 //----------INSTRUCTIONS-------------------------------------------------------
 6927 //
 6928 // match      -- States which machine-independent subtree may be replaced
 6929 //               by this instruction.
 6930 // ins_cost   -- The estimated cost of this instruction is used by instruction
 6931 //               selection to identify a minimum cost tree of machine
 6932 //               instructions that matches a tree of machine-independent
 6933 //               instructions.
 6934 // format     -- A string providing the disassembly for this instruction.
 6935 //               The value of an instruction's operand may be inserted
 6936 //               by referring to it with a '$' prefix.
 6937 // opcode     -- Three instruction opcodes may be provided.  These are referred
 6938 //               to within an encode class as $primary, $secondary, and $tertiary
 6939 //               rrspectively.  The primary opcode is commonly used to
 6940 //               indicate the type of machine instruction, while secondary
 6941 //               and tertiary are often used for prefix options or addressing
 6942 //               modes.
 6943 // ins_encode -- A list of encode classes with parameters. The encode class
 6944 //               name must have been defined in an 'enc_class' specification
 6945 //               in the encode section of the architecture description.
 6946 
 6947 // ============================================================================
 6948 // Memory (Load/Store) Instructions
 6949 
 6950 // Load Instructions
 6951 
 6952 // Load Byte (8 bit signed)
 6953 instruct loadB(iRegINoSp dst, memory1 mem)
 6954 %{
 6955   match(Set dst (LoadB mem));
 6956   predicate(!needs_acquiring_load(n));
 6957 
 6958   ins_cost(4 * INSN_COST);
 6959   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 6960 
 6961   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 6962 
 6963   ins_pipe(iload_reg_mem);
 6964 %}
 6965 
 6966 // Load Byte (8 bit signed) into long
 6967 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 6968 %{
 6969   match(Set dst (ConvI2L (LoadB mem)));
 6970   predicate(!needs_acquiring_load(n->in(1)));
 6971 
 6972   ins_cost(4 * INSN_COST);
 6973   format %{ "ldrsb  $dst, $mem\t# byte" %}
 6974 
 6975   ins_encode(aarch64_enc_ldrsb(dst, mem));
 6976 
 6977   ins_pipe(iload_reg_mem);
 6978 %}
 6979 
 6980 // Load Byte (8 bit unsigned)
 6981 instruct loadUB(iRegINoSp dst, memory1 mem)
 6982 %{
 6983   match(Set dst (LoadUB mem));
 6984   predicate(!needs_acquiring_load(n));
 6985 
 6986   ins_cost(4 * INSN_COST);
 6987   format %{ "ldrbw  $dst, $mem\t# byte" %}
 6988 
 6989   ins_encode(aarch64_enc_ldrb(dst, mem));
 6990 
 6991   ins_pipe(iload_reg_mem);
 6992 %}
 6993 
 6994 // Load Byte (8 bit unsigned) into long
 6995 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 6996 %{
 6997   match(Set dst (ConvI2L (LoadUB mem)));
 6998   predicate(!needs_acquiring_load(n->in(1)));
 6999 
 7000   ins_cost(4 * INSN_COST);
 7001   format %{ "ldrb  $dst, $mem\t# byte" %}
 7002 
 7003   ins_encode(aarch64_enc_ldrb(dst, mem));
 7004 
 7005   ins_pipe(iload_reg_mem);
 7006 %}
 7007 
 7008 // Load Short (16 bit signed)
 7009 instruct loadS(iRegINoSp dst, memory2 mem)
 7010 %{
 7011   match(Set dst (LoadS mem));
 7012   predicate(!needs_acquiring_load(n));
 7013 
 7014   ins_cost(4 * INSN_COST);
 7015   format %{ "ldrshw  $dst, $mem\t# short" %}
 7016 
 7017   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7018 
 7019   ins_pipe(iload_reg_mem);
 7020 %}
 7021 
 7022 // Load Short (16 bit signed) into long
 7023 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7024 %{
 7025   match(Set dst (ConvI2L (LoadS mem)));
 7026   predicate(!needs_acquiring_load(n->in(1)));
 7027 
 7028   ins_cost(4 * INSN_COST);
 7029   format %{ "ldrsh  $dst, $mem\t# short" %}
 7030 
 7031   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7032 
 7033   ins_pipe(iload_reg_mem);
 7034 %}
 7035 
 7036 // Load Char (16 bit unsigned)
 7037 instruct loadUS(iRegINoSp dst, memory2 mem)
 7038 %{
 7039   match(Set dst (LoadUS mem));
 7040   predicate(!needs_acquiring_load(n));
 7041 
 7042   ins_cost(4 * INSN_COST);
 7043   format %{ "ldrh  $dst, $mem\t# short" %}
 7044 
 7045   ins_encode(aarch64_enc_ldrh(dst, mem));
 7046 
 7047   ins_pipe(iload_reg_mem);
 7048 %}
 7049 
 7050 // Load Short/Char (16 bit unsigned) into long
 7051 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7052 %{
 7053   match(Set dst (ConvI2L (LoadUS mem)));
 7054   predicate(!needs_acquiring_load(n->in(1)));
 7055 
 7056   ins_cost(4 * INSN_COST);
 7057   format %{ "ldrh  $dst, $mem\t# short" %}
 7058 
 7059   ins_encode(aarch64_enc_ldrh(dst, mem));
 7060 
 7061   ins_pipe(iload_reg_mem);
 7062 %}
 7063 
 7064 // Load Integer (32 bit signed)
 7065 instruct loadI(iRegINoSp dst, memory4 mem)
 7066 %{
 7067   match(Set dst (LoadI mem));
 7068   predicate(!needs_acquiring_load(n));
 7069 
 7070   ins_cost(4 * INSN_COST);
 7071   format %{ "ldrw  $dst, $mem\t# int" %}
 7072 
 7073   ins_encode(aarch64_enc_ldrw(dst, mem));
 7074 
 7075   ins_pipe(iload_reg_mem);
 7076 %}
 7077 
 7078 // Load Integer (32 bit signed) into long
 7079 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7080 %{
 7081   match(Set dst (ConvI2L (LoadI mem)));
 7082   predicate(!needs_acquiring_load(n->in(1)));
 7083 
 7084   ins_cost(4 * INSN_COST);
 7085   format %{ "ldrsw  $dst, $mem\t# int" %}
 7086 
 7087   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7088 
 7089   ins_pipe(iload_reg_mem);
 7090 %}
 7091 
 7092 // Load Integer (32 bit unsigned) into long
 7093 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7094 %{
 7095   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7096   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7097 
 7098   ins_cost(4 * INSN_COST);
 7099   format %{ "ldrw  $dst, $mem\t# int" %}
 7100 
 7101   ins_encode(aarch64_enc_ldrw(dst, mem));
 7102 
 7103   ins_pipe(iload_reg_mem);
 7104 %}
 7105 
 7106 // Load Long (64 bit signed)
 7107 instruct loadL(iRegLNoSp dst, memory8 mem)
 7108 %{
 7109   match(Set dst (LoadL mem));
 7110   predicate(!needs_acquiring_load(n));
 7111 
 7112   ins_cost(4 * INSN_COST);
 7113   format %{ "ldr  $dst, $mem\t# int" %}
 7114 
 7115   ins_encode(aarch64_enc_ldr(dst, mem));
 7116 
 7117   ins_pipe(iload_reg_mem);
 7118 %}
 7119 
 7120 // Load Range
 7121 instruct loadRange(iRegINoSp dst, memory4 mem)
 7122 %{
 7123   match(Set dst (LoadRange mem));
 7124 
 7125   ins_cost(4 * INSN_COST);
 7126   format %{ "ldrw  $dst, $mem\t# range" %}
 7127 
 7128   ins_encode(aarch64_enc_ldrw(dst, mem));
 7129 
 7130   ins_pipe(iload_reg_mem);
 7131 %}
 7132 
 7133 // Load Pointer
 7134 instruct loadP(iRegPNoSp dst, memory8 mem)
 7135 %{
 7136   match(Set dst (LoadP mem));
 7137   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7138 
 7139   ins_cost(4 * INSN_COST);
 7140   format %{ "ldr  $dst, $mem\t# ptr" %}
 7141 
 7142   ins_encode(aarch64_enc_ldr(dst, mem));
 7143 
 7144   ins_pipe(iload_reg_mem);
 7145 %}
 7146 
 7147 // Load Compressed Pointer
 7148 instruct loadN(iRegNNoSp dst, memory4 mem)
 7149 %{
 7150   match(Set dst (LoadN mem));
 7151   predicate(!needs_acquiring_load(n));
 7152 
 7153   ins_cost(4 * INSN_COST);
 7154   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7155 
 7156   ins_encode(aarch64_enc_ldrw(dst, mem));
 7157 
 7158   ins_pipe(iload_reg_mem);
 7159 %}
 7160 
 7161 // Load Klass Pointer
 7162 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7163 %{
 7164   match(Set dst (LoadKlass mem));
 7165   predicate(!needs_acquiring_load(n));
 7166 
 7167   ins_cost(4 * INSN_COST);
 7168   format %{ "ldr  $dst, $mem\t# class" %}
 7169 
 7170   ins_encode(aarch64_enc_ldr(dst, mem));
 7171 
 7172   ins_pipe(iload_reg_mem);
 7173 %}
 7174 
 7175 // Load Narrow Klass Pointer
 7176 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7177 %{
 7178   match(Set dst (LoadNKlass mem));
 7179   predicate(!needs_acquiring_load(n));
 7180 
 7181   ins_cost(4 * INSN_COST);
 7182   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7183 
 7184   ins_encode(aarch64_enc_ldrw(dst, mem));
 7185 
 7186   ins_pipe(iload_reg_mem);
 7187 %}
 7188 
 7189 // Load Float
 7190 instruct loadF(vRegF dst, memory4 mem)
 7191 %{
 7192   match(Set dst (LoadF mem));
 7193   predicate(!needs_acquiring_load(n));
 7194 
 7195   ins_cost(4 * INSN_COST);
 7196   format %{ "ldrs  $dst, $mem\t# float" %}
 7197 
 7198   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7199 
 7200   ins_pipe(pipe_class_memory);
 7201 %}
 7202 
 7203 // Load Double
 7204 instruct loadD(vRegD dst, memory8 mem)
 7205 %{
 7206   match(Set dst (LoadD mem));
 7207   predicate(!needs_acquiring_load(n));
 7208 
 7209   ins_cost(4 * INSN_COST);
 7210   format %{ "ldrd  $dst, $mem\t# double" %}
 7211 
 7212   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7213 
 7214   ins_pipe(pipe_class_memory);
 7215 %}
 7216 
 7217 
 7218 // Load Int Constant
 7219 instruct loadConI(iRegINoSp dst, immI src)
 7220 %{
 7221   match(Set dst src);
 7222 
 7223   ins_cost(INSN_COST);
 7224   format %{ "mov $dst, $src\t# int" %}
 7225 
 7226   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7227 
 7228   ins_pipe(ialu_imm);
 7229 %}
 7230 
 7231 // Load Long Constant
 7232 instruct loadConL(iRegLNoSp dst, immL src)
 7233 %{
 7234   match(Set dst src);
 7235 
 7236   ins_cost(INSN_COST);
 7237   format %{ "mov $dst, $src\t# long" %}
 7238 
 7239   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7240 
 7241   ins_pipe(ialu_imm);
 7242 %}
 7243 
 7244 // Load Pointer Constant
 7245 
 7246 instruct loadConP(iRegPNoSp dst, immP con)
 7247 %{
 7248   match(Set dst con);
 7249 
 7250   ins_cost(INSN_COST * 4);
 7251   format %{
 7252     "mov  $dst, $con\t# ptr\n\t"
 7253   %}
 7254 
 7255   ins_encode(aarch64_enc_mov_p(dst, con));
 7256 
 7257   ins_pipe(ialu_imm);
 7258 %}
 7259 
 7260 // Load Null Pointer Constant
 7261 
 7262 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7263 %{
 7264   match(Set dst con);
 7265 
 7266   ins_cost(INSN_COST);
 7267   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7268 
 7269   ins_encode(aarch64_enc_mov_p0(dst, con));
 7270 
 7271   ins_pipe(ialu_imm);
 7272 %}
 7273 
 7274 // Load Pointer Constant One
 7275 
 7276 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7277 %{
 7278   match(Set dst con);
 7279 
 7280   ins_cost(INSN_COST);
 7281   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7282 
 7283   ins_encode(aarch64_enc_mov_p1(dst, con));
 7284 
 7285   ins_pipe(ialu_imm);
 7286 %}
 7287 
 7288 // Load Byte Map Base Constant
 7289 
 7290 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7291 %{
 7292   match(Set dst con);
 7293 
 7294   ins_cost(INSN_COST);
 7295   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7296 
 7297   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7298 
 7299   ins_pipe(ialu_imm);
 7300 %}
 7301 
 7302 // Load Narrow Pointer Constant
 7303 
 7304 instruct loadConN(iRegNNoSp dst, immN con)
 7305 %{
 7306   match(Set dst con);
 7307 
 7308   ins_cost(INSN_COST * 4);
 7309   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7310 
 7311   ins_encode(aarch64_enc_mov_n(dst, con));
 7312 
 7313   ins_pipe(ialu_imm);
 7314 %}
 7315 
 7316 // Load Narrow Null Pointer Constant
 7317 
 7318 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7319 %{
 7320   match(Set dst con);
 7321 
 7322   ins_cost(INSN_COST);
 7323   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7324 
 7325   ins_encode(aarch64_enc_mov_n0(dst, con));
 7326 
 7327   ins_pipe(ialu_imm);
 7328 %}
 7329 
 7330 // Load Narrow Klass Constant
 7331 
 7332 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7333 %{
 7334   match(Set dst con);
 7335 
 7336   ins_cost(INSN_COST);
 7337   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7338 
 7339   ins_encode(aarch64_enc_mov_nk(dst, con));
 7340 
 7341   ins_pipe(ialu_imm);
 7342 %}
 7343 
 7344 // Load Packed Float Constant
 7345 
 7346 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7347   match(Set dst con);
 7348   ins_cost(INSN_COST * 4);
 7349   format %{ "fmovs  $dst, $con"%}
 7350   ins_encode %{
 7351     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7352   %}
 7353 
 7354   ins_pipe(fp_imm_s);
 7355 %}
 7356 
 7357 // Load Float Constant
 7358 
 7359 instruct loadConF(vRegF dst, immF con) %{
 7360   match(Set dst con);
 7361 
 7362   ins_cost(INSN_COST * 4);
 7363 
 7364   format %{
 7365     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7366   %}
 7367 
 7368   ins_encode %{
 7369     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7370   %}
 7371 
 7372   ins_pipe(fp_load_constant_s);
 7373 %}
 7374 
 7375 // Load Packed Double Constant
 7376 
 7377 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7378   match(Set dst con);
 7379   ins_cost(INSN_COST);
 7380   format %{ "fmovd  $dst, $con"%}
 7381   ins_encode %{
 7382     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7383   %}
 7384 
 7385   ins_pipe(fp_imm_d);
 7386 %}
 7387 
 7388 // Load Double Constant
 7389 
 7390 instruct loadConD(vRegD dst, immD con) %{
 7391   match(Set dst con);
 7392 
 7393   ins_cost(INSN_COST * 5);
 7394   format %{
 7395     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7396   %}
 7397 
 7398   ins_encode %{
 7399     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7400   %}
 7401 
 7402   ins_pipe(fp_load_constant_d);
 7403 %}
 7404 
 7405 // Store Instructions
 7406 
 7407 // Store CMS card-mark Immediate
 7408 instruct storeimmCM0(immI0 zero, memory1 mem)
 7409 %{
 7410   match(Set mem (StoreCM mem zero));
 7411 
 7412   ins_cost(INSN_COST);
 7413   format %{ "storestore (elided)\n\t"
 7414             "strb zr, $mem\t# byte" %}
 7415 
 7416   ins_encode(aarch64_enc_strb0(mem));
 7417 
 7418   ins_pipe(istore_mem);
 7419 %}
 7420 
 7421 // Store CMS card-mark Immediate with intervening StoreStore
 7422 // needed when using CMS with no conditional card marking
 7423 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7424 %{
 7425   match(Set mem (StoreCM mem zero));
 7426 
 7427   ins_cost(INSN_COST * 2);
 7428   format %{ "storestore\n\t"
 7429             "dmb ishst"
 7430             "\n\tstrb zr, $mem\t# byte" %}
 7431 
 7432   ins_encode(aarch64_enc_strb0_ordered(mem));
 7433 
 7434   ins_pipe(istore_mem);
 7435 %}
 7436 
 7437 // Store Byte
 7438 instruct storeB(iRegIorL2I src, memory1 mem)
 7439 %{
 7440   match(Set mem (StoreB mem src));
 7441   predicate(!needs_releasing_store(n));
 7442 
 7443   ins_cost(INSN_COST);
 7444   format %{ "strb  $src, $mem\t# byte" %}
 7445 
 7446   ins_encode(aarch64_enc_strb(src, mem));
 7447 
 7448   ins_pipe(istore_reg_mem);
 7449 %}
 7450 
 7451 
 7452 instruct storeimmB0(immI0 zero, memory1 mem)
 7453 %{
 7454   match(Set mem (StoreB mem zero));
 7455   predicate(!needs_releasing_store(n));
 7456 
 7457   ins_cost(INSN_COST);
 7458   format %{ "strb rscractch2, $mem\t# byte" %}
 7459 
 7460   ins_encode(aarch64_enc_strb0(mem));
 7461 
 7462   ins_pipe(istore_mem);
 7463 %}
 7464 
 7465 // Store Char/Short
 7466 instruct storeC(iRegIorL2I src, memory2 mem)
 7467 %{
 7468   match(Set mem (StoreC mem src));
 7469   predicate(!needs_releasing_store(n));
 7470 
 7471   ins_cost(INSN_COST);
 7472   format %{ "strh  $src, $mem\t# short" %}
 7473 
 7474   ins_encode(aarch64_enc_strh(src, mem));
 7475 
 7476   ins_pipe(istore_reg_mem);
 7477 %}
 7478 
 7479 instruct storeimmC0(immI0 zero, memory2 mem)
 7480 %{
 7481   match(Set mem (StoreC mem zero));
 7482   predicate(!needs_releasing_store(n));
 7483 
 7484   ins_cost(INSN_COST);
 7485   format %{ "strh  zr, $mem\t# short" %}
 7486 
 7487   ins_encode(aarch64_enc_strh0(mem));
 7488 
 7489   ins_pipe(istore_mem);
 7490 %}
 7491 
 7492 // Store Integer
 7493 
 7494 instruct storeI(iRegIorL2I src, memory4 mem)
 7495 %{
 7496   match(Set mem(StoreI mem src));
 7497   predicate(!needs_releasing_store(n));
 7498 
 7499   ins_cost(INSN_COST);
 7500   format %{ "strw  $src, $mem\t# int" %}
 7501 
 7502   ins_encode(aarch64_enc_strw(src, mem));
 7503 
 7504   ins_pipe(istore_reg_mem);
 7505 %}
 7506 
 7507 instruct storeimmI0(immI0 zero, memory4 mem)
 7508 %{
 7509   match(Set mem(StoreI mem zero));
 7510   predicate(!needs_releasing_store(n));
 7511 
 7512   ins_cost(INSN_COST);
 7513   format %{ "strw  zr, $mem\t# int" %}
 7514 
 7515   ins_encode(aarch64_enc_strw0(mem));
 7516 
 7517   ins_pipe(istore_mem);
 7518 %}
 7519 
 7520 // Store Long (64 bit signed)
 7521 instruct storeL(iRegL src, memory8 mem)
 7522 %{
 7523   match(Set mem (StoreL mem src));
 7524   predicate(!needs_releasing_store(n));
 7525 
 7526   ins_cost(INSN_COST);
 7527   format %{ "str  $src, $mem\t# int" %}
 7528 
 7529   ins_encode(aarch64_enc_str(src, mem));
 7530 
 7531   ins_pipe(istore_reg_mem);
 7532 %}
 7533 
 7534 // Store Long (64 bit signed)
 7535 instruct storeimmL0(immL0 zero, memory8 mem)
 7536 %{
 7537   match(Set mem (StoreL mem zero));
 7538   predicate(!needs_releasing_store(n));
 7539 
 7540   ins_cost(INSN_COST);
 7541   format %{ "str  zr, $mem\t# int" %}
 7542 
 7543   ins_encode(aarch64_enc_str0(mem));
 7544 
 7545   ins_pipe(istore_mem);
 7546 %}
 7547 
 7548 // Store Pointer
 7549 instruct storeP(iRegP src, memory8 mem)
 7550 %{
 7551   match(Set mem (StoreP mem src));
 7552   predicate(!needs_releasing_store(n));
 7553 
 7554   ins_cost(INSN_COST);
 7555   format %{ "str  $src, $mem\t# ptr" %}
 7556 
 7557   ins_encode(aarch64_enc_str(src, mem));
 7558 
 7559   ins_pipe(istore_reg_mem);
 7560 %}
 7561 
 7562 // Store Pointer
 7563 instruct storeimmP0(immP0 zero, memory8 mem)
 7564 %{
 7565   match(Set mem (StoreP mem zero));
 7566   predicate(!needs_releasing_store(n));
 7567 
 7568   ins_cost(INSN_COST);
 7569   format %{ "str zr, $mem\t# ptr" %}
 7570 
 7571   ins_encode(aarch64_enc_str0(mem));
 7572 
 7573   ins_pipe(istore_mem);
 7574 %}
 7575 
 7576 // Store Compressed Pointer
 7577 instruct storeN(iRegN src, memory4 mem)
 7578 %{
 7579   match(Set mem (StoreN mem src));
 7580   predicate(!needs_releasing_store(n));
 7581 
 7582   ins_cost(INSN_COST);
 7583   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7584 
 7585   ins_encode(aarch64_enc_strw(src, mem));
 7586 
 7587   ins_pipe(istore_reg_mem);
 7588 %}
 7589 
 7590 instruct storeImmN0(immN0 zero, memory4 mem)
 7591 %{
 7592   match(Set mem (StoreN mem zero));
 7593   predicate(!needs_releasing_store(n));
 7594 
 7595   ins_cost(INSN_COST);
 7596   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7597 
 7598   ins_encode(aarch64_enc_strw0(mem));
 7599 
 7600   ins_pipe(istore_mem);
 7601 %}
 7602 
 7603 // Store Float
 7604 instruct storeF(vRegF src, memory4 mem)
 7605 %{
 7606   match(Set mem (StoreF mem src));
 7607   predicate(!needs_releasing_store(n));
 7608 
 7609   ins_cost(INSN_COST);
 7610   format %{ "strs  $src, $mem\t# float" %}
 7611 
 7612   ins_encode( aarch64_enc_strs(src, mem) );
 7613 
 7614   ins_pipe(pipe_class_memory);
 7615 %}
 7616 
 7617 // TODO
 7618 // implement storeImmF0 and storeFImmPacked
 7619 
 7620 // Store Double
 7621 instruct storeD(vRegD src, memory8 mem)
 7622 %{
 7623   match(Set mem (StoreD mem src));
 7624   predicate(!needs_releasing_store(n));
 7625 
 7626   ins_cost(INSN_COST);
 7627   format %{ "strd  $src, $mem\t# double" %}
 7628 
 7629   ins_encode( aarch64_enc_strd(src, mem) );
 7630 
 7631   ins_pipe(pipe_class_memory);
 7632 %}
 7633 
 7634 // Store Compressed Klass Pointer
 7635 instruct storeNKlass(iRegN src, memory4 mem)
 7636 %{
 7637   predicate(!needs_releasing_store(n));
 7638   match(Set mem (StoreNKlass mem src));
 7639 
 7640   ins_cost(INSN_COST);
 7641   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7642 
 7643   ins_encode(aarch64_enc_strw(src, mem));
 7644 
 7645   ins_pipe(istore_reg_mem);
 7646 %}
 7647 
 7648 // TODO
 7649 // implement storeImmD0 and storeDImmPacked
 7650 
 7651 // prefetch instructions
 7652 // Must be safe to execute with invalid address (cannot fault).
 7653 
 7654 instruct prefetchalloc( memory8 mem ) %{
 7655   match(PrefetchAllocation mem);
 7656 
 7657   ins_cost(INSN_COST);
 7658   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7659 
 7660   ins_encode( aarch64_enc_prefetchw(mem) );
 7661 
 7662   ins_pipe(iload_prefetch);
 7663 %}
 7664 
 7665 //  ---------------- volatile loads and stores ----------------
 7666 
 7667 // Load Byte (8 bit signed)
 7668 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7669 %{
 7670   match(Set dst (LoadB mem));
 7671 
 7672   ins_cost(VOLATILE_REF_COST);
 7673   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7674 
 7675   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7676 
 7677   ins_pipe(pipe_serial);
 7678 %}
 7679 
 7680 // Load Byte (8 bit signed) into long
 7681 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7682 %{
 7683   match(Set dst (ConvI2L (LoadB mem)));
 7684 
 7685   ins_cost(VOLATILE_REF_COST);
 7686   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7687 
 7688   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7689 
 7690   ins_pipe(pipe_serial);
 7691 %}
 7692 
 7693 // Load Byte (8 bit unsigned)
 7694 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7695 %{
 7696   match(Set dst (LoadUB mem));
 7697 
 7698   ins_cost(VOLATILE_REF_COST);
 7699   format %{ "ldarb  $dst, $mem\t# byte" %}
 7700 
 7701   ins_encode(aarch64_enc_ldarb(dst, mem));
 7702 
 7703   ins_pipe(pipe_serial);
 7704 %}
 7705 
 7706 // Load Byte (8 bit unsigned) into long
 7707 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7708 %{
 7709   match(Set dst (ConvI2L (LoadUB mem)));
 7710 
 7711   ins_cost(VOLATILE_REF_COST);
 7712   format %{ "ldarb  $dst, $mem\t# byte" %}
 7713 
 7714   ins_encode(aarch64_enc_ldarb(dst, mem));
 7715 
 7716   ins_pipe(pipe_serial);
 7717 %}
 7718 
 7719 // Load Short (16 bit signed)
 7720 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7721 %{
 7722   match(Set dst (LoadS mem));
 7723 
 7724   ins_cost(VOLATILE_REF_COST);
 7725   format %{ "ldarshw  $dst, $mem\t# short" %}
 7726 
 7727   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7728 
 7729   ins_pipe(pipe_serial);
 7730 %}
 7731 
 7732 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7733 %{
 7734   match(Set dst (LoadUS mem));
 7735 
 7736   ins_cost(VOLATILE_REF_COST);
 7737   format %{ "ldarhw  $dst, $mem\t# short" %}
 7738 
 7739   ins_encode(aarch64_enc_ldarhw(dst, mem));
 7740 
 7741   ins_pipe(pipe_serial);
 7742 %}
 7743 
 7744 // Load Short/Char (16 bit unsigned) into long
 7745 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7746 %{
 7747   match(Set dst (ConvI2L (LoadUS mem)));
 7748 
 7749   ins_cost(VOLATILE_REF_COST);
 7750   format %{ "ldarh  $dst, $mem\t# short" %}
 7751 
 7752   ins_encode(aarch64_enc_ldarh(dst, mem));
 7753 
 7754   ins_pipe(pipe_serial);
 7755 %}
 7756 
 7757 // Load Short/Char (16 bit signed) into long
 7758 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7759 %{
 7760   match(Set dst (ConvI2L (LoadS mem)));
 7761 
 7762   ins_cost(VOLATILE_REF_COST);
 7763   format %{ "ldarh  $dst, $mem\t# short" %}
 7764 
 7765   ins_encode(aarch64_enc_ldarsh(dst, mem));
 7766 
 7767   ins_pipe(pipe_serial);
 7768 %}
 7769 
 7770 // Load Integer (32 bit signed)
 7771 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7772 %{
 7773   match(Set dst (LoadI mem));
 7774 
 7775   ins_cost(VOLATILE_REF_COST);
 7776   format %{ "ldarw  $dst, $mem\t# int" %}
 7777 
 7778   ins_encode(aarch64_enc_ldarw(dst, mem));
 7779 
 7780   ins_pipe(pipe_serial);
 7781 %}
 7782 
 7783 // Load Integer (32 bit unsigned) into long
 7784 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 7785 %{
 7786   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7787 
 7788   ins_cost(VOLATILE_REF_COST);
 7789   format %{ "ldarw  $dst, $mem\t# int" %}
 7790 
 7791   ins_encode(aarch64_enc_ldarw(dst, mem));
 7792 
 7793   ins_pipe(pipe_serial);
 7794 %}
 7795 
 7796 // Load Long (64 bit signed)
 7797 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7798 %{
 7799   match(Set dst (LoadL mem));
 7800 
 7801   ins_cost(VOLATILE_REF_COST);
 7802   format %{ "ldar  $dst, $mem\t# int" %}
 7803 
 7804   ins_encode(aarch64_enc_ldar(dst, mem));
 7805 
 7806   ins_pipe(pipe_serial);
 7807 %}
 7808 
 7809 // Load Pointer
 7810 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 7811 %{
 7812   match(Set dst (LoadP mem));
 7813   predicate(n->as_Load()->barrier_data() == 0);
 7814 
 7815   ins_cost(VOLATILE_REF_COST);
 7816   format %{ "ldar  $dst, $mem\t# ptr" %}
 7817 
 7818   ins_encode(aarch64_enc_ldar(dst, mem));
 7819 
 7820   ins_pipe(pipe_serial);
 7821 %}
 7822 
 7823 // Load Compressed Pointer
 7824 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 7825 %{
 7826   match(Set dst (LoadN mem));
 7827 
 7828   ins_cost(VOLATILE_REF_COST);
 7829   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 7830 
 7831   ins_encode(aarch64_enc_ldarw(dst, mem));
 7832 
 7833   ins_pipe(pipe_serial);
 7834 %}
 7835 
 7836 // Load Float
 7837 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 7838 %{
 7839   match(Set dst (LoadF mem));
 7840 
 7841   ins_cost(VOLATILE_REF_COST);
 7842   format %{ "ldars  $dst, $mem\t# float" %}
 7843 
 7844   ins_encode( aarch64_enc_fldars(dst, mem) );
 7845 
 7846   ins_pipe(pipe_serial);
 7847 %}
 7848 
 7849 // Load Double
 7850 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 7851 %{
 7852   match(Set dst (LoadD mem));
 7853 
 7854   ins_cost(VOLATILE_REF_COST);
 7855   format %{ "ldard  $dst, $mem\t# double" %}
 7856 
 7857   ins_encode( aarch64_enc_fldard(dst, mem) );
 7858 
 7859   ins_pipe(pipe_serial);
 7860 %}
 7861 
 7862 // Store Byte
 7863 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7864 %{
 7865   match(Set mem (StoreB mem src));
 7866 
 7867   ins_cost(VOLATILE_REF_COST);
 7868   format %{ "stlrb  $src, $mem\t# byte" %}
 7869 
 7870   ins_encode(aarch64_enc_stlrb(src, mem));
 7871 
 7872   ins_pipe(pipe_class_memory);
 7873 %}
 7874 
 7875 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7876 %{
 7877   match(Set mem (StoreB mem zero));
 7878 
 7879   ins_cost(VOLATILE_REF_COST);
 7880   format %{ "stlrb  zr, $mem\t# byte" %}
 7881 
 7882   ins_encode(aarch64_enc_stlrb0(mem));
 7883 
 7884   ins_pipe(pipe_class_memory);
 7885 %}
 7886 
 7887 // Store Char/Short
 7888 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7889 %{
 7890   match(Set mem (StoreC mem src));
 7891 
 7892   ins_cost(VOLATILE_REF_COST);
 7893   format %{ "stlrh  $src, $mem\t# short" %}
 7894 
 7895   ins_encode(aarch64_enc_stlrh(src, mem));
 7896 
 7897   ins_pipe(pipe_class_memory);
 7898 %}
 7899 
 7900 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7901 %{
 7902   match(Set mem (StoreC mem zero));
 7903 
 7904   ins_cost(VOLATILE_REF_COST);
 7905   format %{ "stlrh  zr, $mem\t# short" %}
 7906 
 7907   ins_encode(aarch64_enc_stlrh0(mem));
 7908 
 7909   ins_pipe(pipe_class_memory);
 7910 %}
 7911 
 7912 // Store Integer
 7913 
 7914 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7915 %{
 7916   match(Set mem(StoreI mem src));
 7917 
 7918   ins_cost(VOLATILE_REF_COST);
 7919   format %{ "stlrw  $src, $mem\t# int" %}
 7920 
 7921   ins_encode(aarch64_enc_stlrw(src, mem));
 7922 
 7923   ins_pipe(pipe_class_memory);
 7924 %}
 7925 
 7926 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7927 %{
 7928   match(Set mem(StoreI mem zero));
 7929 
 7930   ins_cost(VOLATILE_REF_COST);
 7931   format %{ "stlrw  zr, $mem\t# int" %}
 7932 
 7933   ins_encode(aarch64_enc_stlrw0(mem));
 7934 
 7935   ins_pipe(pipe_class_memory);
 7936 %}
 7937 
 7938 // Store Long (64 bit signed)
 7939 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 7940 %{
 7941   match(Set mem (StoreL mem src));
 7942 
 7943   ins_cost(VOLATILE_REF_COST);
 7944   format %{ "stlr  $src, $mem\t# int" %}
 7945 
 7946   ins_encode(aarch64_enc_stlr(src, mem));
 7947 
 7948   ins_pipe(pipe_class_memory);
 7949 %}
 7950 
 7951 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 7952 %{
 7953   match(Set mem (StoreL mem zero));
 7954 
 7955   ins_cost(VOLATILE_REF_COST);
 7956   format %{ "stlr  zr, $mem\t# int" %}
 7957 
 7958   ins_encode(aarch64_enc_stlr0(mem));
 7959 
 7960   ins_pipe(pipe_class_memory);
 7961 %}
 7962 
 7963 // Store Pointer
 7964 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 7965 %{
 7966   match(Set mem (StoreP mem src));
 7967 
 7968   ins_cost(VOLATILE_REF_COST);
 7969   format %{ "stlr  $src, $mem\t# ptr" %}
 7970 
 7971   ins_encode(aarch64_enc_stlr(src, mem));
 7972 
 7973   ins_pipe(pipe_class_memory);
 7974 %}
 7975 
 7976 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 7977 %{
 7978   match(Set mem (StoreP mem zero));
 7979 
 7980   ins_cost(VOLATILE_REF_COST);
 7981   format %{ "stlr  zr, $mem\t# ptr" %}
 7982 
 7983   ins_encode(aarch64_enc_stlr0(mem));
 7984 
 7985   ins_pipe(pipe_class_memory);
 7986 %}
 7987 
 7988 // Store Compressed Pointer
 7989 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 7990 %{
 7991   match(Set mem (StoreN mem src));
 7992 
 7993   ins_cost(VOLATILE_REF_COST);
 7994   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 7995 
 7996   ins_encode(aarch64_enc_stlrw(src, mem));
 7997 
 7998   ins_pipe(pipe_class_memory);
 7999 %}
 8000 
 8001 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 8002 %{
 8003   match(Set mem (StoreN mem zero));
 8004 
 8005   ins_cost(VOLATILE_REF_COST);
 8006   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 8007 
 8008   ins_encode(aarch64_enc_stlrw0(mem));
 8009 
 8010   ins_pipe(pipe_class_memory);
 8011 %}
 8012 
 8013 // Store Float
 8014 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8015 %{
 8016   match(Set mem (StoreF mem src));
 8017 
 8018   ins_cost(VOLATILE_REF_COST);
 8019   format %{ "stlrs  $src, $mem\t# float" %}
 8020 
 8021   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8022 
 8023   ins_pipe(pipe_class_memory);
 8024 %}
 8025 
 8026 // TODO
 8027 // implement storeImmF0 and storeFImmPacked
 8028 
 8029 // Store Double
 8030 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8031 %{
 8032   match(Set mem (StoreD mem src));
 8033 
 8034   ins_cost(VOLATILE_REF_COST);
 8035   format %{ "stlrd  $src, $mem\t# double" %}
 8036 
 8037   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8038 
 8039   ins_pipe(pipe_class_memory);
 8040 %}
 8041 
 8042 //  ---------------- end of volatile loads and stores ----------------
 8043 
 8044 instruct cacheWB(indirect addr)
 8045 %{
 8046   predicate(VM_Version::supports_data_cache_line_flush());
 8047   match(CacheWB addr);
 8048 
 8049   ins_cost(100);
 8050   format %{"cache wb $addr" %}
 8051   ins_encode %{
 8052     assert($addr->index_position() < 0, "should be");
 8053     assert($addr$$disp == 0, "should be");
 8054     __ cache_wb(Address($addr$$base$$Register, 0));
 8055   %}
 8056   ins_pipe(pipe_slow); // XXX
 8057 %}
 8058 
 8059 instruct cacheWBPreSync()
 8060 %{
 8061   predicate(VM_Version::supports_data_cache_line_flush());
 8062   match(CacheWBPreSync);
 8063 
 8064   ins_cost(100);
 8065   format %{"cache wb presync" %}
 8066   ins_encode %{
 8067     __ cache_wbsync(true);
 8068   %}
 8069   ins_pipe(pipe_slow); // XXX
 8070 %}
 8071 
 8072 instruct cacheWBPostSync()
 8073 %{
 8074   predicate(VM_Version::supports_data_cache_line_flush());
 8075   match(CacheWBPostSync);
 8076 
 8077   ins_cost(100);
 8078   format %{"cache wb postsync" %}
 8079   ins_encode %{
 8080     __ cache_wbsync(false);
 8081   %}
 8082   ins_pipe(pipe_slow); // XXX
 8083 %}
 8084 
 8085 // ============================================================================
 8086 // BSWAP Instructions
 8087 
 8088 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8089   match(Set dst (ReverseBytesI src));
 8090 
 8091   ins_cost(INSN_COST);
 8092   format %{ "revw  $dst, $src" %}
 8093 
 8094   ins_encode %{
 8095     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8096   %}
 8097 
 8098   ins_pipe(ialu_reg);
 8099 %}
 8100 
 8101 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8102   match(Set dst (ReverseBytesL src));
 8103 
 8104   ins_cost(INSN_COST);
 8105   format %{ "rev  $dst, $src" %}
 8106 
 8107   ins_encode %{
 8108     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8109   %}
 8110 
 8111   ins_pipe(ialu_reg);
 8112 %}
 8113 
 8114 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8115   match(Set dst (ReverseBytesUS src));
 8116 
 8117   ins_cost(INSN_COST);
 8118   format %{ "rev16w  $dst, $src" %}
 8119 
 8120   ins_encode %{
 8121     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8122   %}
 8123 
 8124   ins_pipe(ialu_reg);
 8125 %}
 8126 
 8127 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8128   match(Set dst (ReverseBytesS src));
 8129 
 8130   ins_cost(INSN_COST);
 8131   format %{ "rev16w  $dst, $src\n\t"
 8132             "sbfmw $dst, $dst, #0, #15" %}
 8133 
 8134   ins_encode %{
 8135     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8136     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8137   %}
 8138 
 8139   ins_pipe(ialu_reg);
 8140 %}
 8141 
 8142 // ============================================================================
 8143 // Zero Count Instructions
 8144 
 8145 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8146   match(Set dst (CountLeadingZerosI src));
 8147 
 8148   ins_cost(INSN_COST);
 8149   format %{ "clzw  $dst, $src" %}
 8150   ins_encode %{
 8151     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8152   %}
 8153 
 8154   ins_pipe(ialu_reg);
 8155 %}
 8156 
 8157 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8158   match(Set dst (CountLeadingZerosL src));
 8159 
 8160   ins_cost(INSN_COST);
 8161   format %{ "clz   $dst, $src" %}
 8162   ins_encode %{
 8163     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8164   %}
 8165 
 8166   ins_pipe(ialu_reg);
 8167 %}
 8168 
 8169 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8170   match(Set dst (CountTrailingZerosI src));
 8171 
 8172   ins_cost(INSN_COST * 2);
 8173   format %{ "rbitw  $dst, $src\n\t"
 8174             "clzw   $dst, $dst" %}
 8175   ins_encode %{
 8176     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8177     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8178   %}
 8179 
 8180   ins_pipe(ialu_reg);
 8181 %}
 8182 
 8183 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8184   match(Set dst (CountTrailingZerosL src));
 8185 
 8186   ins_cost(INSN_COST * 2);
 8187   format %{ "rbit   $dst, $src\n\t"
 8188             "clz    $dst, $dst" %}
 8189   ins_encode %{
 8190     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8191     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8192   %}
 8193 
 8194   ins_pipe(ialu_reg);
 8195 %}
 8196 
 8197 //---------- Population Count Instructions -------------------------------------
 8198 //
 8199 
 8200 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8201   match(Set dst (PopCountI src));
 8202   effect(TEMP tmp);
 8203   ins_cost(INSN_COST * 13);
 8204 
 8205   format %{ "movw   $src, $src\n\t"
 8206             "mov    $tmp, $src\t# vector (1D)\n\t"
 8207             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8208             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8209             "mov    $dst, $tmp\t# vector (1D)" %}
 8210   ins_encode %{
 8211     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8212     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8213     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8214     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8215     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8216   %}
 8217 
 8218   ins_pipe(pipe_class_default);
 8219 %}
 8220 
 8221 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8222   match(Set dst (PopCountI (LoadI mem)));
 8223   effect(TEMP tmp);
 8224   ins_cost(INSN_COST * 13);
 8225 
 8226   format %{ "ldrs   $tmp, $mem\n\t"
 8227             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8228             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8229             "mov    $dst, $tmp\t# vector (1D)" %}
 8230   ins_encode %{
 8231     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8232     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8233               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8234     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8235     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8236     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8237   %}
 8238 
 8239   ins_pipe(pipe_class_default);
 8240 %}
 8241 
 8242 // Note: Long.bitCount(long) returns an int.
 8243 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8244   match(Set dst (PopCountL src));
 8245   effect(TEMP tmp);
 8246   ins_cost(INSN_COST * 13);
 8247 
 8248   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8249             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8250             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8251             "mov    $dst, $tmp\t# vector (1D)" %}
 8252   ins_encode %{
 8253     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8254     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8255     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8256     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8257   %}
 8258 
 8259   ins_pipe(pipe_class_default);
 8260 %}
 8261 
 8262 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8263   match(Set dst (PopCountL (LoadL mem)));
 8264   effect(TEMP tmp);
 8265   ins_cost(INSN_COST * 13);
 8266 
 8267   format %{ "ldrd   $tmp, $mem\n\t"
 8268             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8269             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8270             "mov    $dst, $tmp\t# vector (1D)" %}
 8271   ins_encode %{
 8272     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8273     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8274               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8275     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8276     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8277     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8278   %}
 8279 
 8280   ins_pipe(pipe_class_default);
 8281 %}
 8282 
 8283 // ============================================================================
 8284 // MemBar Instruction
 8285 
 8286 instruct load_fence() %{
 8287   match(LoadFence);
 8288   ins_cost(VOLATILE_REF_COST);
 8289 
 8290   format %{ "load_fence" %}
 8291 
 8292   ins_encode %{
 8293     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8294   %}
 8295   ins_pipe(pipe_serial);
 8296 %}
 8297 
 8298 instruct unnecessary_membar_acquire() %{
 8299   predicate(unnecessary_acquire(n));
 8300   match(MemBarAcquire);
 8301   ins_cost(0);
 8302 
 8303   format %{ "membar_acquire (elided)" %}
 8304 
 8305   ins_encode %{
 8306     __ block_comment("membar_acquire (elided)");
 8307   %}
 8308 
 8309   ins_pipe(pipe_class_empty);
 8310 %}
 8311 
 8312 instruct membar_acquire() %{
 8313   match(MemBarAcquire);
 8314   ins_cost(VOLATILE_REF_COST);
 8315 
 8316   format %{ "membar_acquire\n\t"
 8317             "dmb ish" %}
 8318 
 8319   ins_encode %{
 8320     __ block_comment("membar_acquire");
 8321     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8322   %}
 8323 
 8324   ins_pipe(pipe_serial);
 8325 %}
 8326 
 8327 
 8328 instruct membar_acquire_lock() %{
 8329   match(MemBarAcquireLock);
 8330   ins_cost(VOLATILE_REF_COST);
 8331 
 8332   format %{ "membar_acquire_lock (elided)" %}
 8333 
 8334   ins_encode %{
 8335     __ block_comment("membar_acquire_lock (elided)");
 8336   %}
 8337 
 8338   ins_pipe(pipe_serial);
 8339 %}
 8340 
 8341 instruct store_fence() %{
 8342   match(StoreFence);
 8343   ins_cost(VOLATILE_REF_COST);
 8344 
 8345   format %{ "store_fence" %}
 8346 
 8347   ins_encode %{
 8348     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8349   %}
 8350   ins_pipe(pipe_serial);
 8351 %}
 8352 
 8353 instruct unnecessary_membar_release() %{
 8354   predicate(unnecessary_release(n));
 8355   match(MemBarRelease);
 8356   ins_cost(0);
 8357 
 8358   format %{ "membar_release (elided)" %}
 8359 
 8360   ins_encode %{
 8361     __ block_comment("membar_release (elided)");
 8362   %}
 8363   ins_pipe(pipe_serial);
 8364 %}
 8365 
 8366 instruct membar_release() %{
 8367   match(MemBarRelease);
 8368   ins_cost(VOLATILE_REF_COST);
 8369 
 8370   format %{ "membar_release\n\t"
 8371             "dmb ish" %}
 8372 
 8373   ins_encode %{
 8374     __ block_comment("membar_release");
 8375     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8376   %}
 8377   ins_pipe(pipe_serial);
 8378 %}
 8379 
 8380 instruct membar_storestore() %{
 8381   match(MemBarStoreStore);
 8382   match(StoreStoreFence);
 8383   ins_cost(VOLATILE_REF_COST);
 8384 
 8385   format %{ "MEMBAR-store-store" %}
 8386 
 8387   ins_encode %{
 8388     __ membar(Assembler::StoreStore);
 8389   %}
 8390   ins_pipe(pipe_serial);
 8391 %}
 8392 
 8393 instruct membar_release_lock() %{
 8394   match(MemBarReleaseLock);
 8395   ins_cost(VOLATILE_REF_COST);
 8396 
 8397   format %{ "membar_release_lock (elided)" %}
 8398 
 8399   ins_encode %{
 8400     __ block_comment("membar_release_lock (elided)");
 8401   %}
 8402 
 8403   ins_pipe(pipe_serial);
 8404 %}
 8405 
 8406 instruct unnecessary_membar_volatile() %{
 8407   predicate(unnecessary_volatile(n));
 8408   match(MemBarVolatile);
 8409   ins_cost(0);
 8410 
 8411   format %{ "membar_volatile (elided)" %}
 8412 
 8413   ins_encode %{
 8414     __ block_comment("membar_volatile (elided)");
 8415   %}
 8416 
 8417   ins_pipe(pipe_serial);
 8418 %}
 8419 
 8420 instruct membar_volatile() %{
 8421   match(MemBarVolatile);
 8422   ins_cost(VOLATILE_REF_COST*100);
 8423 
 8424   format %{ "membar_volatile\n\t"
 8425              "dmb ish"%}
 8426 
 8427   ins_encode %{
 8428     __ block_comment("membar_volatile");
 8429     __ membar(Assembler::StoreLoad);
 8430   %}
 8431 
 8432   ins_pipe(pipe_serial);
 8433 %}
 8434 
 8435 // ============================================================================
 8436 // Cast/Convert Instructions
 8437 
 8438 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8439   match(Set dst (CastX2P src));
 8440 
 8441   ins_cost(INSN_COST);
 8442   format %{ "mov $dst, $src\t# long -> ptr" %}
 8443 
 8444   ins_encode %{
 8445     if ($dst$$reg != $src$$reg) {
 8446       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8447     }
 8448   %}
 8449 
 8450   ins_pipe(ialu_reg);
 8451 %}
 8452 
 8453 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8454   match(Set dst (CastP2X src));
 8455 
 8456   ins_cost(INSN_COST);
 8457   format %{ "mov $dst, $src\t# ptr -> long" %}
 8458 
 8459   ins_encode %{
 8460     if ($dst$$reg != $src$$reg) {
 8461       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8462     }
 8463   %}
 8464 
 8465   ins_pipe(ialu_reg);
 8466 %}
 8467 
 8468 // Convert oop into int for vectors alignment masking
 8469 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8470   match(Set dst (ConvL2I (CastP2X src)));
 8471 
 8472   ins_cost(INSN_COST);
 8473   format %{ "movw $dst, $src\t# ptr -> int" %}
 8474   ins_encode %{
 8475     __ movw($dst$$Register, $src$$Register);
 8476   %}
 8477 
 8478   ins_pipe(ialu_reg);
 8479 %}
 8480 
 8481 // Convert compressed oop into int for vectors alignment masking
 8482 // in case of 32bit oops (heap < 4Gb).
 8483 instruct convN2I(iRegINoSp dst, iRegN src)
 8484 %{
 8485   predicate(CompressedOops::shift() == 0);
 8486   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8487 
 8488   ins_cost(INSN_COST);
 8489   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8490   ins_encode %{
 8491     __ movw($dst$$Register, $src$$Register);
 8492   %}
 8493 
 8494   ins_pipe(ialu_reg);
 8495 %}
 8496 
 8497 
 8498 // Convert oop pointer into compressed form
 8499 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8500   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8501   match(Set dst (EncodeP src));
 8502   effect(KILL cr);
 8503   ins_cost(INSN_COST * 3);
 8504   format %{ "encode_heap_oop $dst, $src" %}
 8505   ins_encode %{
 8506     Register s = $src$$Register;
 8507     Register d = $dst$$Register;
 8508     __ encode_heap_oop(d, s);
 8509   %}
 8510   ins_pipe(ialu_reg);
 8511 %}
 8512 
 8513 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8514   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8515   match(Set dst (EncodeP src));
 8516   ins_cost(INSN_COST * 3);
 8517   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8518   ins_encode %{
 8519     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8520   %}
 8521   ins_pipe(ialu_reg);
 8522 %}
 8523 
 8524 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8525   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8526             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8527   match(Set dst (DecodeN src));
 8528   ins_cost(INSN_COST * 3);
 8529   format %{ "decode_heap_oop $dst, $src" %}
 8530   ins_encode %{
 8531     Register s = $src$$Register;
 8532     Register d = $dst$$Register;
 8533     __ decode_heap_oop(d, s);
 8534   %}
 8535   ins_pipe(ialu_reg);
 8536 %}
 8537 
 8538 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8539   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8540             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8541   match(Set dst (DecodeN src));
 8542   ins_cost(INSN_COST * 3);
 8543   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8544   ins_encode %{
 8545     Register s = $src$$Register;
 8546     Register d = $dst$$Register;
 8547     __ decode_heap_oop_not_null(d, s);
 8548   %}
 8549   ins_pipe(ialu_reg);
 8550 %}
 8551 
 8552 // n.b. AArch64 implementations of encode_klass_not_null and
 8553 // decode_klass_not_null do not modify the flags register so, unlike
 8554 // Intel, we don't kill CR as a side effect here
 8555 
 8556 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8557   match(Set dst (EncodePKlass src));
 8558 
 8559   ins_cost(INSN_COST * 3);
 8560   format %{ "encode_klass_not_null $dst,$src" %}
 8561 
 8562   ins_encode %{
 8563     Register src_reg = as_Register($src$$reg);
 8564     Register dst_reg = as_Register($dst$$reg);
 8565     __ encode_klass_not_null(dst_reg, src_reg);
 8566   %}
 8567 
 8568    ins_pipe(ialu_reg);
 8569 %}
 8570 
 8571 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8572   match(Set dst (DecodeNKlass src));
 8573 
 8574   ins_cost(INSN_COST * 3);
 8575   format %{ "decode_klass_not_null $dst,$src" %}
 8576 
 8577   ins_encode %{
 8578     Register src_reg = as_Register($src$$reg);
 8579     Register dst_reg = as_Register($dst$$reg);
 8580     if (dst_reg != src_reg) {
 8581       __ decode_klass_not_null(dst_reg, src_reg);
 8582     } else {
 8583       __ decode_klass_not_null(dst_reg);
 8584     }
 8585   %}
 8586 
 8587    ins_pipe(ialu_reg);
 8588 %}
 8589 
 8590 instruct checkCastPP(iRegPNoSp dst)
 8591 %{
 8592   match(Set dst (CheckCastPP dst));
 8593 
 8594   size(0);
 8595   format %{ "# checkcastPP of $dst" %}
 8596   ins_encode(/* empty encoding */);
 8597   ins_pipe(pipe_class_empty);
 8598 %}
 8599 
 8600 instruct castPP(iRegPNoSp dst)
 8601 %{
 8602   match(Set dst (CastPP dst));
 8603 
 8604   size(0);
 8605   format %{ "# castPP of $dst" %}
 8606   ins_encode(/* empty encoding */);
 8607   ins_pipe(pipe_class_empty);
 8608 %}
 8609 
 8610 instruct castII(iRegI dst)
 8611 %{
 8612   match(Set dst (CastII dst));
 8613 
 8614   size(0);
 8615   format %{ "# castII of $dst" %}
 8616   ins_encode(/* empty encoding */);
 8617   ins_cost(0);
 8618   ins_pipe(pipe_class_empty);
 8619 %}
 8620 
 8621 instruct castLL(iRegL dst)
 8622 %{
 8623   match(Set dst (CastLL dst));
 8624 
 8625   size(0);
 8626   format %{ "# castLL of $dst" %}
 8627   ins_encode(/* empty encoding */);
 8628   ins_cost(0);
 8629   ins_pipe(pipe_class_empty);
 8630 %}
 8631 
 8632 instruct castFF(vRegF dst)
 8633 %{
 8634   match(Set dst (CastFF dst));
 8635 
 8636   size(0);
 8637   format %{ "# castFF of $dst" %}
 8638   ins_encode(/* empty encoding */);
 8639   ins_cost(0);
 8640   ins_pipe(pipe_class_empty);
 8641 %}
 8642 
 8643 instruct castDD(vRegD dst)
 8644 %{
 8645   match(Set dst (CastDD dst));
 8646 
 8647   size(0);
 8648   format %{ "# castDD of $dst" %}
 8649   ins_encode(/* empty encoding */);
 8650   ins_cost(0);
 8651   ins_pipe(pipe_class_empty);
 8652 %}
 8653 
 8654 instruct castVV(vReg dst)
 8655 %{
 8656   match(Set dst (CastVV dst));
 8657 
 8658   size(0);
 8659   format %{ "# castVV of $dst" %}
 8660   ins_encode(/* empty encoding */);
 8661   ins_cost(0);
 8662   ins_pipe(pipe_class_empty);
 8663 %}
 8664 
 8665 instruct castVVMask(pRegGov dst)
 8666 %{
 8667   match(Set dst (CastVV dst));
 8668 
 8669   size(0);
 8670   format %{ "# castVV of $dst" %}
 8671   ins_encode(/* empty encoding */);
 8672   ins_cost(0);
 8673   ins_pipe(pipe_class_empty);
 8674 %}
 8675 
 8676 // ============================================================================
 8677 // Atomic operation instructions
 8678 //
 8679 
 8680 // standard CompareAndSwapX when we are using barriers
 8681 // these have higher priority than the rules selected by a predicate
 8682 
 8683 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 8684 // can't match them
 8685 
 8686 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8687 
 8688   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8689   ins_cost(2 * VOLATILE_REF_COST);
 8690 
 8691   effect(KILL cr);
 8692 
 8693   format %{
 8694     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8695     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8696   %}
 8697 
 8698   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 8699             aarch64_enc_cset_eq(res));
 8700 
 8701   ins_pipe(pipe_slow);
 8702 %}
 8703 
 8704 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8705 
 8706   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8707   ins_cost(2 * VOLATILE_REF_COST);
 8708 
 8709   effect(KILL cr);
 8710 
 8711   format %{
 8712     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8713     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8714   %}
 8715 
 8716   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 8717             aarch64_enc_cset_eq(res));
 8718 
 8719   ins_pipe(pipe_slow);
 8720 %}
 8721 
 8722 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8723 
 8724   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8725   ins_cost(2 * VOLATILE_REF_COST);
 8726 
 8727   effect(KILL cr);
 8728 
 8729  format %{
 8730     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8731     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8732  %}
 8733 
 8734  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8735             aarch64_enc_cset_eq(res));
 8736 
 8737   ins_pipe(pipe_slow);
 8738 %}
 8739 
 8740 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8741 
 8742   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8743   ins_cost(2 * VOLATILE_REF_COST);
 8744 
 8745   effect(KILL cr);
 8746 
 8747  format %{
 8748     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8749     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8750  %}
 8751 
 8752  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8753             aarch64_enc_cset_eq(res));
 8754 
 8755   ins_pipe(pipe_slow);
 8756 %}
 8757 
 8758 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8759 
 8760   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8761   predicate(n->as_LoadStore()->barrier_data() == 0);
 8762   ins_cost(2 * VOLATILE_REF_COST);
 8763 
 8764   effect(KILL cr);
 8765 
 8766  format %{
 8767     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8768     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8769  %}
 8770 
 8771  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8772             aarch64_enc_cset_eq(res));
 8773 
 8774   ins_pipe(pipe_slow);
 8775 %}
 8776 
 8777 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8778 
 8779   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8780   ins_cost(2 * VOLATILE_REF_COST);
 8781 
 8782   effect(KILL cr);
 8783 
 8784  format %{
 8785     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8786     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8787  %}
 8788 
 8789  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8790             aarch64_enc_cset_eq(res));
 8791 
 8792   ins_pipe(pipe_slow);
 8793 %}
 8794 
 8795 // alternative CompareAndSwapX when we are eliding barriers
 8796 
 8797 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8798 
 8799   predicate(needs_acquiring_load_exclusive(n));
 8800   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8801   ins_cost(VOLATILE_REF_COST);
 8802 
 8803   effect(KILL cr);
 8804 
 8805   format %{
 8806     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8807     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8808   %}
 8809 
 8810   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 8811             aarch64_enc_cset_eq(res));
 8812 
 8813   ins_pipe(pipe_slow);
 8814 %}
 8815 
 8816 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8817 
 8818   predicate(needs_acquiring_load_exclusive(n));
 8819   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8820   ins_cost(VOLATILE_REF_COST);
 8821 
 8822   effect(KILL cr);
 8823 
 8824   format %{
 8825     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8826     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8827   %}
 8828 
 8829   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 8830             aarch64_enc_cset_eq(res));
 8831 
 8832   ins_pipe(pipe_slow);
 8833 %}
 8834 
 8835 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8836 
 8837   predicate(needs_acquiring_load_exclusive(n));
 8838   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8839   ins_cost(VOLATILE_REF_COST);
 8840 
 8841   effect(KILL cr);
 8842 
 8843  format %{
 8844     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8845     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8846  %}
 8847 
 8848  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8849             aarch64_enc_cset_eq(res));
 8850 
 8851   ins_pipe(pipe_slow);
 8852 %}
 8853 
 8854 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8855 
 8856   predicate(needs_acquiring_load_exclusive(n));
 8857   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8858   ins_cost(VOLATILE_REF_COST);
 8859 
 8860   effect(KILL cr);
 8861 
 8862  format %{
 8863     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8864     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8865  %}
 8866 
 8867  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8868             aarch64_enc_cset_eq(res));
 8869 
 8870   ins_pipe(pipe_slow);
 8871 %}
 8872 
 8873 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8874 
 8875   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8876   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8877   ins_cost(VOLATILE_REF_COST);
 8878 
 8879   effect(KILL cr);
 8880 
 8881  format %{
 8882     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8883     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8884  %}
 8885 
 8886  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8887             aarch64_enc_cset_eq(res));
 8888 
 8889   ins_pipe(pipe_slow);
 8890 %}
 8891 
 8892 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8893 
 8894   predicate(needs_acquiring_load_exclusive(n));
 8895   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8896   ins_cost(VOLATILE_REF_COST);
 8897 
 8898   effect(KILL cr);
 8899 
 8900  format %{
 8901     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8902     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8903  %}
 8904 
 8905  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8906             aarch64_enc_cset_eq(res));
 8907 
 8908   ins_pipe(pipe_slow);
 8909 %}
 8910 
 8911 
 8912 // ---------------------------------------------------------------------
 8913 
 8914 // BEGIN This section of the file is automatically generated. Do not edit --------------
 8915 
 8916 // Sundry CAS operations.  Note that release is always true,
 8917 // regardless of the memory ordering of the CAS.  This is because we
 8918 // need the volatile case to be sequentially consistent but there is
 8919 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 8920 // can't check the type of memory ordering here, so we always emit a
 8921 // STLXR.
 8922 
 8923 // This section is generated from aarch64_ad_cas.m4
 8924 
 8925 
 8926 
 8927 // This pattern is generated automatically from cas.m4.
 8928 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8929 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8930 
 8931   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8932   ins_cost(2 * VOLATILE_REF_COST);
 8933   effect(TEMP_DEF res, KILL cr);
 8934   format %{
 8935     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8936   %}
 8937   ins_encode %{
 8938     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8939                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8940                /*weak*/ false, $res$$Register);
 8941     __ sxtbw($res$$Register, $res$$Register);
 8942   %}
 8943   ins_pipe(pipe_slow);
 8944 %}
 8945 
 8946 // This pattern is generated automatically from cas.m4.
 8947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8948 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8949 
 8950   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 8951   ins_cost(2 * VOLATILE_REF_COST);
 8952   effect(TEMP_DEF res, KILL cr);
 8953   format %{
 8954     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8955   %}
 8956   ins_encode %{
 8957     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8958                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 8959                /*weak*/ false, $res$$Register);
 8960     __ sxthw($res$$Register, $res$$Register);
 8961   %}
 8962   ins_pipe(pipe_slow);
 8963 %}
 8964 
 8965 // This pattern is generated automatically from cas.m4.
 8966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8967 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8968 
 8969   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 8970   ins_cost(2 * VOLATILE_REF_COST);
 8971   effect(TEMP_DEF res, KILL cr);
 8972   format %{
 8973     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8974   %}
 8975   ins_encode %{
 8976     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8977                Assembler::word, /*acquire*/ false, /*release*/ true,
 8978                /*weak*/ false, $res$$Register);
 8979   %}
 8980   ins_pipe(pipe_slow);
 8981 %}
 8982 
 8983 // This pattern is generated automatically from cas.m4.
 8984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8985 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8986 
 8987   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 8988   ins_cost(2 * VOLATILE_REF_COST);
 8989   effect(TEMP_DEF res, KILL cr);
 8990   format %{
 8991     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8992   %}
 8993   ins_encode %{
 8994     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8995                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8996                /*weak*/ false, $res$$Register);
 8997   %}
 8998   ins_pipe(pipe_slow);
 8999 %}
 9000 
 9001 // This pattern is generated automatically from cas.m4.
 9002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9003 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9004 
 9005   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9006   ins_cost(2 * VOLATILE_REF_COST);
 9007   effect(TEMP_DEF res, KILL cr);
 9008   format %{
 9009     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9010   %}
 9011   ins_encode %{
 9012     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9013                Assembler::word, /*acquire*/ false, /*release*/ true,
 9014                /*weak*/ false, $res$$Register);
 9015   %}
 9016   ins_pipe(pipe_slow);
 9017 %}
 9018 
 9019 // This pattern is generated automatically from cas.m4.
 9020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9021 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9022   predicate(n->as_LoadStore()->barrier_data() == 0);
 9023   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9024   ins_cost(2 * VOLATILE_REF_COST);
 9025   effect(TEMP_DEF res, KILL cr);
 9026   format %{
 9027     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9028   %}
 9029   ins_encode %{
 9030     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9031                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9032                /*weak*/ false, $res$$Register);
 9033   %}
 9034   ins_pipe(pipe_slow);
 9035 %}
 9036 
 9037 // This pattern is generated automatically from cas.m4.
 9038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9039 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9040   predicate(needs_acquiring_load_exclusive(n));
 9041   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9042   ins_cost(VOLATILE_REF_COST);
 9043   effect(TEMP_DEF res, KILL cr);
 9044   format %{
 9045     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9046   %}
 9047   ins_encode %{
 9048     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9049                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9050                /*weak*/ false, $res$$Register);
 9051     __ sxtbw($res$$Register, $res$$Register);
 9052   %}
 9053   ins_pipe(pipe_slow);
 9054 %}
 9055 
 9056 // This pattern is generated automatically from cas.m4.
 9057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9058 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9059   predicate(needs_acquiring_load_exclusive(n));
 9060   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9061   ins_cost(VOLATILE_REF_COST);
 9062   effect(TEMP_DEF res, KILL cr);
 9063   format %{
 9064     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9065   %}
 9066   ins_encode %{
 9067     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9068                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9069                /*weak*/ false, $res$$Register);
 9070     __ sxthw($res$$Register, $res$$Register);
 9071   %}
 9072   ins_pipe(pipe_slow);
 9073 %}
 9074 
 9075 // This pattern is generated automatically from cas.m4.
 9076 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9077 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9078   predicate(needs_acquiring_load_exclusive(n));
 9079   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9080   ins_cost(VOLATILE_REF_COST);
 9081   effect(TEMP_DEF res, KILL cr);
 9082   format %{
 9083     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9084   %}
 9085   ins_encode %{
 9086     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9087                Assembler::word, /*acquire*/ true, /*release*/ true,
 9088                /*weak*/ false, $res$$Register);
 9089   %}
 9090   ins_pipe(pipe_slow);
 9091 %}
 9092 
 9093 // This pattern is generated automatically from cas.m4.
 9094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9095 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9096   predicate(needs_acquiring_load_exclusive(n));
 9097   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9098   ins_cost(VOLATILE_REF_COST);
 9099   effect(TEMP_DEF res, KILL cr);
 9100   format %{
 9101     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9102   %}
 9103   ins_encode %{
 9104     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9105                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9106                /*weak*/ false, $res$$Register);
 9107   %}
 9108   ins_pipe(pipe_slow);
 9109 %}
 9110 
 9111 // This pattern is generated automatically from cas.m4.
 9112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9113 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9114   predicate(needs_acquiring_load_exclusive(n));
 9115   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9116   ins_cost(VOLATILE_REF_COST);
 9117   effect(TEMP_DEF res, KILL cr);
 9118   format %{
 9119     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9120   %}
 9121   ins_encode %{
 9122     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9123                Assembler::word, /*acquire*/ true, /*release*/ true,
 9124                /*weak*/ false, $res$$Register);
 9125   %}
 9126   ins_pipe(pipe_slow);
 9127 %}
 9128 
 9129 // This pattern is generated automatically from cas.m4.
 9130 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9131 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9132   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9133   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9134   ins_cost(VOLATILE_REF_COST);
 9135   effect(TEMP_DEF res, KILL cr);
 9136   format %{
 9137     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9138   %}
 9139   ins_encode %{
 9140     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9141                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9142                /*weak*/ false, $res$$Register);
 9143   %}
 9144   ins_pipe(pipe_slow);
 9145 %}
 9146 
 9147 // This pattern is generated automatically from cas.m4.
 9148 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9149 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9150 
 9151   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9152   ins_cost(2 * VOLATILE_REF_COST);
 9153   effect(KILL cr);
 9154   format %{
 9155     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9156     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9157   %}
 9158   ins_encode %{
 9159     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9160                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9161                /*weak*/ true, noreg);
 9162     __ csetw($res$$Register, Assembler::EQ);
 9163   %}
 9164   ins_pipe(pipe_slow);
 9165 %}
 9166 
 9167 // This pattern is generated automatically from cas.m4.
 9168 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9169 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9170 
 9171   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9172   ins_cost(2 * VOLATILE_REF_COST);
 9173   effect(KILL cr);
 9174   format %{
 9175     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9176     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9177   %}
 9178   ins_encode %{
 9179     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9180                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9181                /*weak*/ true, noreg);
 9182     __ csetw($res$$Register, Assembler::EQ);
 9183   %}
 9184   ins_pipe(pipe_slow);
 9185 %}
 9186 
 9187 // This pattern is generated automatically from cas.m4.
 9188 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9189 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9190 
 9191   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9192   ins_cost(2 * VOLATILE_REF_COST);
 9193   effect(KILL cr);
 9194   format %{
 9195     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9196     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9197   %}
 9198   ins_encode %{
 9199     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9200                Assembler::word, /*acquire*/ false, /*release*/ true,
 9201                /*weak*/ true, noreg);
 9202     __ csetw($res$$Register, Assembler::EQ);
 9203   %}
 9204   ins_pipe(pipe_slow);
 9205 %}
 9206 
 9207 // This pattern is generated automatically from cas.m4.
 9208 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9209 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9210 
 9211   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9212   ins_cost(2 * VOLATILE_REF_COST);
 9213   effect(KILL cr);
 9214   format %{
 9215     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9216     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9217   %}
 9218   ins_encode %{
 9219     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9220                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9221                /*weak*/ true, noreg);
 9222     __ csetw($res$$Register, Assembler::EQ);
 9223   %}
 9224   ins_pipe(pipe_slow);
 9225 %}
 9226 
 9227 // This pattern is generated automatically from cas.m4.
 9228 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9229 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9230 
 9231   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9232   ins_cost(2 * VOLATILE_REF_COST);
 9233   effect(KILL cr);
 9234   format %{
 9235     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9236     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9237   %}
 9238   ins_encode %{
 9239     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9240                Assembler::word, /*acquire*/ false, /*release*/ true,
 9241                /*weak*/ true, noreg);
 9242     __ csetw($res$$Register, Assembler::EQ);
 9243   %}
 9244   ins_pipe(pipe_slow);
 9245 %}
 9246 
 9247 // This pattern is generated automatically from cas.m4.
 9248 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9249 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9250   predicate(n->as_LoadStore()->barrier_data() == 0);
 9251   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9252   ins_cost(2 * VOLATILE_REF_COST);
 9253   effect(KILL cr);
 9254   format %{
 9255     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9256     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9257   %}
 9258   ins_encode %{
 9259     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9260                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9261                /*weak*/ true, noreg);
 9262     __ csetw($res$$Register, Assembler::EQ);
 9263   %}
 9264   ins_pipe(pipe_slow);
 9265 %}
 9266 
 9267 // This pattern is generated automatically from cas.m4.
 9268 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9269 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9270   predicate(needs_acquiring_load_exclusive(n));
 9271   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9272   ins_cost(VOLATILE_REF_COST);
 9273   effect(KILL cr);
 9274   format %{
 9275     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9276     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9277   %}
 9278   ins_encode %{
 9279     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9280                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9281                /*weak*/ true, noreg);
 9282     __ csetw($res$$Register, Assembler::EQ);
 9283   %}
 9284   ins_pipe(pipe_slow);
 9285 %}
 9286 
 9287 // This pattern is generated automatically from cas.m4.
 9288 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9289 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9290   predicate(needs_acquiring_load_exclusive(n));
 9291   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9292   ins_cost(VOLATILE_REF_COST);
 9293   effect(KILL cr);
 9294   format %{
 9295     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9296     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9297   %}
 9298   ins_encode %{
 9299     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9300                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9301                /*weak*/ true, noreg);
 9302     __ csetw($res$$Register, Assembler::EQ);
 9303   %}
 9304   ins_pipe(pipe_slow);
 9305 %}
 9306 
 9307 // This pattern is generated automatically from cas.m4.
 9308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9309 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9310   predicate(needs_acquiring_load_exclusive(n));
 9311   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9312   ins_cost(VOLATILE_REF_COST);
 9313   effect(KILL cr);
 9314   format %{
 9315     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9316     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9317   %}
 9318   ins_encode %{
 9319     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9320                Assembler::word, /*acquire*/ true, /*release*/ true,
 9321                /*weak*/ true, noreg);
 9322     __ csetw($res$$Register, Assembler::EQ);
 9323   %}
 9324   ins_pipe(pipe_slow);
 9325 %}
 9326 
 9327 // This pattern is generated automatically from cas.m4.
 9328 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9329 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9330   predicate(needs_acquiring_load_exclusive(n));
 9331   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9332   ins_cost(VOLATILE_REF_COST);
 9333   effect(KILL cr);
 9334   format %{
 9335     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9336     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9337   %}
 9338   ins_encode %{
 9339     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9340                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9341                /*weak*/ true, noreg);
 9342     __ csetw($res$$Register, Assembler::EQ);
 9343   %}
 9344   ins_pipe(pipe_slow);
 9345 %}
 9346 
 9347 // This pattern is generated automatically from cas.m4.
 9348 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9349 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9350   predicate(needs_acquiring_load_exclusive(n));
 9351   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9352   ins_cost(VOLATILE_REF_COST);
 9353   effect(KILL cr);
 9354   format %{
 9355     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9356     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9357   %}
 9358   ins_encode %{
 9359     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9360                Assembler::word, /*acquire*/ true, /*release*/ true,
 9361                /*weak*/ true, noreg);
 9362     __ csetw($res$$Register, Assembler::EQ);
 9363   %}
 9364   ins_pipe(pipe_slow);
 9365 %}
 9366 
 9367 // This pattern is generated automatically from cas.m4.
 9368 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9369 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9370   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9371   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9372   ins_cost(VOLATILE_REF_COST);
 9373   effect(KILL cr);
 9374   format %{
 9375     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9376     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9377   %}
 9378   ins_encode %{
 9379     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9380                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9381                /*weak*/ true, noreg);
 9382     __ csetw($res$$Register, Assembler::EQ);
 9383   %}
 9384   ins_pipe(pipe_slow);
 9385 %}
 9386 
 9387 // END This section of the file is automatically generated. Do not edit --------------
 9388 // ---------------------------------------------------------------------
 9389 
 9390 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9391   match(Set prev (GetAndSetI mem newv));
 9392   ins_cost(2 * VOLATILE_REF_COST);
 9393   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9394   ins_encode %{
 9395     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9396   %}
 9397   ins_pipe(pipe_serial);
 9398 %}
 9399 
 9400 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9401   match(Set prev (GetAndSetL mem newv));
 9402   ins_cost(2 * VOLATILE_REF_COST);
 9403   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9404   ins_encode %{
 9405     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9406   %}
 9407   ins_pipe(pipe_serial);
 9408 %}
 9409 
 9410 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9411   match(Set prev (GetAndSetN mem newv));
 9412   ins_cost(2 * VOLATILE_REF_COST);
 9413   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9414   ins_encode %{
 9415     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9416   %}
 9417   ins_pipe(pipe_serial);
 9418 %}
 9419 
 9420 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9421   predicate(n->as_LoadStore()->barrier_data() == 0);
 9422   match(Set prev (GetAndSetP mem newv));
 9423   ins_cost(2 * VOLATILE_REF_COST);
 9424   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9425   ins_encode %{
 9426     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9427   %}
 9428   ins_pipe(pipe_serial);
 9429 %}
 9430 
 9431 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9432   predicate(needs_acquiring_load_exclusive(n));
 9433   match(Set prev (GetAndSetI mem newv));
 9434   ins_cost(VOLATILE_REF_COST);
 9435   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9436   ins_encode %{
 9437     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9438   %}
 9439   ins_pipe(pipe_serial);
 9440 %}
 9441 
 9442 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9443   predicate(needs_acquiring_load_exclusive(n));
 9444   match(Set prev (GetAndSetL mem newv));
 9445   ins_cost(VOLATILE_REF_COST);
 9446   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9447   ins_encode %{
 9448     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9449   %}
 9450   ins_pipe(pipe_serial);
 9451 %}
 9452 
 9453 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9454   predicate(needs_acquiring_load_exclusive(n));
 9455   match(Set prev (GetAndSetN mem newv));
 9456   ins_cost(VOLATILE_REF_COST);
 9457   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9458   ins_encode %{
 9459     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9460   %}
 9461   ins_pipe(pipe_serial);
 9462 %}
 9463 
 9464 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9465   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9466   match(Set prev (GetAndSetP mem newv));
 9467   ins_cost(VOLATILE_REF_COST);
 9468   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9469   ins_encode %{
 9470     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9471   %}
 9472   ins_pipe(pipe_serial);
 9473 %}
 9474 
 9475 
 9476 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9477   match(Set newval (GetAndAddL mem incr));
 9478   ins_cost(2 * VOLATILE_REF_COST + 1);
 9479   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9480   ins_encode %{
 9481     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9482   %}
 9483   ins_pipe(pipe_serial);
 9484 %}
 9485 
 9486 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9487   predicate(n->as_LoadStore()->result_not_used());
 9488   match(Set dummy (GetAndAddL mem incr));
 9489   ins_cost(2 * VOLATILE_REF_COST);
 9490   format %{ "get_and_addL [$mem], $incr" %}
 9491   ins_encode %{
 9492     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9493   %}
 9494   ins_pipe(pipe_serial);
 9495 %}
 9496 
 9497 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9498   match(Set newval (GetAndAddL mem incr));
 9499   ins_cost(2 * VOLATILE_REF_COST + 1);
 9500   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9501   ins_encode %{
 9502     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9503   %}
 9504   ins_pipe(pipe_serial);
 9505 %}
 9506 
 9507 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9508   predicate(n->as_LoadStore()->result_not_used());
 9509   match(Set dummy (GetAndAddL mem incr));
 9510   ins_cost(2 * VOLATILE_REF_COST);
 9511   format %{ "get_and_addL [$mem], $incr" %}
 9512   ins_encode %{
 9513     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9514   %}
 9515   ins_pipe(pipe_serial);
 9516 %}
 9517 
 9518 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9519   match(Set newval (GetAndAddI mem incr));
 9520   ins_cost(2 * VOLATILE_REF_COST + 1);
 9521   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9522   ins_encode %{
 9523     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9524   %}
 9525   ins_pipe(pipe_serial);
 9526 %}
 9527 
 9528 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9529   predicate(n->as_LoadStore()->result_not_used());
 9530   match(Set dummy (GetAndAddI mem incr));
 9531   ins_cost(2 * VOLATILE_REF_COST);
 9532   format %{ "get_and_addI [$mem], $incr" %}
 9533   ins_encode %{
 9534     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9535   %}
 9536   ins_pipe(pipe_serial);
 9537 %}
 9538 
 9539 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9540   match(Set newval (GetAndAddI mem incr));
 9541   ins_cost(2 * VOLATILE_REF_COST + 1);
 9542   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9543   ins_encode %{
 9544     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9545   %}
 9546   ins_pipe(pipe_serial);
 9547 %}
 9548 
 9549 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9550   predicate(n->as_LoadStore()->result_not_used());
 9551   match(Set dummy (GetAndAddI mem incr));
 9552   ins_cost(2 * VOLATILE_REF_COST);
 9553   format %{ "get_and_addI [$mem], $incr" %}
 9554   ins_encode %{
 9555     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9556   %}
 9557   ins_pipe(pipe_serial);
 9558 %}
 9559 
 9560 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9561   predicate(needs_acquiring_load_exclusive(n));
 9562   match(Set newval (GetAndAddL mem incr));
 9563   ins_cost(VOLATILE_REF_COST + 1);
 9564   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9565   ins_encode %{
 9566     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9567   %}
 9568   ins_pipe(pipe_serial);
 9569 %}
 9570 
 9571 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9572   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9573   match(Set dummy (GetAndAddL mem incr));
 9574   ins_cost(VOLATILE_REF_COST);
 9575   format %{ "get_and_addL_acq [$mem], $incr" %}
 9576   ins_encode %{
 9577     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9578   %}
 9579   ins_pipe(pipe_serial);
 9580 %}
 9581 
 9582 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9583   predicate(needs_acquiring_load_exclusive(n));
 9584   match(Set newval (GetAndAddL mem incr));
 9585   ins_cost(VOLATILE_REF_COST + 1);
 9586   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9587   ins_encode %{
 9588     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9589   %}
 9590   ins_pipe(pipe_serial);
 9591 %}
 9592 
 9593 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9594   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9595   match(Set dummy (GetAndAddL mem incr));
 9596   ins_cost(VOLATILE_REF_COST);
 9597   format %{ "get_and_addL_acq [$mem], $incr" %}
 9598   ins_encode %{
 9599     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9600   %}
 9601   ins_pipe(pipe_serial);
 9602 %}
 9603 
 9604 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9605   predicate(needs_acquiring_load_exclusive(n));
 9606   match(Set newval (GetAndAddI mem incr));
 9607   ins_cost(VOLATILE_REF_COST + 1);
 9608   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9609   ins_encode %{
 9610     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9611   %}
 9612   ins_pipe(pipe_serial);
 9613 %}
 9614 
 9615 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9616   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9617   match(Set dummy (GetAndAddI mem incr));
 9618   ins_cost(VOLATILE_REF_COST);
 9619   format %{ "get_and_addI_acq [$mem], $incr" %}
 9620   ins_encode %{
 9621     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9622   %}
 9623   ins_pipe(pipe_serial);
 9624 %}
 9625 
 9626 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9627   predicate(needs_acquiring_load_exclusive(n));
 9628   match(Set newval (GetAndAddI mem incr));
 9629   ins_cost(VOLATILE_REF_COST + 1);
 9630   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9631   ins_encode %{
 9632     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9633   %}
 9634   ins_pipe(pipe_serial);
 9635 %}
 9636 
 9637 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9638   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9639   match(Set dummy (GetAndAddI mem incr));
 9640   ins_cost(VOLATILE_REF_COST);
 9641   format %{ "get_and_addI_acq [$mem], $incr" %}
 9642   ins_encode %{
 9643     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9644   %}
 9645   ins_pipe(pipe_serial);
 9646 %}
 9647 
 9648 // Manifest a CmpL result in an integer register.
 9649 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9650 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9651 %{
 9652   match(Set dst (CmpL3 src1 src2));
 9653   effect(KILL flags);
 9654 
 9655   ins_cost(INSN_COST * 6);
 9656   format %{
 9657       "cmp $src1, $src2"
 9658       "csetw $dst, ne"
 9659       "cnegw $dst, lt"
 9660   %}
 9661   // format %{ "CmpL3 $dst, $src1, $src2" %}
 9662   ins_encode %{
 9663     __ cmp($src1$$Register, $src2$$Register);
 9664     __ csetw($dst$$Register, Assembler::NE);
 9665     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9666   %}
 9667 
 9668   ins_pipe(pipe_class_default);
 9669 %}
 9670 
 9671 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9672 %{
 9673   match(Set dst (CmpL3 src1 src2));
 9674   effect(KILL flags);
 9675 
 9676   ins_cost(INSN_COST * 6);
 9677   format %{
 9678       "cmp $src1, $src2"
 9679       "csetw $dst, ne"
 9680       "cnegw $dst, lt"
 9681   %}
 9682   ins_encode %{
 9683     int32_t con = (int32_t)$src2$$constant;
 9684      if (con < 0) {
 9685       __ adds(zr, $src1$$Register, -con);
 9686     } else {
 9687       __ subs(zr, $src1$$Register, con);
 9688     }
 9689     __ csetw($dst$$Register, Assembler::NE);
 9690     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9691   %}
 9692 
 9693   ins_pipe(pipe_class_default);
 9694 %}
 9695 
 9696 // ============================================================================
 9697 // Conditional Move Instructions
 9698 
 9699 // n.b. we have identical rules for both a signed compare op (cmpOp)
 9700 // and an unsigned compare op (cmpOpU). it would be nice if we could
 9701 // define an op class which merged both inputs and use it to type the
 9702 // argument to a single rule. unfortunatelyt his fails because the
 9703 // opclass does not live up to the COND_INTER interface of its
 9704 // component operands. When the generic code tries to negate the
 9705 // operand it ends up running the generci Machoper::negate method
 9706 // which throws a ShouldNotHappen. So, we have to provide two flavours
 9707 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
 9708 
 9709 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9710   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9711 
 9712   ins_cost(INSN_COST * 2);
 9713   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
 9714 
 9715   ins_encode %{
 9716     __ cselw(as_Register($dst$$reg),
 9717              as_Register($src2$$reg),
 9718              as_Register($src1$$reg),
 9719              (Assembler::Condition)$cmp$$cmpcode);
 9720   %}
 9721 
 9722   ins_pipe(icond_reg_reg);
 9723 %}
 9724 
 9725 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9726   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9727 
 9728   ins_cost(INSN_COST * 2);
 9729   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
 9730 
 9731   ins_encode %{
 9732     __ cselw(as_Register($dst$$reg),
 9733              as_Register($src2$$reg),
 9734              as_Register($src1$$reg),
 9735              (Assembler::Condition)$cmp$$cmpcode);
 9736   %}
 9737 
 9738   ins_pipe(icond_reg_reg);
 9739 %}
 9740 
 9741 // special cases where one arg is zero
 9742 
 9743 // n.b. this is selected in preference to the rule above because it
 9744 // avoids loading constant 0 into a source register
 9745 
 9746 // TODO
 9747 // we ought only to be able to cull one of these variants as the ideal
 9748 // transforms ought always to order the zero consistently (to left/right?)
 9749 
 9750 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9751   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9752 
 9753   ins_cost(INSN_COST * 2);
 9754   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
 9755 
 9756   ins_encode %{
 9757     __ cselw(as_Register($dst$$reg),
 9758              as_Register($src$$reg),
 9759              zr,
 9760              (Assembler::Condition)$cmp$$cmpcode);
 9761   %}
 9762 
 9763   ins_pipe(icond_reg);
 9764 %}
 9765 
 9766 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9767   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9768 
 9769   ins_cost(INSN_COST * 2);
 9770   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
 9771 
 9772   ins_encode %{
 9773     __ cselw(as_Register($dst$$reg),
 9774              as_Register($src$$reg),
 9775              zr,
 9776              (Assembler::Condition)$cmp$$cmpcode);
 9777   %}
 9778 
 9779   ins_pipe(icond_reg);
 9780 %}
 9781 
 9782 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9783   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9784 
 9785   ins_cost(INSN_COST * 2);
 9786   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
 9787 
 9788   ins_encode %{
 9789     __ cselw(as_Register($dst$$reg),
 9790              zr,
 9791              as_Register($src$$reg),
 9792              (Assembler::Condition)$cmp$$cmpcode);
 9793   %}
 9794 
 9795   ins_pipe(icond_reg);
 9796 %}
 9797 
 9798 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9799   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9800 
 9801   ins_cost(INSN_COST * 2);
 9802   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
 9803 
 9804   ins_encode %{
 9805     __ cselw(as_Register($dst$$reg),
 9806              zr,
 9807              as_Register($src$$reg),
 9808              (Assembler::Condition)$cmp$$cmpcode);
 9809   %}
 9810 
 9811   ins_pipe(icond_reg);
 9812 %}
 9813 
 9814 // special case for creating a boolean 0 or 1
 9815 
 9816 // n.b. this is selected in preference to the rule above because it
 9817 // avoids loading constants 0 and 1 into a source register
 9818 
 9819 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9820   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9821 
 9822   ins_cost(INSN_COST * 2);
 9823   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
 9824 
 9825   ins_encode %{
 9826     // equivalently
 9827     // cset(as_Register($dst$$reg),
 9828     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9829     __ csincw(as_Register($dst$$reg),
 9830              zr,
 9831              zr,
 9832              (Assembler::Condition)$cmp$$cmpcode);
 9833   %}
 9834 
 9835   ins_pipe(icond_none);
 9836 %}
 9837 
 9838 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9839   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9840 
 9841   ins_cost(INSN_COST * 2);
 9842   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
 9843 
 9844   ins_encode %{
 9845     // equivalently
 9846     // cset(as_Register($dst$$reg),
 9847     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9848     __ csincw(as_Register($dst$$reg),
 9849              zr,
 9850              zr,
 9851              (Assembler::Condition)$cmp$$cmpcode);
 9852   %}
 9853 
 9854   ins_pipe(icond_none);
 9855 %}
 9856 
 9857 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9858   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9859 
 9860   ins_cost(INSN_COST * 2);
 9861   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
 9862 
 9863   ins_encode %{
 9864     __ csel(as_Register($dst$$reg),
 9865             as_Register($src2$$reg),
 9866             as_Register($src1$$reg),
 9867             (Assembler::Condition)$cmp$$cmpcode);
 9868   %}
 9869 
 9870   ins_pipe(icond_reg_reg);
 9871 %}
 9872 
 9873 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9874   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9875 
 9876   ins_cost(INSN_COST * 2);
 9877   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
 9878 
 9879   ins_encode %{
 9880     __ csel(as_Register($dst$$reg),
 9881             as_Register($src2$$reg),
 9882             as_Register($src1$$reg),
 9883             (Assembler::Condition)$cmp$$cmpcode);
 9884   %}
 9885 
 9886   ins_pipe(icond_reg_reg);
 9887 %}
 9888 
 9889 // special cases where one arg is zero
 9890 
 9891 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9892   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9893 
 9894   ins_cost(INSN_COST * 2);
 9895   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
 9896 
 9897   ins_encode %{
 9898     __ csel(as_Register($dst$$reg),
 9899             zr,
 9900             as_Register($src$$reg),
 9901             (Assembler::Condition)$cmp$$cmpcode);
 9902   %}
 9903 
 9904   ins_pipe(icond_reg);
 9905 %}
 9906 
 9907 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9908   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9909 
 9910   ins_cost(INSN_COST * 2);
 9911   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
 9912 
 9913   ins_encode %{
 9914     __ csel(as_Register($dst$$reg),
 9915             zr,
 9916             as_Register($src$$reg),
 9917             (Assembler::Condition)$cmp$$cmpcode);
 9918   %}
 9919 
 9920   ins_pipe(icond_reg);
 9921 %}
 9922 
 9923 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9924   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9925 
 9926   ins_cost(INSN_COST * 2);
 9927   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
 9928 
 9929   ins_encode %{
 9930     __ csel(as_Register($dst$$reg),
 9931             as_Register($src$$reg),
 9932             zr,
 9933             (Assembler::Condition)$cmp$$cmpcode);
 9934   %}
 9935 
 9936   ins_pipe(icond_reg);
 9937 %}
 9938 
 9939 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9940   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9941 
 9942   ins_cost(INSN_COST * 2);
 9943   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
 9944 
 9945   ins_encode %{
 9946     __ csel(as_Register($dst$$reg),
 9947             as_Register($src$$reg),
 9948             zr,
 9949             (Assembler::Condition)$cmp$$cmpcode);
 9950   %}
 9951 
 9952   ins_pipe(icond_reg);
 9953 %}
 9954 
 9955 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
 9956   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
 9957 
 9958   ins_cost(INSN_COST * 2);
 9959   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
 9960 
 9961   ins_encode %{
 9962     __ csel(as_Register($dst$$reg),
 9963             as_Register($src2$$reg),
 9964             as_Register($src1$$reg),
 9965             (Assembler::Condition)$cmp$$cmpcode);
 9966   %}
 9967 
 9968   ins_pipe(icond_reg_reg);
 9969 %}
 9970 
 9971 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
 9972   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
 9973 
 9974   ins_cost(INSN_COST * 2);
 9975   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
 9976 
 9977   ins_encode %{
 9978     __ csel(as_Register($dst$$reg),
 9979             as_Register($src2$$reg),
 9980             as_Register($src1$$reg),
 9981             (Assembler::Condition)$cmp$$cmpcode);
 9982   %}
 9983 
 9984   ins_pipe(icond_reg_reg);
 9985 %}
 9986 
 9987 // special cases where one arg is zero
 9988 
 9989 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
 9990   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
 9991 
 9992   ins_cost(INSN_COST * 2);
 9993   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
 9994 
 9995   ins_encode %{
 9996     __ csel(as_Register($dst$$reg),
 9997             zr,
 9998             as_Register($src$$reg),
 9999             (Assembler::Condition)$cmp$$cmpcode);
10000   %}
10001 
10002   ins_pipe(icond_reg);
10003 %}
10004 
10005 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10006   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10007 
10008   ins_cost(INSN_COST * 2);
10009   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10010 
10011   ins_encode %{
10012     __ csel(as_Register($dst$$reg),
10013             zr,
10014             as_Register($src$$reg),
10015             (Assembler::Condition)$cmp$$cmpcode);
10016   %}
10017 
10018   ins_pipe(icond_reg);
10019 %}
10020 
10021 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10022   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10023 
10024   ins_cost(INSN_COST * 2);
10025   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10026 
10027   ins_encode %{
10028     __ csel(as_Register($dst$$reg),
10029             as_Register($src$$reg),
10030             zr,
10031             (Assembler::Condition)$cmp$$cmpcode);
10032   %}
10033 
10034   ins_pipe(icond_reg);
10035 %}
10036 
10037 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10038   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10039 
10040   ins_cost(INSN_COST * 2);
10041   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10042 
10043   ins_encode %{
10044     __ csel(as_Register($dst$$reg),
10045             as_Register($src$$reg),
10046             zr,
10047             (Assembler::Condition)$cmp$$cmpcode);
10048   %}
10049 
10050   ins_pipe(icond_reg);
10051 %}
10052 
10053 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10054   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10055 
10056   ins_cost(INSN_COST * 2);
10057   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10058 
10059   ins_encode %{
10060     __ cselw(as_Register($dst$$reg),
10061              as_Register($src2$$reg),
10062              as_Register($src1$$reg),
10063              (Assembler::Condition)$cmp$$cmpcode);
10064   %}
10065 
10066   ins_pipe(icond_reg_reg);
10067 %}
10068 
10069 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10070   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10071 
10072   ins_cost(INSN_COST * 2);
10073   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10074 
10075   ins_encode %{
10076     __ cselw(as_Register($dst$$reg),
10077              as_Register($src2$$reg),
10078              as_Register($src1$$reg),
10079              (Assembler::Condition)$cmp$$cmpcode);
10080   %}
10081 
10082   ins_pipe(icond_reg_reg);
10083 %}
10084 
10085 // special cases where one arg is zero
10086 
10087 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10088   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10089 
10090   ins_cost(INSN_COST * 2);
10091   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10092 
10093   ins_encode %{
10094     __ cselw(as_Register($dst$$reg),
10095              zr,
10096              as_Register($src$$reg),
10097              (Assembler::Condition)$cmp$$cmpcode);
10098   %}
10099 
10100   ins_pipe(icond_reg);
10101 %}
10102 
10103 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10104   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10105 
10106   ins_cost(INSN_COST * 2);
10107   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10108 
10109   ins_encode %{
10110     __ cselw(as_Register($dst$$reg),
10111              zr,
10112              as_Register($src$$reg),
10113              (Assembler::Condition)$cmp$$cmpcode);
10114   %}
10115 
10116   ins_pipe(icond_reg);
10117 %}
10118 
10119 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10120   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10121 
10122   ins_cost(INSN_COST * 2);
10123   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10124 
10125   ins_encode %{
10126     __ cselw(as_Register($dst$$reg),
10127              as_Register($src$$reg),
10128              zr,
10129              (Assembler::Condition)$cmp$$cmpcode);
10130   %}
10131 
10132   ins_pipe(icond_reg);
10133 %}
10134 
10135 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10136   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10137 
10138   ins_cost(INSN_COST * 2);
10139   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10140 
10141   ins_encode %{
10142     __ cselw(as_Register($dst$$reg),
10143              as_Register($src$$reg),
10144              zr,
10145              (Assembler::Condition)$cmp$$cmpcode);
10146   %}
10147 
10148   ins_pipe(icond_reg);
10149 %}
10150 
10151 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10152 %{
10153   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10154 
10155   ins_cost(INSN_COST * 3);
10156 
10157   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10158   ins_encode %{
10159     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10160     __ fcsels(as_FloatRegister($dst$$reg),
10161               as_FloatRegister($src2$$reg),
10162               as_FloatRegister($src1$$reg),
10163               cond);
10164   %}
10165 
10166   ins_pipe(fp_cond_reg_reg_s);
10167 %}
10168 
10169 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10170 %{
10171   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10172 
10173   ins_cost(INSN_COST * 3);
10174 
10175   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10176   ins_encode %{
10177     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10178     __ fcsels(as_FloatRegister($dst$$reg),
10179               as_FloatRegister($src2$$reg),
10180               as_FloatRegister($src1$$reg),
10181               cond);
10182   %}
10183 
10184   ins_pipe(fp_cond_reg_reg_s);
10185 %}
10186 
10187 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10188 %{
10189   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10190 
10191   ins_cost(INSN_COST * 3);
10192 
10193   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10194   ins_encode %{
10195     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10196     __ fcseld(as_FloatRegister($dst$$reg),
10197               as_FloatRegister($src2$$reg),
10198               as_FloatRegister($src1$$reg),
10199               cond);
10200   %}
10201 
10202   ins_pipe(fp_cond_reg_reg_d);
10203 %}
10204 
10205 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10206 %{
10207   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10208 
10209   ins_cost(INSN_COST * 3);
10210 
10211   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10212   ins_encode %{
10213     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10214     __ fcseld(as_FloatRegister($dst$$reg),
10215               as_FloatRegister($src2$$reg),
10216               as_FloatRegister($src1$$reg),
10217               cond);
10218   %}
10219 
10220   ins_pipe(fp_cond_reg_reg_d);
10221 %}
10222 
10223 // ============================================================================
10224 // Arithmetic Instructions
10225 //
10226 
10227 // Integer Addition
10228 
10229 // TODO
10230 // these currently employ operations which do not set CR and hence are
10231 // not flagged as killing CR but we would like to isolate the cases
10232 // where we want to set flags from those where we don't. need to work
10233 // out how to do that.
10234 
10235 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10236   match(Set dst (AddI src1 src2));
10237 
10238   ins_cost(INSN_COST);
10239   format %{ "addw  $dst, $src1, $src2" %}
10240 
10241   ins_encode %{
10242     __ addw(as_Register($dst$$reg),
10243             as_Register($src1$$reg),
10244             as_Register($src2$$reg));
10245   %}
10246 
10247   ins_pipe(ialu_reg_reg);
10248 %}
10249 
10250 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10251   match(Set dst (AddI src1 src2));
10252 
10253   ins_cost(INSN_COST);
10254   format %{ "addw $dst, $src1, $src2" %}
10255 
10256   // use opcode to indicate that this is an add not a sub
10257   opcode(0x0);
10258 
10259   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10260 
10261   ins_pipe(ialu_reg_imm);
10262 %}
10263 
10264 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10265   match(Set dst (AddI (ConvL2I src1) src2));
10266 
10267   ins_cost(INSN_COST);
10268   format %{ "addw $dst, $src1, $src2" %}
10269 
10270   // use opcode to indicate that this is an add not a sub
10271   opcode(0x0);
10272 
10273   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10274 
10275   ins_pipe(ialu_reg_imm);
10276 %}
10277 
10278 // Pointer Addition
10279 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10280   match(Set dst (AddP src1 src2));
10281 
10282   ins_cost(INSN_COST);
10283   format %{ "add $dst, $src1, $src2\t# ptr" %}
10284 
10285   ins_encode %{
10286     __ add(as_Register($dst$$reg),
10287            as_Register($src1$$reg),
10288            as_Register($src2$$reg));
10289   %}
10290 
10291   ins_pipe(ialu_reg_reg);
10292 %}
10293 
10294 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10295   match(Set dst (AddP src1 (ConvI2L src2)));
10296 
10297   ins_cost(1.9 * INSN_COST);
10298   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10299 
10300   ins_encode %{
10301     __ add(as_Register($dst$$reg),
10302            as_Register($src1$$reg),
10303            as_Register($src2$$reg), ext::sxtw);
10304   %}
10305 
10306   ins_pipe(ialu_reg_reg);
10307 %}
10308 
10309 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10310   match(Set dst (AddP src1 (LShiftL src2 scale)));
10311 
10312   ins_cost(1.9 * INSN_COST);
10313   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10314 
10315   ins_encode %{
10316     __ lea(as_Register($dst$$reg),
10317            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10318                    Address::lsl($scale$$constant)));
10319   %}
10320 
10321   ins_pipe(ialu_reg_reg_shift);
10322 %}
10323 
10324 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10325   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10326 
10327   ins_cost(1.9 * INSN_COST);
10328   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10329 
10330   ins_encode %{
10331     __ lea(as_Register($dst$$reg),
10332            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10333                    Address::sxtw($scale$$constant)));
10334   %}
10335 
10336   ins_pipe(ialu_reg_reg_shift);
10337 %}
10338 
10339 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10340   match(Set dst (LShiftL (ConvI2L src) scale));
10341 
10342   ins_cost(INSN_COST);
10343   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10344 
10345   ins_encode %{
10346     __ sbfiz(as_Register($dst$$reg),
10347           as_Register($src$$reg),
10348           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10349   %}
10350 
10351   ins_pipe(ialu_reg_shift);
10352 %}
10353 
10354 // Pointer Immediate Addition
10355 // n.b. this needs to be more expensive than using an indirect memory
10356 // operand
10357 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10358   match(Set dst (AddP src1 src2));
10359 
10360   ins_cost(INSN_COST);
10361   format %{ "add $dst, $src1, $src2\t# ptr" %}
10362 
10363   // use opcode to indicate that this is an add not a sub
10364   opcode(0x0);
10365 
10366   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10367 
10368   ins_pipe(ialu_reg_imm);
10369 %}
10370 
10371 // Long Addition
10372 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10373 
10374   match(Set dst (AddL src1 src2));
10375 
10376   ins_cost(INSN_COST);
10377   format %{ "add  $dst, $src1, $src2" %}
10378 
10379   ins_encode %{
10380     __ add(as_Register($dst$$reg),
10381            as_Register($src1$$reg),
10382            as_Register($src2$$reg));
10383   %}
10384 
10385   ins_pipe(ialu_reg_reg);
10386 %}
10387 
10388 // No constant pool entries requiredLong Immediate Addition.
10389 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10390   match(Set dst (AddL src1 src2));
10391 
10392   ins_cost(INSN_COST);
10393   format %{ "add $dst, $src1, $src2" %}
10394 
10395   // use opcode to indicate that this is an add not a sub
10396   opcode(0x0);
10397 
10398   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10399 
10400   ins_pipe(ialu_reg_imm);
10401 %}
10402 
10403 // Integer Subtraction
10404 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10405   match(Set dst (SubI src1 src2));
10406 
10407   ins_cost(INSN_COST);
10408   format %{ "subw  $dst, $src1, $src2" %}
10409 
10410   ins_encode %{
10411     __ subw(as_Register($dst$$reg),
10412             as_Register($src1$$reg),
10413             as_Register($src2$$reg));
10414   %}
10415 
10416   ins_pipe(ialu_reg_reg);
10417 %}
10418 
10419 // Immediate Subtraction
10420 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10421   match(Set dst (SubI src1 src2));
10422 
10423   ins_cost(INSN_COST);
10424   format %{ "subw $dst, $src1, $src2" %}
10425 
10426   // use opcode to indicate that this is a sub not an add
10427   opcode(0x1);
10428 
10429   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10430 
10431   ins_pipe(ialu_reg_imm);
10432 %}
10433 
10434 // Long Subtraction
10435 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10436 
10437   match(Set dst (SubL src1 src2));
10438 
10439   ins_cost(INSN_COST);
10440   format %{ "sub  $dst, $src1, $src2" %}
10441 
10442   ins_encode %{
10443     __ sub(as_Register($dst$$reg),
10444            as_Register($src1$$reg),
10445            as_Register($src2$$reg));
10446   %}
10447 
10448   ins_pipe(ialu_reg_reg);
10449 %}
10450 
10451 // No constant pool entries requiredLong Immediate Subtraction.
10452 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10453   match(Set dst (SubL src1 src2));
10454 
10455   ins_cost(INSN_COST);
10456   format %{ "sub$dst, $src1, $src2" %}
10457 
10458   // use opcode to indicate that this is a sub not an add
10459   opcode(0x1);
10460 
10461   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10462 
10463   ins_pipe(ialu_reg_imm);
10464 %}
10465 
10466 // Integer Negation (special case for sub)
10467 
10468 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10469   match(Set dst (SubI zero src));
10470 
10471   ins_cost(INSN_COST);
10472   format %{ "negw $dst, $src\t# int" %}
10473 
10474   ins_encode %{
10475     __ negw(as_Register($dst$$reg),
10476             as_Register($src$$reg));
10477   %}
10478 
10479   ins_pipe(ialu_reg);
10480 %}
10481 
10482 // Long Negation
10483 
10484 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10485   match(Set dst (SubL zero src));
10486 
10487   ins_cost(INSN_COST);
10488   format %{ "neg $dst, $src\t# long" %}
10489 
10490   ins_encode %{
10491     __ neg(as_Register($dst$$reg),
10492            as_Register($src$$reg));
10493   %}
10494 
10495   ins_pipe(ialu_reg);
10496 %}
10497 
10498 // Integer Multiply
10499 
10500 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10501   match(Set dst (MulI src1 src2));
10502 
10503   ins_cost(INSN_COST * 3);
10504   format %{ "mulw  $dst, $src1, $src2" %}
10505 
10506   ins_encode %{
10507     __ mulw(as_Register($dst$$reg),
10508             as_Register($src1$$reg),
10509             as_Register($src2$$reg));
10510   %}
10511 
10512   ins_pipe(imul_reg_reg);
10513 %}
10514 
10515 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10516   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10517 
10518   ins_cost(INSN_COST * 3);
10519   format %{ "smull  $dst, $src1, $src2" %}
10520 
10521   ins_encode %{
10522     __ smull(as_Register($dst$$reg),
10523              as_Register($src1$$reg),
10524              as_Register($src2$$reg));
10525   %}
10526 
10527   ins_pipe(imul_reg_reg);
10528 %}
10529 
10530 // Long Multiply
10531 
10532 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10533   match(Set dst (MulL src1 src2));
10534 
10535   ins_cost(INSN_COST * 5);
10536   format %{ "mul  $dst, $src1, $src2" %}
10537 
10538   ins_encode %{
10539     __ mul(as_Register($dst$$reg),
10540            as_Register($src1$$reg),
10541            as_Register($src2$$reg));
10542   %}
10543 
10544   ins_pipe(lmul_reg_reg);
10545 %}
10546 
10547 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10548 %{
10549   match(Set dst (MulHiL src1 src2));
10550 
10551   ins_cost(INSN_COST * 7);
10552   format %{ "smulh   $dst, $src1, $src2\t# mulhi" %}
10553 
10554   ins_encode %{
10555     __ smulh(as_Register($dst$$reg),
10556              as_Register($src1$$reg),
10557              as_Register($src2$$reg));
10558   %}
10559 
10560   ins_pipe(lmul_reg_reg);
10561 %}
10562 
10563 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10564 %{
10565   match(Set dst (UMulHiL src1 src2));
10566 
10567   ins_cost(INSN_COST * 7);
10568   format %{ "umulh   $dst, $src1, $src2\t# umulhi" %}
10569 
10570   ins_encode %{
10571     __ umulh(as_Register($dst$$reg),
10572              as_Register($src1$$reg),
10573              as_Register($src2$$reg));
10574   %}
10575 
10576   ins_pipe(lmul_reg_reg);
10577 %}
10578 
10579 // Combined Integer Multiply & Add/Sub
10580 
10581 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10582   match(Set dst (AddI src3 (MulI src1 src2)));
10583 
10584   ins_cost(INSN_COST * 3);
10585   format %{ "madd  $dst, $src1, $src2, $src3" %}
10586 
10587   ins_encode %{
10588     __ maddw(as_Register($dst$$reg),
10589              as_Register($src1$$reg),
10590              as_Register($src2$$reg),
10591              as_Register($src3$$reg));
10592   %}
10593 
10594   ins_pipe(imac_reg_reg);
10595 %}
10596 
10597 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10598   match(Set dst (SubI src3 (MulI src1 src2)));
10599 
10600   ins_cost(INSN_COST * 3);
10601   format %{ "msub  $dst, $src1, $src2, $src3" %}
10602 
10603   ins_encode %{
10604     __ msubw(as_Register($dst$$reg),
10605              as_Register($src1$$reg),
10606              as_Register($src2$$reg),
10607              as_Register($src3$$reg));
10608   %}
10609 
10610   ins_pipe(imac_reg_reg);
10611 %}
10612 
10613 // Combined Integer Multiply & Neg
10614 
10615 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10616   match(Set dst (MulI (SubI zero src1) src2));
10617 
10618   ins_cost(INSN_COST * 3);
10619   format %{ "mneg  $dst, $src1, $src2" %}
10620 
10621   ins_encode %{
10622     __ mnegw(as_Register($dst$$reg),
10623              as_Register($src1$$reg),
10624              as_Register($src2$$reg));
10625   %}
10626 
10627   ins_pipe(imac_reg_reg);
10628 %}
10629 
10630 // Combined Long Multiply & Add/Sub
10631 
10632 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10633   match(Set dst (AddL src3 (MulL src1 src2)));
10634 
10635   ins_cost(INSN_COST * 5);
10636   format %{ "madd  $dst, $src1, $src2, $src3" %}
10637 
10638   ins_encode %{
10639     __ madd(as_Register($dst$$reg),
10640             as_Register($src1$$reg),
10641             as_Register($src2$$reg),
10642             as_Register($src3$$reg));
10643   %}
10644 
10645   ins_pipe(lmac_reg_reg);
10646 %}
10647 
10648 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10649   match(Set dst (SubL src3 (MulL src1 src2)));
10650 
10651   ins_cost(INSN_COST * 5);
10652   format %{ "msub  $dst, $src1, $src2, $src3" %}
10653 
10654   ins_encode %{
10655     __ msub(as_Register($dst$$reg),
10656             as_Register($src1$$reg),
10657             as_Register($src2$$reg),
10658             as_Register($src3$$reg));
10659   %}
10660 
10661   ins_pipe(lmac_reg_reg);
10662 %}
10663 
10664 // Combined Long Multiply & Neg
10665 
10666 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10667   match(Set dst (MulL (SubL zero src1) src2));
10668 
10669   ins_cost(INSN_COST * 5);
10670   format %{ "mneg  $dst, $src1, $src2" %}
10671 
10672   ins_encode %{
10673     __ mneg(as_Register($dst$$reg),
10674             as_Register($src1$$reg),
10675             as_Register($src2$$reg));
10676   %}
10677 
10678   ins_pipe(lmac_reg_reg);
10679 %}
10680 
10681 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10682 
10683 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10684   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10685 
10686   ins_cost(INSN_COST * 3);
10687   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10688 
10689   ins_encode %{
10690     __ smaddl(as_Register($dst$$reg),
10691               as_Register($src1$$reg),
10692               as_Register($src2$$reg),
10693               as_Register($src3$$reg));
10694   %}
10695 
10696   ins_pipe(imac_reg_reg);
10697 %}
10698 
10699 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10700   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10701 
10702   ins_cost(INSN_COST * 3);
10703   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10704 
10705   ins_encode %{
10706     __ smsubl(as_Register($dst$$reg),
10707               as_Register($src1$$reg),
10708               as_Register($src2$$reg),
10709               as_Register($src3$$reg));
10710   %}
10711 
10712   ins_pipe(imac_reg_reg);
10713 %}
10714 
10715 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10716   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10717 
10718   ins_cost(INSN_COST * 3);
10719   format %{ "smnegl  $dst, $src1, $src2" %}
10720 
10721   ins_encode %{
10722     __ smnegl(as_Register($dst$$reg),
10723               as_Register($src1$$reg),
10724               as_Register($src2$$reg));
10725   %}
10726 
10727   ins_pipe(imac_reg_reg);
10728 %}
10729 
10730 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10731 
10732 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10733   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10734 
10735   ins_cost(INSN_COST * 5);
10736   format %{ "mulw  rscratch1, $src1, $src2\n\t"
10737             "maddw $dst, $src3, $src4, rscratch1" %}
10738 
10739   ins_encode %{
10740     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10741     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10742 
10743   ins_pipe(imac_reg_reg);
10744 %}
10745 
10746 // Integer Divide
10747 
10748 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10749   match(Set dst (DivI src1 src2));
10750 
10751   ins_cost(INSN_COST * 19);
10752   format %{ "sdivw  $dst, $src1, $src2" %}
10753 
10754   ins_encode(aarch64_enc_divw(dst, src1, src2));
10755   ins_pipe(idiv_reg_reg);
10756 %}
10757 
10758 // Long Divide
10759 
10760 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10761   match(Set dst (DivL src1 src2));
10762 
10763   ins_cost(INSN_COST * 35);
10764   format %{ "sdiv   $dst, $src1, $src2" %}
10765 
10766   ins_encode(aarch64_enc_div(dst, src1, src2));
10767   ins_pipe(ldiv_reg_reg);
10768 %}
10769 
10770 // Integer Remainder
10771 
10772 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10773   match(Set dst (ModI src1 src2));
10774 
10775   ins_cost(INSN_COST * 22);
10776   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
10777             "msubw  $dst, rscratch1, $src2, $src1" %}
10778 
10779   ins_encode(aarch64_enc_modw(dst, src1, src2));
10780   ins_pipe(idiv_reg_reg);
10781 %}
10782 
10783 // Long Remainder
10784 
10785 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10786   match(Set dst (ModL src1 src2));
10787 
10788   ins_cost(INSN_COST * 38);
10789   format %{ "sdiv   rscratch1, $src1, $src2\n"
10790             "msub   $dst, rscratch1, $src2, $src1" %}
10791 
10792   ins_encode(aarch64_enc_mod(dst, src1, src2));
10793   ins_pipe(ldiv_reg_reg);
10794 %}
10795 
10796 // Unsigned Integer Divide
10797 
10798 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10799   match(Set dst (UDivI src1 src2));
10800 
10801   ins_cost(INSN_COST * 19);
10802   format %{ "udivw  $dst, $src1, $src2" %}
10803 
10804   ins_encode %{
10805     __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10806   %}
10807 
10808   ins_pipe(idiv_reg_reg);
10809 %}
10810 
10811 //  Unsigned Long Divide
10812 
10813 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10814   match(Set dst (UDivL src1 src2));
10815 
10816   ins_cost(INSN_COST * 35);
10817   format %{ "udiv   $dst, $src1, $src2" %}
10818 
10819   ins_encode %{
10820     __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10821   %}
10822 
10823   ins_pipe(ldiv_reg_reg);
10824 %}
10825 
10826 // Unsigned Integer Remainder
10827 
10828 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10829   match(Set dst (UModI src1 src2));
10830 
10831   ins_cost(INSN_COST * 22);
10832   format %{ "udivw  rscratch1, $src1, $src2\n\t"
10833             "msubw  $dst, rscratch1, $src2, $src1" %}
10834 
10835   ins_encode %{
10836     __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10837     __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10838   %}
10839 
10840   ins_pipe(idiv_reg_reg);
10841 %}
10842 
10843 // Unsigned Long Remainder
10844 
10845 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10846   match(Set dst (UModL src1 src2));
10847 
10848   ins_cost(INSN_COST * 38);
10849   format %{ "udiv   rscratch1, $src1, $src2\n"
10850             "msub   $dst, rscratch1, $src2, $src1" %}
10851 
10852   ins_encode %{
10853     __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10854     __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10855   %}
10856 
10857   ins_pipe(ldiv_reg_reg);
10858 %}
10859 
10860 // Integer Shifts
10861 
10862 // Shift Left Register
10863 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10864   match(Set dst (LShiftI src1 src2));
10865 
10866   ins_cost(INSN_COST * 2);
10867   format %{ "lslvw  $dst, $src1, $src2" %}
10868 
10869   ins_encode %{
10870     __ lslvw(as_Register($dst$$reg),
10871              as_Register($src1$$reg),
10872              as_Register($src2$$reg));
10873   %}
10874 
10875   ins_pipe(ialu_reg_reg_vshift);
10876 %}
10877 
10878 // Shift Left Immediate
10879 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10880   match(Set dst (LShiftI src1 src2));
10881 
10882   ins_cost(INSN_COST);
10883   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10884 
10885   ins_encode %{
10886     __ lslw(as_Register($dst$$reg),
10887             as_Register($src1$$reg),
10888             $src2$$constant & 0x1f);
10889   %}
10890 
10891   ins_pipe(ialu_reg_shift);
10892 %}
10893 
10894 // Shift Right Logical Register
10895 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10896   match(Set dst (URShiftI src1 src2));
10897 
10898   ins_cost(INSN_COST * 2);
10899   format %{ "lsrvw  $dst, $src1, $src2" %}
10900 
10901   ins_encode %{
10902     __ lsrvw(as_Register($dst$$reg),
10903              as_Register($src1$$reg),
10904              as_Register($src2$$reg));
10905   %}
10906 
10907   ins_pipe(ialu_reg_reg_vshift);
10908 %}
10909 
10910 // Shift Right Logical Immediate
10911 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10912   match(Set dst (URShiftI src1 src2));
10913 
10914   ins_cost(INSN_COST);
10915   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10916 
10917   ins_encode %{
10918     __ lsrw(as_Register($dst$$reg),
10919             as_Register($src1$$reg),
10920             $src2$$constant & 0x1f);
10921   %}
10922 
10923   ins_pipe(ialu_reg_shift);
10924 %}
10925 
10926 // Shift Right Arithmetic Register
10927 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10928   match(Set dst (RShiftI src1 src2));
10929 
10930   ins_cost(INSN_COST * 2);
10931   format %{ "asrvw  $dst, $src1, $src2" %}
10932 
10933   ins_encode %{
10934     __ asrvw(as_Register($dst$$reg),
10935              as_Register($src1$$reg),
10936              as_Register($src2$$reg));
10937   %}
10938 
10939   ins_pipe(ialu_reg_reg_vshift);
10940 %}
10941 
10942 // Shift Right Arithmetic Immediate
10943 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10944   match(Set dst (RShiftI src1 src2));
10945 
10946   ins_cost(INSN_COST);
10947   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10948 
10949   ins_encode %{
10950     __ asrw(as_Register($dst$$reg),
10951             as_Register($src1$$reg),
10952             $src2$$constant & 0x1f);
10953   %}
10954 
10955   ins_pipe(ialu_reg_shift);
10956 %}
10957 
10958 // Combined Int Mask and Right Shift (using UBFM)
10959 // TODO
10960 
10961 // Long Shifts
10962 
10963 // Shift Left Register
10964 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10965   match(Set dst (LShiftL src1 src2));
10966 
10967   ins_cost(INSN_COST * 2);
10968   format %{ "lslv  $dst, $src1, $src2" %}
10969 
10970   ins_encode %{
10971     __ lslv(as_Register($dst$$reg),
10972             as_Register($src1$$reg),
10973             as_Register($src2$$reg));
10974   %}
10975 
10976   ins_pipe(ialu_reg_reg_vshift);
10977 %}
10978 
10979 // Shift Left Immediate
10980 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10981   match(Set dst (LShiftL src1 src2));
10982 
10983   ins_cost(INSN_COST);
10984   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10985 
10986   ins_encode %{
10987     __ lsl(as_Register($dst$$reg),
10988             as_Register($src1$$reg),
10989             $src2$$constant & 0x3f);
10990   %}
10991 
10992   ins_pipe(ialu_reg_shift);
10993 %}
10994 
10995 // Shift Right Logical Register
10996 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10997   match(Set dst (URShiftL src1 src2));
10998 
10999   ins_cost(INSN_COST * 2);
11000   format %{ "lsrv  $dst, $src1, $src2" %}
11001 
11002   ins_encode %{
11003     __ lsrv(as_Register($dst$$reg),
11004             as_Register($src1$$reg),
11005             as_Register($src2$$reg));
11006   %}
11007 
11008   ins_pipe(ialu_reg_reg_vshift);
11009 %}
11010 
11011 // Shift Right Logical Immediate
11012 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11013   match(Set dst (URShiftL src1 src2));
11014 
11015   ins_cost(INSN_COST);
11016   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11017 
11018   ins_encode %{
11019     __ lsr(as_Register($dst$$reg),
11020            as_Register($src1$$reg),
11021            $src2$$constant & 0x3f);
11022   %}
11023 
11024   ins_pipe(ialu_reg_shift);
11025 %}
11026 
11027 // A special-case pattern for card table stores.
11028 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11029   match(Set dst (URShiftL (CastP2X src1) src2));
11030 
11031   ins_cost(INSN_COST);
11032   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11033 
11034   ins_encode %{
11035     __ lsr(as_Register($dst$$reg),
11036            as_Register($src1$$reg),
11037            $src2$$constant & 0x3f);
11038   %}
11039 
11040   ins_pipe(ialu_reg_shift);
11041 %}
11042 
11043 // Shift Right Arithmetic Register
11044 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11045   match(Set dst (RShiftL src1 src2));
11046 
11047   ins_cost(INSN_COST * 2);
11048   format %{ "asrv  $dst, $src1, $src2" %}
11049 
11050   ins_encode %{
11051     __ asrv(as_Register($dst$$reg),
11052             as_Register($src1$$reg),
11053             as_Register($src2$$reg));
11054   %}
11055 
11056   ins_pipe(ialu_reg_reg_vshift);
11057 %}
11058 
11059 // Shift Right Arithmetic Immediate
11060 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11061   match(Set dst (RShiftL src1 src2));
11062 
11063   ins_cost(INSN_COST);
11064   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11065 
11066   ins_encode %{
11067     __ asr(as_Register($dst$$reg),
11068            as_Register($src1$$reg),
11069            $src2$$constant & 0x3f);
11070   %}
11071 
11072   ins_pipe(ialu_reg_shift);
11073 %}
11074 
11075 // BEGIN This section of the file is automatically generated. Do not edit --------------
11076 // This section is generated from aarch64_ad.m4
11077 
11078 
11079 // This pattern is automatically generated from aarch64_ad.m4
11080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11081 instruct regL_not_reg(iRegLNoSp dst,
11082                          iRegL src1, immL_M1 m1,
11083                          rFlagsReg cr) %{
11084   match(Set dst (XorL src1 m1));
11085   ins_cost(INSN_COST);
11086   format %{ "eon  $dst, $src1, zr" %}
11087 
11088   ins_encode %{
11089     __ eon(as_Register($dst$$reg),
11090               as_Register($src1$$reg),
11091               zr,
11092               Assembler::LSL, 0);
11093   %}
11094 
11095   ins_pipe(ialu_reg);
11096 %}
11097 
11098 // This pattern is automatically generated from aarch64_ad.m4
11099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11100 instruct regI_not_reg(iRegINoSp dst,
11101                          iRegIorL2I src1, immI_M1 m1,
11102                          rFlagsReg cr) %{
11103   match(Set dst (XorI src1 m1));
11104   ins_cost(INSN_COST);
11105   format %{ "eonw  $dst, $src1, zr" %}
11106 
11107   ins_encode %{
11108     __ eonw(as_Register($dst$$reg),
11109               as_Register($src1$$reg),
11110               zr,
11111               Assembler::LSL, 0);
11112   %}
11113 
11114   ins_pipe(ialu_reg);
11115 %}
11116 
11117 // This pattern is automatically generated from aarch64_ad.m4
11118 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11119 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11120                               immI0 zero, iRegIorL2I src1, immI src2) %{
11121   match(Set dst (SubI zero (URShiftI src1 src2)));
11122 
11123   ins_cost(1.9 * INSN_COST);
11124   format %{ "negw  $dst, $src1, LSR $src2" %}
11125 
11126   ins_encode %{
11127     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11128             Assembler::LSR, $src2$$constant & 0x1f);
11129   %}
11130 
11131   ins_pipe(ialu_reg_shift);
11132 %}
11133 
11134 // This pattern is automatically generated from aarch64_ad.m4
11135 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11136 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11137                               immI0 zero, iRegIorL2I src1, immI src2) %{
11138   match(Set dst (SubI zero (RShiftI src1 src2)));
11139 
11140   ins_cost(1.9 * INSN_COST);
11141   format %{ "negw  $dst, $src1, ASR $src2" %}
11142 
11143   ins_encode %{
11144     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11145             Assembler::ASR, $src2$$constant & 0x1f);
11146   %}
11147 
11148   ins_pipe(ialu_reg_shift);
11149 %}
11150 
11151 // This pattern is automatically generated from aarch64_ad.m4
11152 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11153 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11154                               immI0 zero, iRegIorL2I src1, immI src2) %{
11155   match(Set dst (SubI zero (LShiftI src1 src2)));
11156 
11157   ins_cost(1.9 * INSN_COST);
11158   format %{ "negw  $dst, $src1, LSL $src2" %}
11159 
11160   ins_encode %{
11161     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11162             Assembler::LSL, $src2$$constant & 0x1f);
11163   %}
11164 
11165   ins_pipe(ialu_reg_shift);
11166 %}
11167 
11168 // This pattern is automatically generated from aarch64_ad.m4
11169 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11170 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11171                               immL0 zero, iRegL src1, immI src2) %{
11172   match(Set dst (SubL zero (URShiftL src1 src2)));
11173 
11174   ins_cost(1.9 * INSN_COST);
11175   format %{ "neg  $dst, $src1, LSR $src2" %}
11176 
11177   ins_encode %{
11178     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11179             Assembler::LSR, $src2$$constant & 0x3f);
11180   %}
11181 
11182   ins_pipe(ialu_reg_shift);
11183 %}
11184 
11185 // This pattern is automatically generated from aarch64_ad.m4
11186 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11187 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11188                               immL0 zero, iRegL src1, immI src2) %{
11189   match(Set dst (SubL zero (RShiftL src1 src2)));
11190 
11191   ins_cost(1.9 * INSN_COST);
11192   format %{ "neg  $dst, $src1, ASR $src2" %}
11193 
11194   ins_encode %{
11195     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11196             Assembler::ASR, $src2$$constant & 0x3f);
11197   %}
11198 
11199   ins_pipe(ialu_reg_shift);
11200 %}
11201 
11202 // This pattern is automatically generated from aarch64_ad.m4
11203 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11204 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11205                               immL0 zero, iRegL src1, immI src2) %{
11206   match(Set dst (SubL zero (LShiftL src1 src2)));
11207 
11208   ins_cost(1.9 * INSN_COST);
11209   format %{ "neg  $dst, $src1, LSL $src2" %}
11210 
11211   ins_encode %{
11212     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11213             Assembler::LSL, $src2$$constant & 0x3f);
11214   %}
11215 
11216   ins_pipe(ialu_reg_shift);
11217 %}
11218 
11219 // This pattern is automatically generated from aarch64_ad.m4
11220 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11221 instruct AndI_reg_not_reg(iRegINoSp dst,
11222                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11223   match(Set dst (AndI src1 (XorI src2 m1)));
11224   ins_cost(INSN_COST);
11225   format %{ "bicw  $dst, $src1, $src2" %}
11226 
11227   ins_encode %{
11228     __ bicw(as_Register($dst$$reg),
11229               as_Register($src1$$reg),
11230               as_Register($src2$$reg),
11231               Assembler::LSL, 0);
11232   %}
11233 
11234   ins_pipe(ialu_reg_reg);
11235 %}
11236 
11237 // This pattern is automatically generated from aarch64_ad.m4
11238 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11239 instruct AndL_reg_not_reg(iRegLNoSp dst,
11240                          iRegL src1, iRegL src2, immL_M1 m1) %{
11241   match(Set dst (AndL src1 (XorL src2 m1)));
11242   ins_cost(INSN_COST);
11243   format %{ "bic  $dst, $src1, $src2" %}
11244 
11245   ins_encode %{
11246     __ bic(as_Register($dst$$reg),
11247               as_Register($src1$$reg),
11248               as_Register($src2$$reg),
11249               Assembler::LSL, 0);
11250   %}
11251 
11252   ins_pipe(ialu_reg_reg);
11253 %}
11254 
11255 // This pattern is automatically generated from aarch64_ad.m4
11256 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11257 instruct OrI_reg_not_reg(iRegINoSp dst,
11258                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11259   match(Set dst (OrI src1 (XorI src2 m1)));
11260   ins_cost(INSN_COST);
11261   format %{ "ornw  $dst, $src1, $src2" %}
11262 
11263   ins_encode %{
11264     __ ornw(as_Register($dst$$reg),
11265               as_Register($src1$$reg),
11266               as_Register($src2$$reg),
11267               Assembler::LSL, 0);
11268   %}
11269 
11270   ins_pipe(ialu_reg_reg);
11271 %}
11272 
11273 // This pattern is automatically generated from aarch64_ad.m4
11274 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11275 instruct OrL_reg_not_reg(iRegLNoSp dst,
11276                          iRegL src1, iRegL src2, immL_M1 m1) %{
11277   match(Set dst (OrL src1 (XorL src2 m1)));
11278   ins_cost(INSN_COST);
11279   format %{ "orn  $dst, $src1, $src2" %}
11280 
11281   ins_encode %{
11282     __ orn(as_Register($dst$$reg),
11283               as_Register($src1$$reg),
11284               as_Register($src2$$reg),
11285               Assembler::LSL, 0);
11286   %}
11287 
11288   ins_pipe(ialu_reg_reg);
11289 %}
11290 
11291 // This pattern is automatically generated from aarch64_ad.m4
11292 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11293 instruct XorI_reg_not_reg(iRegINoSp dst,
11294                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11295   match(Set dst (XorI m1 (XorI src2 src1)));
11296   ins_cost(INSN_COST);
11297   format %{ "eonw  $dst, $src1, $src2" %}
11298 
11299   ins_encode %{
11300     __ eonw(as_Register($dst$$reg),
11301               as_Register($src1$$reg),
11302               as_Register($src2$$reg),
11303               Assembler::LSL, 0);
11304   %}
11305 
11306   ins_pipe(ialu_reg_reg);
11307 %}
11308 
11309 // This pattern is automatically generated from aarch64_ad.m4
11310 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11311 instruct XorL_reg_not_reg(iRegLNoSp dst,
11312                          iRegL src1, iRegL src2, immL_M1 m1) %{
11313   match(Set dst (XorL m1 (XorL src2 src1)));
11314   ins_cost(INSN_COST);
11315   format %{ "eon  $dst, $src1, $src2" %}
11316 
11317   ins_encode %{
11318     __ eon(as_Register($dst$$reg),
11319               as_Register($src1$$reg),
11320               as_Register($src2$$reg),
11321               Assembler::LSL, 0);
11322   %}
11323 
11324   ins_pipe(ialu_reg_reg);
11325 %}
11326 
11327 // This pattern is automatically generated from aarch64_ad.m4
11328 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11329 // val & (-1 ^ (val >>> shift)) ==> bicw
11330 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11331                          iRegIorL2I src1, iRegIorL2I src2,
11332                          immI src3, immI_M1 src4) %{
11333   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11334   ins_cost(1.9 * INSN_COST);
11335   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11336 
11337   ins_encode %{
11338     __ bicw(as_Register($dst$$reg),
11339               as_Register($src1$$reg),
11340               as_Register($src2$$reg),
11341               Assembler::LSR,
11342               $src3$$constant & 0x1f);
11343   %}
11344 
11345   ins_pipe(ialu_reg_reg_shift);
11346 %}
11347 
11348 // This pattern is automatically generated from aarch64_ad.m4
11349 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11350 // val & (-1 ^ (val >>> shift)) ==> bic
11351 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11352                          iRegL src1, iRegL src2,
11353                          immI src3, immL_M1 src4) %{
11354   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11355   ins_cost(1.9 * INSN_COST);
11356   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11357 
11358   ins_encode %{
11359     __ bic(as_Register($dst$$reg),
11360               as_Register($src1$$reg),
11361               as_Register($src2$$reg),
11362               Assembler::LSR,
11363               $src3$$constant & 0x3f);
11364   %}
11365 
11366   ins_pipe(ialu_reg_reg_shift);
11367 %}
11368 
11369 // This pattern is automatically generated from aarch64_ad.m4
11370 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11371 // val & (-1 ^ (val >> shift)) ==> bicw
11372 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11373                          iRegIorL2I src1, iRegIorL2I src2,
11374                          immI src3, immI_M1 src4) %{
11375   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11376   ins_cost(1.9 * INSN_COST);
11377   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11378 
11379   ins_encode %{
11380     __ bicw(as_Register($dst$$reg),
11381               as_Register($src1$$reg),
11382               as_Register($src2$$reg),
11383               Assembler::ASR,
11384               $src3$$constant & 0x1f);
11385   %}
11386 
11387   ins_pipe(ialu_reg_reg_shift);
11388 %}
11389 
11390 // This pattern is automatically generated from aarch64_ad.m4
11391 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11392 // val & (-1 ^ (val >> shift)) ==> bic
11393 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11394                          iRegL src1, iRegL src2,
11395                          immI src3, immL_M1 src4) %{
11396   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11397   ins_cost(1.9 * INSN_COST);
11398   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11399 
11400   ins_encode %{
11401     __ bic(as_Register($dst$$reg),
11402               as_Register($src1$$reg),
11403               as_Register($src2$$reg),
11404               Assembler::ASR,
11405               $src3$$constant & 0x3f);
11406   %}
11407 
11408   ins_pipe(ialu_reg_reg_shift);
11409 %}
11410 
11411 // This pattern is automatically generated from aarch64_ad.m4
11412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11413 // val & (-1 ^ (val ror shift)) ==> bicw
11414 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11415                          iRegIorL2I src1, iRegIorL2I src2,
11416                          immI src3, immI_M1 src4) %{
11417   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11418   ins_cost(1.9 * INSN_COST);
11419   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11420 
11421   ins_encode %{
11422     __ bicw(as_Register($dst$$reg),
11423               as_Register($src1$$reg),
11424               as_Register($src2$$reg),
11425               Assembler::ROR,
11426               $src3$$constant & 0x1f);
11427   %}
11428 
11429   ins_pipe(ialu_reg_reg_shift);
11430 %}
11431 
11432 // This pattern is automatically generated from aarch64_ad.m4
11433 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11434 // val & (-1 ^ (val ror shift)) ==> bic
11435 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11436                          iRegL src1, iRegL src2,
11437                          immI src3, immL_M1 src4) %{
11438   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11439   ins_cost(1.9 * INSN_COST);
11440   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11441 
11442   ins_encode %{
11443     __ bic(as_Register($dst$$reg),
11444               as_Register($src1$$reg),
11445               as_Register($src2$$reg),
11446               Assembler::ROR,
11447               $src3$$constant & 0x3f);
11448   %}
11449 
11450   ins_pipe(ialu_reg_reg_shift);
11451 %}
11452 
11453 // This pattern is automatically generated from aarch64_ad.m4
11454 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11455 // val & (-1 ^ (val << shift)) ==> bicw
11456 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11457                          iRegIorL2I src1, iRegIorL2I src2,
11458                          immI src3, immI_M1 src4) %{
11459   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11460   ins_cost(1.9 * INSN_COST);
11461   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11462 
11463   ins_encode %{
11464     __ bicw(as_Register($dst$$reg),
11465               as_Register($src1$$reg),
11466               as_Register($src2$$reg),
11467               Assembler::LSL,
11468               $src3$$constant & 0x1f);
11469   %}
11470 
11471   ins_pipe(ialu_reg_reg_shift);
11472 %}
11473 
11474 // This pattern is automatically generated from aarch64_ad.m4
11475 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11476 // val & (-1 ^ (val << shift)) ==> bic
11477 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11478                          iRegL src1, iRegL src2,
11479                          immI src3, immL_M1 src4) %{
11480   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11481   ins_cost(1.9 * INSN_COST);
11482   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11483 
11484   ins_encode %{
11485     __ bic(as_Register($dst$$reg),
11486               as_Register($src1$$reg),
11487               as_Register($src2$$reg),
11488               Assembler::LSL,
11489               $src3$$constant & 0x3f);
11490   %}
11491 
11492   ins_pipe(ialu_reg_reg_shift);
11493 %}
11494 
11495 // This pattern is automatically generated from aarch64_ad.m4
11496 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11497 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11498 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11499                          iRegIorL2I src1, iRegIorL2I src2,
11500                          immI src3, immI_M1 src4) %{
11501   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11502   ins_cost(1.9 * INSN_COST);
11503   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11504 
11505   ins_encode %{
11506     __ eonw(as_Register($dst$$reg),
11507               as_Register($src1$$reg),
11508               as_Register($src2$$reg),
11509               Assembler::LSR,
11510               $src3$$constant & 0x1f);
11511   %}
11512 
11513   ins_pipe(ialu_reg_reg_shift);
11514 %}
11515 
11516 // This pattern is automatically generated from aarch64_ad.m4
11517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11518 // val ^ (-1 ^ (val >>> shift)) ==> eon
11519 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11520                          iRegL src1, iRegL src2,
11521                          immI src3, immL_M1 src4) %{
11522   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11523   ins_cost(1.9 * INSN_COST);
11524   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11525 
11526   ins_encode %{
11527     __ eon(as_Register($dst$$reg),
11528               as_Register($src1$$reg),
11529               as_Register($src2$$reg),
11530               Assembler::LSR,
11531               $src3$$constant & 0x3f);
11532   %}
11533 
11534   ins_pipe(ialu_reg_reg_shift);
11535 %}
11536 
11537 // This pattern is automatically generated from aarch64_ad.m4
11538 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11539 // val ^ (-1 ^ (val >> shift)) ==> eonw
11540 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11541                          iRegIorL2I src1, iRegIorL2I src2,
11542                          immI src3, immI_M1 src4) %{
11543   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11544   ins_cost(1.9 * INSN_COST);
11545   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11546 
11547   ins_encode %{
11548     __ eonw(as_Register($dst$$reg),
11549               as_Register($src1$$reg),
11550               as_Register($src2$$reg),
11551               Assembler::ASR,
11552               $src3$$constant & 0x1f);
11553   %}
11554 
11555   ins_pipe(ialu_reg_reg_shift);
11556 %}
11557 
11558 // This pattern is automatically generated from aarch64_ad.m4
11559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11560 // val ^ (-1 ^ (val >> shift)) ==> eon
11561 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11562                          iRegL src1, iRegL src2,
11563                          immI src3, immL_M1 src4) %{
11564   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11565   ins_cost(1.9 * INSN_COST);
11566   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11567 
11568   ins_encode %{
11569     __ eon(as_Register($dst$$reg),
11570               as_Register($src1$$reg),
11571               as_Register($src2$$reg),
11572               Assembler::ASR,
11573               $src3$$constant & 0x3f);
11574   %}
11575 
11576   ins_pipe(ialu_reg_reg_shift);
11577 %}
11578 
11579 // This pattern is automatically generated from aarch64_ad.m4
11580 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11581 // val ^ (-1 ^ (val ror shift)) ==> eonw
11582 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11583                          iRegIorL2I src1, iRegIorL2I src2,
11584                          immI src3, immI_M1 src4) %{
11585   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11586   ins_cost(1.9 * INSN_COST);
11587   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11588 
11589   ins_encode %{
11590     __ eonw(as_Register($dst$$reg),
11591               as_Register($src1$$reg),
11592               as_Register($src2$$reg),
11593               Assembler::ROR,
11594               $src3$$constant & 0x1f);
11595   %}
11596 
11597   ins_pipe(ialu_reg_reg_shift);
11598 %}
11599 
11600 // This pattern is automatically generated from aarch64_ad.m4
11601 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11602 // val ^ (-1 ^ (val ror shift)) ==> eon
11603 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11604                          iRegL src1, iRegL src2,
11605                          immI src3, immL_M1 src4) %{
11606   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11607   ins_cost(1.9 * INSN_COST);
11608   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11609 
11610   ins_encode %{
11611     __ eon(as_Register($dst$$reg),
11612               as_Register($src1$$reg),
11613               as_Register($src2$$reg),
11614               Assembler::ROR,
11615               $src3$$constant & 0x3f);
11616   %}
11617 
11618   ins_pipe(ialu_reg_reg_shift);
11619 %}
11620 
11621 // This pattern is automatically generated from aarch64_ad.m4
11622 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11623 // val ^ (-1 ^ (val << shift)) ==> eonw
11624 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11625                          iRegIorL2I src1, iRegIorL2I src2,
11626                          immI src3, immI_M1 src4) %{
11627   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11628   ins_cost(1.9 * INSN_COST);
11629   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11630 
11631   ins_encode %{
11632     __ eonw(as_Register($dst$$reg),
11633               as_Register($src1$$reg),
11634               as_Register($src2$$reg),
11635               Assembler::LSL,
11636               $src3$$constant & 0x1f);
11637   %}
11638 
11639   ins_pipe(ialu_reg_reg_shift);
11640 %}
11641 
11642 // This pattern is automatically generated from aarch64_ad.m4
11643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11644 // val ^ (-1 ^ (val << shift)) ==> eon
11645 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11646                          iRegL src1, iRegL src2,
11647                          immI src3, immL_M1 src4) %{
11648   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11649   ins_cost(1.9 * INSN_COST);
11650   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11651 
11652   ins_encode %{
11653     __ eon(as_Register($dst$$reg),
11654               as_Register($src1$$reg),
11655               as_Register($src2$$reg),
11656               Assembler::LSL,
11657               $src3$$constant & 0x3f);
11658   %}
11659 
11660   ins_pipe(ialu_reg_reg_shift);
11661 %}
11662 
11663 // This pattern is automatically generated from aarch64_ad.m4
11664 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11665 // val | (-1 ^ (val >>> shift)) ==> ornw
11666 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11667                          iRegIorL2I src1, iRegIorL2I src2,
11668                          immI src3, immI_M1 src4) %{
11669   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11670   ins_cost(1.9 * INSN_COST);
11671   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11672 
11673   ins_encode %{
11674     __ ornw(as_Register($dst$$reg),
11675               as_Register($src1$$reg),
11676               as_Register($src2$$reg),
11677               Assembler::LSR,
11678               $src3$$constant & 0x1f);
11679   %}
11680 
11681   ins_pipe(ialu_reg_reg_shift);
11682 %}
11683 
11684 // This pattern is automatically generated from aarch64_ad.m4
11685 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11686 // val | (-1 ^ (val >>> shift)) ==> orn
11687 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11688                          iRegL src1, iRegL src2,
11689                          immI src3, immL_M1 src4) %{
11690   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11691   ins_cost(1.9 * INSN_COST);
11692   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11693 
11694   ins_encode %{
11695     __ orn(as_Register($dst$$reg),
11696               as_Register($src1$$reg),
11697               as_Register($src2$$reg),
11698               Assembler::LSR,
11699               $src3$$constant & 0x3f);
11700   %}
11701 
11702   ins_pipe(ialu_reg_reg_shift);
11703 %}
11704 
11705 // This pattern is automatically generated from aarch64_ad.m4
11706 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11707 // val | (-1 ^ (val >> shift)) ==> ornw
11708 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11709                          iRegIorL2I src1, iRegIorL2I src2,
11710                          immI src3, immI_M1 src4) %{
11711   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11712   ins_cost(1.9 * INSN_COST);
11713   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11714 
11715   ins_encode %{
11716     __ ornw(as_Register($dst$$reg),
11717               as_Register($src1$$reg),
11718               as_Register($src2$$reg),
11719               Assembler::ASR,
11720               $src3$$constant & 0x1f);
11721   %}
11722 
11723   ins_pipe(ialu_reg_reg_shift);
11724 %}
11725 
11726 // This pattern is automatically generated from aarch64_ad.m4
11727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11728 // val | (-1 ^ (val >> shift)) ==> orn
11729 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11730                          iRegL src1, iRegL src2,
11731                          immI src3, immL_M1 src4) %{
11732   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11733   ins_cost(1.9 * INSN_COST);
11734   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11735 
11736   ins_encode %{
11737     __ orn(as_Register($dst$$reg),
11738               as_Register($src1$$reg),
11739               as_Register($src2$$reg),
11740               Assembler::ASR,
11741               $src3$$constant & 0x3f);
11742   %}
11743 
11744   ins_pipe(ialu_reg_reg_shift);
11745 %}
11746 
11747 // This pattern is automatically generated from aarch64_ad.m4
11748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11749 // val | (-1 ^ (val ror shift)) ==> ornw
11750 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11751                          iRegIorL2I src1, iRegIorL2I src2,
11752                          immI src3, immI_M1 src4) %{
11753   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11754   ins_cost(1.9 * INSN_COST);
11755   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11756 
11757   ins_encode %{
11758     __ ornw(as_Register($dst$$reg),
11759               as_Register($src1$$reg),
11760               as_Register($src2$$reg),
11761               Assembler::ROR,
11762               $src3$$constant & 0x1f);
11763   %}
11764 
11765   ins_pipe(ialu_reg_reg_shift);
11766 %}
11767 
11768 // This pattern is automatically generated from aarch64_ad.m4
11769 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11770 // val | (-1 ^ (val ror shift)) ==> orn
11771 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11772                          iRegL src1, iRegL src2,
11773                          immI src3, immL_M1 src4) %{
11774   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11775   ins_cost(1.9 * INSN_COST);
11776   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11777 
11778   ins_encode %{
11779     __ orn(as_Register($dst$$reg),
11780               as_Register($src1$$reg),
11781               as_Register($src2$$reg),
11782               Assembler::ROR,
11783               $src3$$constant & 0x3f);
11784   %}
11785 
11786   ins_pipe(ialu_reg_reg_shift);
11787 %}
11788 
11789 // This pattern is automatically generated from aarch64_ad.m4
11790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11791 // val | (-1 ^ (val << shift)) ==> ornw
11792 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11793                          iRegIorL2I src1, iRegIorL2I src2,
11794                          immI src3, immI_M1 src4) %{
11795   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11796   ins_cost(1.9 * INSN_COST);
11797   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11798 
11799   ins_encode %{
11800     __ ornw(as_Register($dst$$reg),
11801               as_Register($src1$$reg),
11802               as_Register($src2$$reg),
11803               Assembler::LSL,
11804               $src3$$constant & 0x1f);
11805   %}
11806 
11807   ins_pipe(ialu_reg_reg_shift);
11808 %}
11809 
11810 // This pattern is automatically generated from aarch64_ad.m4
11811 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11812 // val | (-1 ^ (val << shift)) ==> orn
11813 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11814                          iRegL src1, iRegL src2,
11815                          immI src3, immL_M1 src4) %{
11816   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11817   ins_cost(1.9 * INSN_COST);
11818   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11819 
11820   ins_encode %{
11821     __ orn(as_Register($dst$$reg),
11822               as_Register($src1$$reg),
11823               as_Register($src2$$reg),
11824               Assembler::LSL,
11825               $src3$$constant & 0x3f);
11826   %}
11827 
11828   ins_pipe(ialu_reg_reg_shift);
11829 %}
11830 
11831 // This pattern is automatically generated from aarch64_ad.m4
11832 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11833 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11834                          iRegIorL2I src1, iRegIorL2I src2,
11835                          immI src3) %{
11836   match(Set dst (AndI src1 (URShiftI src2 src3)));
11837 
11838   ins_cost(1.9 * INSN_COST);
11839   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
11840 
11841   ins_encode %{
11842     __ andw(as_Register($dst$$reg),
11843               as_Register($src1$$reg),
11844               as_Register($src2$$reg),
11845               Assembler::LSR,
11846               $src3$$constant & 0x1f);
11847   %}
11848 
11849   ins_pipe(ialu_reg_reg_shift);
11850 %}
11851 
11852 // This pattern is automatically generated from aarch64_ad.m4
11853 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11854 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11855                          iRegL src1, iRegL src2,
11856                          immI src3) %{
11857   match(Set dst (AndL src1 (URShiftL src2 src3)));
11858 
11859   ins_cost(1.9 * INSN_COST);
11860   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
11861 
11862   ins_encode %{
11863     __ andr(as_Register($dst$$reg),
11864               as_Register($src1$$reg),
11865               as_Register($src2$$reg),
11866               Assembler::LSR,
11867               $src3$$constant & 0x3f);
11868   %}
11869 
11870   ins_pipe(ialu_reg_reg_shift);
11871 %}
11872 
11873 // This pattern is automatically generated from aarch64_ad.m4
11874 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11875 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11876                          iRegIorL2I src1, iRegIorL2I src2,
11877                          immI src3) %{
11878   match(Set dst (AndI src1 (RShiftI src2 src3)));
11879 
11880   ins_cost(1.9 * INSN_COST);
11881   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
11882 
11883   ins_encode %{
11884     __ andw(as_Register($dst$$reg),
11885               as_Register($src1$$reg),
11886               as_Register($src2$$reg),
11887               Assembler::ASR,
11888               $src3$$constant & 0x1f);
11889   %}
11890 
11891   ins_pipe(ialu_reg_reg_shift);
11892 %}
11893 
11894 // This pattern is automatically generated from aarch64_ad.m4
11895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11896 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11897                          iRegL src1, iRegL src2,
11898                          immI src3) %{
11899   match(Set dst (AndL src1 (RShiftL src2 src3)));
11900 
11901   ins_cost(1.9 * INSN_COST);
11902   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
11903 
11904   ins_encode %{
11905     __ andr(as_Register($dst$$reg),
11906               as_Register($src1$$reg),
11907               as_Register($src2$$reg),
11908               Assembler::ASR,
11909               $src3$$constant & 0x3f);
11910   %}
11911 
11912   ins_pipe(ialu_reg_reg_shift);
11913 %}
11914 
11915 // This pattern is automatically generated from aarch64_ad.m4
11916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11917 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11918                          iRegIorL2I src1, iRegIorL2I src2,
11919                          immI src3) %{
11920   match(Set dst (AndI src1 (LShiftI src2 src3)));
11921 
11922   ins_cost(1.9 * INSN_COST);
11923   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
11924 
11925   ins_encode %{
11926     __ andw(as_Register($dst$$reg),
11927               as_Register($src1$$reg),
11928               as_Register($src2$$reg),
11929               Assembler::LSL,
11930               $src3$$constant & 0x1f);
11931   %}
11932 
11933   ins_pipe(ialu_reg_reg_shift);
11934 %}
11935 
11936 // This pattern is automatically generated from aarch64_ad.m4
11937 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11938 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11939                          iRegL src1, iRegL src2,
11940                          immI src3) %{
11941   match(Set dst (AndL src1 (LShiftL src2 src3)));
11942 
11943   ins_cost(1.9 * INSN_COST);
11944   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
11945 
11946   ins_encode %{
11947     __ andr(as_Register($dst$$reg),
11948               as_Register($src1$$reg),
11949               as_Register($src2$$reg),
11950               Assembler::LSL,
11951               $src3$$constant & 0x3f);
11952   %}
11953 
11954   ins_pipe(ialu_reg_reg_shift);
11955 %}
11956 
11957 // This pattern is automatically generated from aarch64_ad.m4
11958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11959 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11960                          iRegIorL2I src1, iRegIorL2I src2,
11961                          immI src3) %{
11962   match(Set dst (AndI src1 (RotateRight src2 src3)));
11963 
11964   ins_cost(1.9 * INSN_COST);
11965   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
11966 
11967   ins_encode %{
11968     __ andw(as_Register($dst$$reg),
11969               as_Register($src1$$reg),
11970               as_Register($src2$$reg),
11971               Assembler::ROR,
11972               $src3$$constant & 0x1f);
11973   %}
11974 
11975   ins_pipe(ialu_reg_reg_shift);
11976 %}
11977 
11978 // This pattern is automatically generated from aarch64_ad.m4
11979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11980 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11981                          iRegL src1, iRegL src2,
11982                          immI src3) %{
11983   match(Set dst (AndL src1 (RotateRight src2 src3)));
11984 
11985   ins_cost(1.9 * INSN_COST);
11986   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
11987 
11988   ins_encode %{
11989     __ andr(as_Register($dst$$reg),
11990               as_Register($src1$$reg),
11991               as_Register($src2$$reg),
11992               Assembler::ROR,
11993               $src3$$constant & 0x3f);
11994   %}
11995 
11996   ins_pipe(ialu_reg_reg_shift);
11997 %}
11998 
11999 // This pattern is automatically generated from aarch64_ad.m4
12000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12001 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12002                          iRegIorL2I src1, iRegIorL2I src2,
12003                          immI src3) %{
12004   match(Set dst (XorI src1 (URShiftI src2 src3)));
12005 
12006   ins_cost(1.9 * INSN_COST);
12007   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12008 
12009   ins_encode %{
12010     __ eorw(as_Register($dst$$reg),
12011               as_Register($src1$$reg),
12012               as_Register($src2$$reg),
12013               Assembler::LSR,
12014               $src3$$constant & 0x1f);
12015   %}
12016 
12017   ins_pipe(ialu_reg_reg_shift);
12018 %}
12019 
12020 // This pattern is automatically generated from aarch64_ad.m4
12021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12022 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12023                          iRegL src1, iRegL src2,
12024                          immI src3) %{
12025   match(Set dst (XorL src1 (URShiftL src2 src3)));
12026 
12027   ins_cost(1.9 * INSN_COST);
12028   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12029 
12030   ins_encode %{
12031     __ eor(as_Register($dst$$reg),
12032               as_Register($src1$$reg),
12033               as_Register($src2$$reg),
12034               Assembler::LSR,
12035               $src3$$constant & 0x3f);
12036   %}
12037 
12038   ins_pipe(ialu_reg_reg_shift);
12039 %}
12040 
12041 // This pattern is automatically generated from aarch64_ad.m4
12042 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12043 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12044                          iRegIorL2I src1, iRegIorL2I src2,
12045                          immI src3) %{
12046   match(Set dst (XorI src1 (RShiftI src2 src3)));
12047 
12048   ins_cost(1.9 * INSN_COST);
12049   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12050 
12051   ins_encode %{
12052     __ eorw(as_Register($dst$$reg),
12053               as_Register($src1$$reg),
12054               as_Register($src2$$reg),
12055               Assembler::ASR,
12056               $src3$$constant & 0x1f);
12057   %}
12058 
12059   ins_pipe(ialu_reg_reg_shift);
12060 %}
12061 
12062 // This pattern is automatically generated from aarch64_ad.m4
12063 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12064 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12065                          iRegL src1, iRegL src2,
12066                          immI src3) %{
12067   match(Set dst (XorL src1 (RShiftL src2 src3)));
12068 
12069   ins_cost(1.9 * INSN_COST);
12070   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12071 
12072   ins_encode %{
12073     __ eor(as_Register($dst$$reg),
12074               as_Register($src1$$reg),
12075               as_Register($src2$$reg),
12076               Assembler::ASR,
12077               $src3$$constant & 0x3f);
12078   %}
12079 
12080   ins_pipe(ialu_reg_reg_shift);
12081 %}
12082 
12083 // This pattern is automatically generated from aarch64_ad.m4
12084 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12085 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12086                          iRegIorL2I src1, iRegIorL2I src2,
12087                          immI src3) %{
12088   match(Set dst (XorI src1 (LShiftI src2 src3)));
12089 
12090   ins_cost(1.9 * INSN_COST);
12091   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12092 
12093   ins_encode %{
12094     __ eorw(as_Register($dst$$reg),
12095               as_Register($src1$$reg),
12096               as_Register($src2$$reg),
12097               Assembler::LSL,
12098               $src3$$constant & 0x1f);
12099   %}
12100 
12101   ins_pipe(ialu_reg_reg_shift);
12102 %}
12103 
12104 // This pattern is automatically generated from aarch64_ad.m4
12105 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12106 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12107                          iRegL src1, iRegL src2,
12108                          immI src3) %{
12109   match(Set dst (XorL src1 (LShiftL src2 src3)));
12110 
12111   ins_cost(1.9 * INSN_COST);
12112   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12113 
12114   ins_encode %{
12115     __ eor(as_Register($dst$$reg),
12116               as_Register($src1$$reg),
12117               as_Register($src2$$reg),
12118               Assembler::LSL,
12119               $src3$$constant & 0x3f);
12120   %}
12121 
12122   ins_pipe(ialu_reg_reg_shift);
12123 %}
12124 
12125 // This pattern is automatically generated from aarch64_ad.m4
12126 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12127 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12128                          iRegIorL2I src1, iRegIorL2I src2,
12129                          immI src3) %{
12130   match(Set dst (XorI src1 (RotateRight src2 src3)));
12131 
12132   ins_cost(1.9 * INSN_COST);
12133   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12134 
12135   ins_encode %{
12136     __ eorw(as_Register($dst$$reg),
12137               as_Register($src1$$reg),
12138               as_Register($src2$$reg),
12139               Assembler::ROR,
12140               $src3$$constant & 0x1f);
12141   %}
12142 
12143   ins_pipe(ialu_reg_reg_shift);
12144 %}
12145 
12146 // This pattern is automatically generated from aarch64_ad.m4
12147 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12148 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12149                          iRegL src1, iRegL src2,
12150                          immI src3) %{
12151   match(Set dst (XorL src1 (RotateRight src2 src3)));
12152 
12153   ins_cost(1.9 * INSN_COST);
12154   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12155 
12156   ins_encode %{
12157     __ eor(as_Register($dst$$reg),
12158               as_Register($src1$$reg),
12159               as_Register($src2$$reg),
12160               Assembler::ROR,
12161               $src3$$constant & 0x3f);
12162   %}
12163 
12164   ins_pipe(ialu_reg_reg_shift);
12165 %}
12166 
12167 // This pattern is automatically generated from aarch64_ad.m4
12168 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12169 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12170                          iRegIorL2I src1, iRegIorL2I src2,
12171                          immI src3) %{
12172   match(Set dst (OrI src1 (URShiftI src2 src3)));
12173 
12174   ins_cost(1.9 * INSN_COST);
12175   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12176 
12177   ins_encode %{
12178     __ orrw(as_Register($dst$$reg),
12179               as_Register($src1$$reg),
12180               as_Register($src2$$reg),
12181               Assembler::LSR,
12182               $src3$$constant & 0x1f);
12183   %}
12184 
12185   ins_pipe(ialu_reg_reg_shift);
12186 %}
12187 
12188 // This pattern is automatically generated from aarch64_ad.m4
12189 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12190 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12191                          iRegL src1, iRegL src2,
12192                          immI src3) %{
12193   match(Set dst (OrL src1 (URShiftL src2 src3)));
12194 
12195   ins_cost(1.9 * INSN_COST);
12196   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12197 
12198   ins_encode %{
12199     __ orr(as_Register($dst$$reg),
12200               as_Register($src1$$reg),
12201               as_Register($src2$$reg),
12202               Assembler::LSR,
12203               $src3$$constant & 0x3f);
12204   %}
12205 
12206   ins_pipe(ialu_reg_reg_shift);
12207 %}
12208 
12209 // This pattern is automatically generated from aarch64_ad.m4
12210 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12211 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12212                          iRegIorL2I src1, iRegIorL2I src2,
12213                          immI src3) %{
12214   match(Set dst (OrI src1 (RShiftI src2 src3)));
12215 
12216   ins_cost(1.9 * INSN_COST);
12217   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12218 
12219   ins_encode %{
12220     __ orrw(as_Register($dst$$reg),
12221               as_Register($src1$$reg),
12222               as_Register($src2$$reg),
12223               Assembler::ASR,
12224               $src3$$constant & 0x1f);
12225   %}
12226 
12227   ins_pipe(ialu_reg_reg_shift);
12228 %}
12229 
12230 // This pattern is automatically generated from aarch64_ad.m4
12231 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12232 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12233                          iRegL src1, iRegL src2,
12234                          immI src3) %{
12235   match(Set dst (OrL src1 (RShiftL src2 src3)));
12236 
12237   ins_cost(1.9 * INSN_COST);
12238   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12239 
12240   ins_encode %{
12241     __ orr(as_Register($dst$$reg),
12242               as_Register($src1$$reg),
12243               as_Register($src2$$reg),
12244               Assembler::ASR,
12245               $src3$$constant & 0x3f);
12246   %}
12247 
12248   ins_pipe(ialu_reg_reg_shift);
12249 %}
12250 
12251 // This pattern is automatically generated from aarch64_ad.m4
12252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12253 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12254                          iRegIorL2I src1, iRegIorL2I src2,
12255                          immI src3) %{
12256   match(Set dst (OrI src1 (LShiftI src2 src3)));
12257 
12258   ins_cost(1.9 * INSN_COST);
12259   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12260 
12261   ins_encode %{
12262     __ orrw(as_Register($dst$$reg),
12263               as_Register($src1$$reg),
12264               as_Register($src2$$reg),
12265               Assembler::LSL,
12266               $src3$$constant & 0x1f);
12267   %}
12268 
12269   ins_pipe(ialu_reg_reg_shift);
12270 %}
12271 
12272 // This pattern is automatically generated from aarch64_ad.m4
12273 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12274 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12275                          iRegL src1, iRegL src2,
12276                          immI src3) %{
12277   match(Set dst (OrL src1 (LShiftL src2 src3)));
12278 
12279   ins_cost(1.9 * INSN_COST);
12280   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12281 
12282   ins_encode %{
12283     __ orr(as_Register($dst$$reg),
12284               as_Register($src1$$reg),
12285               as_Register($src2$$reg),
12286               Assembler::LSL,
12287               $src3$$constant & 0x3f);
12288   %}
12289 
12290   ins_pipe(ialu_reg_reg_shift);
12291 %}
12292 
12293 // This pattern is automatically generated from aarch64_ad.m4
12294 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12295 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12296                          iRegIorL2I src1, iRegIorL2I src2,
12297                          immI src3) %{
12298   match(Set dst (OrI src1 (RotateRight src2 src3)));
12299 
12300   ins_cost(1.9 * INSN_COST);
12301   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12302 
12303   ins_encode %{
12304     __ orrw(as_Register($dst$$reg),
12305               as_Register($src1$$reg),
12306               as_Register($src2$$reg),
12307               Assembler::ROR,
12308               $src3$$constant & 0x1f);
12309   %}
12310 
12311   ins_pipe(ialu_reg_reg_shift);
12312 %}
12313 
12314 // This pattern is automatically generated from aarch64_ad.m4
12315 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12316 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12317                          iRegL src1, iRegL src2,
12318                          immI src3) %{
12319   match(Set dst (OrL src1 (RotateRight src2 src3)));
12320 
12321   ins_cost(1.9 * INSN_COST);
12322   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12323 
12324   ins_encode %{
12325     __ orr(as_Register($dst$$reg),
12326               as_Register($src1$$reg),
12327               as_Register($src2$$reg),
12328               Assembler::ROR,
12329               $src3$$constant & 0x3f);
12330   %}
12331 
12332   ins_pipe(ialu_reg_reg_shift);
12333 %}
12334 
12335 // This pattern is automatically generated from aarch64_ad.m4
12336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12337 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12338                          iRegIorL2I src1, iRegIorL2I src2,
12339                          immI src3) %{
12340   match(Set dst (AddI src1 (URShiftI src2 src3)));
12341 
12342   ins_cost(1.9 * INSN_COST);
12343   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12344 
12345   ins_encode %{
12346     __ addw(as_Register($dst$$reg),
12347               as_Register($src1$$reg),
12348               as_Register($src2$$reg),
12349               Assembler::LSR,
12350               $src3$$constant & 0x1f);
12351   %}
12352 
12353   ins_pipe(ialu_reg_reg_shift);
12354 %}
12355 
12356 // This pattern is automatically generated from aarch64_ad.m4
12357 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12358 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12359                          iRegL src1, iRegL src2,
12360                          immI src3) %{
12361   match(Set dst (AddL src1 (URShiftL src2 src3)));
12362 
12363   ins_cost(1.9 * INSN_COST);
12364   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12365 
12366   ins_encode %{
12367     __ add(as_Register($dst$$reg),
12368               as_Register($src1$$reg),
12369               as_Register($src2$$reg),
12370               Assembler::LSR,
12371               $src3$$constant & 0x3f);
12372   %}
12373 
12374   ins_pipe(ialu_reg_reg_shift);
12375 %}
12376 
12377 // This pattern is automatically generated from aarch64_ad.m4
12378 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12379 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12380                          iRegIorL2I src1, iRegIorL2I src2,
12381                          immI src3) %{
12382   match(Set dst (AddI src1 (RShiftI src2 src3)));
12383 
12384   ins_cost(1.9 * INSN_COST);
12385   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12386 
12387   ins_encode %{
12388     __ addw(as_Register($dst$$reg),
12389               as_Register($src1$$reg),
12390               as_Register($src2$$reg),
12391               Assembler::ASR,
12392               $src3$$constant & 0x1f);
12393   %}
12394 
12395   ins_pipe(ialu_reg_reg_shift);
12396 %}
12397 
12398 // This pattern is automatically generated from aarch64_ad.m4
12399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12400 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12401                          iRegL src1, iRegL src2,
12402                          immI src3) %{
12403   match(Set dst (AddL src1 (RShiftL src2 src3)));
12404 
12405   ins_cost(1.9 * INSN_COST);
12406   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12407 
12408   ins_encode %{
12409     __ add(as_Register($dst$$reg),
12410               as_Register($src1$$reg),
12411               as_Register($src2$$reg),
12412               Assembler::ASR,
12413               $src3$$constant & 0x3f);
12414   %}
12415 
12416   ins_pipe(ialu_reg_reg_shift);
12417 %}
12418 
12419 // This pattern is automatically generated from aarch64_ad.m4
12420 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12421 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12422                          iRegIorL2I src1, iRegIorL2I src2,
12423                          immI src3) %{
12424   match(Set dst (AddI src1 (LShiftI src2 src3)));
12425 
12426   ins_cost(1.9 * INSN_COST);
12427   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12428 
12429   ins_encode %{
12430     __ addw(as_Register($dst$$reg),
12431               as_Register($src1$$reg),
12432               as_Register($src2$$reg),
12433               Assembler::LSL,
12434               $src3$$constant & 0x1f);
12435   %}
12436 
12437   ins_pipe(ialu_reg_reg_shift);
12438 %}
12439 
12440 // This pattern is automatically generated from aarch64_ad.m4
12441 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12442 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12443                          iRegL src1, iRegL src2,
12444                          immI src3) %{
12445   match(Set dst (AddL src1 (LShiftL src2 src3)));
12446 
12447   ins_cost(1.9 * INSN_COST);
12448   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12449 
12450   ins_encode %{
12451     __ add(as_Register($dst$$reg),
12452               as_Register($src1$$reg),
12453               as_Register($src2$$reg),
12454               Assembler::LSL,
12455               $src3$$constant & 0x3f);
12456   %}
12457 
12458   ins_pipe(ialu_reg_reg_shift);
12459 %}
12460 
12461 // This pattern is automatically generated from aarch64_ad.m4
12462 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12463 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12464                          iRegIorL2I src1, iRegIorL2I src2,
12465                          immI src3) %{
12466   match(Set dst (SubI src1 (URShiftI src2 src3)));
12467 
12468   ins_cost(1.9 * INSN_COST);
12469   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12470 
12471   ins_encode %{
12472     __ subw(as_Register($dst$$reg),
12473               as_Register($src1$$reg),
12474               as_Register($src2$$reg),
12475               Assembler::LSR,
12476               $src3$$constant & 0x1f);
12477   %}
12478 
12479   ins_pipe(ialu_reg_reg_shift);
12480 %}
12481 
12482 // This pattern is automatically generated from aarch64_ad.m4
12483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12484 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12485                          iRegL src1, iRegL src2,
12486                          immI src3) %{
12487   match(Set dst (SubL src1 (URShiftL src2 src3)));
12488 
12489   ins_cost(1.9 * INSN_COST);
12490   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12491 
12492   ins_encode %{
12493     __ sub(as_Register($dst$$reg),
12494               as_Register($src1$$reg),
12495               as_Register($src2$$reg),
12496               Assembler::LSR,
12497               $src3$$constant & 0x3f);
12498   %}
12499 
12500   ins_pipe(ialu_reg_reg_shift);
12501 %}
12502 
12503 // This pattern is automatically generated from aarch64_ad.m4
12504 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12505 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12506                          iRegIorL2I src1, iRegIorL2I src2,
12507                          immI src3) %{
12508   match(Set dst (SubI src1 (RShiftI src2 src3)));
12509 
12510   ins_cost(1.9 * INSN_COST);
12511   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12512 
12513   ins_encode %{
12514     __ subw(as_Register($dst$$reg),
12515               as_Register($src1$$reg),
12516               as_Register($src2$$reg),
12517               Assembler::ASR,
12518               $src3$$constant & 0x1f);
12519   %}
12520 
12521   ins_pipe(ialu_reg_reg_shift);
12522 %}
12523 
12524 // This pattern is automatically generated from aarch64_ad.m4
12525 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12526 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12527                          iRegL src1, iRegL src2,
12528                          immI src3) %{
12529   match(Set dst (SubL src1 (RShiftL src2 src3)));
12530 
12531   ins_cost(1.9 * INSN_COST);
12532   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12533 
12534   ins_encode %{
12535     __ sub(as_Register($dst$$reg),
12536               as_Register($src1$$reg),
12537               as_Register($src2$$reg),
12538               Assembler::ASR,
12539               $src3$$constant & 0x3f);
12540   %}
12541 
12542   ins_pipe(ialu_reg_reg_shift);
12543 %}
12544 
12545 // This pattern is automatically generated from aarch64_ad.m4
12546 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12547 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12548                          iRegIorL2I src1, iRegIorL2I src2,
12549                          immI src3) %{
12550   match(Set dst (SubI src1 (LShiftI src2 src3)));
12551 
12552   ins_cost(1.9 * INSN_COST);
12553   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12554 
12555   ins_encode %{
12556     __ subw(as_Register($dst$$reg),
12557               as_Register($src1$$reg),
12558               as_Register($src2$$reg),
12559               Assembler::LSL,
12560               $src3$$constant & 0x1f);
12561   %}
12562 
12563   ins_pipe(ialu_reg_reg_shift);
12564 %}
12565 
12566 // This pattern is automatically generated from aarch64_ad.m4
12567 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12568 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12569                          iRegL src1, iRegL src2,
12570                          immI src3) %{
12571   match(Set dst (SubL src1 (LShiftL src2 src3)));
12572 
12573   ins_cost(1.9 * INSN_COST);
12574   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12575 
12576   ins_encode %{
12577     __ sub(as_Register($dst$$reg),
12578               as_Register($src1$$reg),
12579               as_Register($src2$$reg),
12580               Assembler::LSL,
12581               $src3$$constant & 0x3f);
12582   %}
12583 
12584   ins_pipe(ialu_reg_reg_shift);
12585 %}
12586 
12587 // This pattern is automatically generated from aarch64_ad.m4
12588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12589 
12590 // Shift Left followed by Shift Right.
12591 // This idiom is used by the compiler for the i2b bytecode etc.
12592 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12593 %{
12594   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12595   ins_cost(INSN_COST * 2);
12596   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12597   ins_encode %{
12598     int lshift = $lshift_count$$constant & 63;
12599     int rshift = $rshift_count$$constant & 63;
12600     int s = 63 - lshift;
12601     int r = (rshift - lshift) & 63;
12602     __ sbfm(as_Register($dst$$reg),
12603             as_Register($src$$reg),
12604             r, s);
12605   %}
12606 
12607   ins_pipe(ialu_reg_shift);
12608 %}
12609 
12610 // This pattern is automatically generated from aarch64_ad.m4
12611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12612 
12613 // Shift Left followed by Shift Right.
12614 // This idiom is used by the compiler for the i2b bytecode etc.
12615 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12616 %{
12617   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12618   ins_cost(INSN_COST * 2);
12619   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12620   ins_encode %{
12621     int lshift = $lshift_count$$constant & 31;
12622     int rshift = $rshift_count$$constant & 31;
12623     int s = 31 - lshift;
12624     int r = (rshift - lshift) & 31;
12625     __ sbfmw(as_Register($dst$$reg),
12626             as_Register($src$$reg),
12627             r, s);
12628   %}
12629 
12630   ins_pipe(ialu_reg_shift);
12631 %}
12632 
12633 // This pattern is automatically generated from aarch64_ad.m4
12634 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12635 
12636 // Shift Left followed by Shift Right.
12637 // This idiom is used by the compiler for the i2b bytecode etc.
12638 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12639 %{
12640   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12641   ins_cost(INSN_COST * 2);
12642   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12643   ins_encode %{
12644     int lshift = $lshift_count$$constant & 63;
12645     int rshift = $rshift_count$$constant & 63;
12646     int s = 63 - lshift;
12647     int r = (rshift - lshift) & 63;
12648     __ ubfm(as_Register($dst$$reg),
12649             as_Register($src$$reg),
12650             r, s);
12651   %}
12652 
12653   ins_pipe(ialu_reg_shift);
12654 %}
12655 
12656 // This pattern is automatically generated from aarch64_ad.m4
12657 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12658 
12659 // Shift Left followed by Shift Right.
12660 // This idiom is used by the compiler for the i2b bytecode etc.
12661 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12662 %{
12663   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12664   ins_cost(INSN_COST * 2);
12665   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12666   ins_encode %{
12667     int lshift = $lshift_count$$constant & 31;
12668     int rshift = $rshift_count$$constant & 31;
12669     int s = 31 - lshift;
12670     int r = (rshift - lshift) & 31;
12671     __ ubfmw(as_Register($dst$$reg),
12672             as_Register($src$$reg),
12673             r, s);
12674   %}
12675 
12676   ins_pipe(ialu_reg_shift);
12677 %}
12678 
12679 // Bitfield extract with shift & mask
12680 
12681 // This pattern is automatically generated from aarch64_ad.m4
12682 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12683 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12684 %{
12685   match(Set dst (AndI (URShiftI src rshift) mask));
12686   // Make sure we are not going to exceed what ubfxw can do.
12687   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12688 
12689   ins_cost(INSN_COST);
12690   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12691   ins_encode %{
12692     int rshift = $rshift$$constant & 31;
12693     intptr_t mask = $mask$$constant;
12694     int width = exact_log2(mask+1);
12695     __ ubfxw(as_Register($dst$$reg),
12696             as_Register($src$$reg), rshift, width);
12697   %}
12698   ins_pipe(ialu_reg_shift);
12699 %}
12700 
12701 // This pattern is automatically generated from aarch64_ad.m4
12702 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12703 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12704 %{
12705   match(Set dst (AndL (URShiftL src rshift) mask));
12706   // Make sure we are not going to exceed what ubfx can do.
12707   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12708 
12709   ins_cost(INSN_COST);
12710   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12711   ins_encode %{
12712     int rshift = $rshift$$constant & 63;
12713     intptr_t mask = $mask$$constant;
12714     int width = exact_log2_long(mask+1);
12715     __ ubfx(as_Register($dst$$reg),
12716             as_Register($src$$reg), rshift, width);
12717   %}
12718   ins_pipe(ialu_reg_shift);
12719 %}
12720 
12721 
12722 // This pattern is automatically generated from aarch64_ad.m4
12723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12724 
12725 // We can use ubfx when extending an And with a mask when we know mask
12726 // is positive.  We know that because immI_bitmask guarantees it.
12727 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12728 %{
12729   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12730   // Make sure we are not going to exceed what ubfxw can do.
12731   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12732 
12733   ins_cost(INSN_COST * 2);
12734   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12735   ins_encode %{
12736     int rshift = $rshift$$constant & 31;
12737     intptr_t mask = $mask$$constant;
12738     int width = exact_log2(mask+1);
12739     __ ubfx(as_Register($dst$$reg),
12740             as_Register($src$$reg), rshift, width);
12741   %}
12742   ins_pipe(ialu_reg_shift);
12743 %}
12744 
12745 
12746 // This pattern is automatically generated from aarch64_ad.m4
12747 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12748 
12749 // We can use ubfiz when masking by a positive number and then left shifting the result.
12750 // We know that the mask is positive because immI_bitmask guarantees it.
12751 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12752 %{
12753   match(Set dst (LShiftI (AndI src mask) lshift));
12754   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12755 
12756   ins_cost(INSN_COST);
12757   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12758   ins_encode %{
12759     int lshift = $lshift$$constant & 31;
12760     intptr_t mask = $mask$$constant;
12761     int width = exact_log2(mask+1);
12762     __ ubfizw(as_Register($dst$$reg),
12763           as_Register($src$$reg), lshift, width);
12764   %}
12765   ins_pipe(ialu_reg_shift);
12766 %}
12767 
12768 // This pattern is automatically generated from aarch64_ad.m4
12769 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12770 
12771 // We can use ubfiz when masking by a positive number and then left shifting the result.
12772 // We know that the mask is positive because immL_bitmask guarantees it.
12773 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12774 %{
12775   match(Set dst (LShiftL (AndL src mask) lshift));
12776   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12777 
12778   ins_cost(INSN_COST);
12779   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12780   ins_encode %{
12781     int lshift = $lshift$$constant & 63;
12782     intptr_t mask = $mask$$constant;
12783     int width = exact_log2_long(mask+1);
12784     __ ubfiz(as_Register($dst$$reg),
12785           as_Register($src$$reg), lshift, width);
12786   %}
12787   ins_pipe(ialu_reg_shift);
12788 %}
12789 
12790 // This pattern is automatically generated from aarch64_ad.m4
12791 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12792 
12793 // We can use ubfiz when masking by a positive number and then left shifting the result.
12794 // We know that the mask is positive because immI_bitmask guarantees it.
12795 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12796 %{
12797   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12798   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12799 
12800   ins_cost(INSN_COST);
12801   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12802   ins_encode %{
12803     int lshift = $lshift$$constant & 31;
12804     intptr_t mask = $mask$$constant;
12805     int width = exact_log2(mask+1);
12806     __ ubfizw(as_Register($dst$$reg),
12807           as_Register($src$$reg), lshift, width);
12808   %}
12809   ins_pipe(ialu_reg_shift);
12810 %}
12811 
12812 // This pattern is automatically generated from aarch64_ad.m4
12813 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12814 
12815 // We can use ubfiz when masking by a positive number and then left shifting the result.
12816 // We know that the mask is positive because immL_bitmask guarantees it.
12817 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12818 %{
12819   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12820   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12821 
12822   ins_cost(INSN_COST);
12823   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12824   ins_encode %{
12825     int lshift = $lshift$$constant & 63;
12826     intptr_t mask = $mask$$constant;
12827     int width = exact_log2_long(mask+1);
12828     __ ubfiz(as_Register($dst$$reg),
12829           as_Register($src$$reg), lshift, width);
12830   %}
12831   ins_pipe(ialu_reg_shift);
12832 %}
12833 
12834 
12835 // This pattern is automatically generated from aarch64_ad.m4
12836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12837 
12838 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12839 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12840 %{
12841   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12842   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12843 
12844   ins_cost(INSN_COST);
12845   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12846   ins_encode %{
12847     int lshift = $lshift$$constant & 63;
12848     intptr_t mask = $mask$$constant;
12849     int width = exact_log2(mask+1);
12850     __ ubfiz(as_Register($dst$$reg),
12851              as_Register($src$$reg), lshift, width);
12852   %}
12853   ins_pipe(ialu_reg_shift);
12854 %}
12855 
12856 // This pattern is automatically generated from aarch64_ad.m4
12857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12858 
12859 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12860 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12861 %{
12862   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12863   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12864 
12865   ins_cost(INSN_COST);
12866   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12867   ins_encode %{
12868     int lshift = $lshift$$constant & 31;
12869     intptr_t mask = $mask$$constant;
12870     int width = exact_log2(mask+1);
12871     __ ubfiz(as_Register($dst$$reg),
12872              as_Register($src$$reg), lshift, width);
12873   %}
12874   ins_pipe(ialu_reg_shift);
12875 %}
12876 
12877 // This pattern is automatically generated from aarch64_ad.m4
12878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12879 
12880 // Can skip int2long conversions after AND with small bitmask
12881 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12882 %{
12883   match(Set dst (ConvI2L (AndI src msk)));
12884   ins_cost(INSN_COST);
12885   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12886   ins_encode %{
12887     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12888   %}
12889   ins_pipe(ialu_reg_shift);
12890 %}
12891 
12892 
12893 // Rotations
12894 // This pattern is automatically generated from aarch64_ad.m4
12895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12896 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12897 %{
12898   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12899   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12900 
12901   ins_cost(INSN_COST);
12902   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12903 
12904   ins_encode %{
12905     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12906             $rshift$$constant & 63);
12907   %}
12908   ins_pipe(ialu_reg_reg_extr);
12909 %}
12910 
12911 
12912 // This pattern is automatically generated from aarch64_ad.m4
12913 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12914 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12915 %{
12916   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12917   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12918 
12919   ins_cost(INSN_COST);
12920   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12921 
12922   ins_encode %{
12923     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12924             $rshift$$constant & 31);
12925   %}
12926   ins_pipe(ialu_reg_reg_extr);
12927 %}
12928 
12929 
12930 // This pattern is automatically generated from aarch64_ad.m4
12931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12932 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12933 %{
12934   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12935   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12936 
12937   ins_cost(INSN_COST);
12938   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12939 
12940   ins_encode %{
12941     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12942             $rshift$$constant & 63);
12943   %}
12944   ins_pipe(ialu_reg_reg_extr);
12945 %}
12946 
12947 
12948 // This pattern is automatically generated from aarch64_ad.m4
12949 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12950 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12951 %{
12952   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12953   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12954 
12955   ins_cost(INSN_COST);
12956   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12957 
12958   ins_encode %{
12959     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12960             $rshift$$constant & 31);
12961   %}
12962   ins_pipe(ialu_reg_reg_extr);
12963 %}
12964 
12965 
12966 // This pattern is automatically generated from aarch64_ad.m4
12967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12968 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12969 %{
12970   match(Set dst (RotateRight src shift));
12971 
12972   ins_cost(INSN_COST);
12973   format %{ "ror    $dst, $src, $shift" %}
12974 
12975   ins_encode %{
12976      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12977                $shift$$constant & 0x1f);
12978   %}
12979   ins_pipe(ialu_reg_reg_vshift);
12980 %}
12981 
12982 // This pattern is automatically generated from aarch64_ad.m4
12983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12984 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12985 %{
12986   match(Set dst (RotateRight src shift));
12987 
12988   ins_cost(INSN_COST);
12989   format %{ "ror    $dst, $src, $shift" %}
12990 
12991   ins_encode %{
12992      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12993                $shift$$constant & 0x3f);
12994   %}
12995   ins_pipe(ialu_reg_reg_vshift);
12996 %}
12997 
12998 // This pattern is automatically generated from aarch64_ad.m4
12999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13000 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13001 %{
13002   match(Set dst (RotateRight src shift));
13003 
13004   ins_cost(INSN_COST);
13005   format %{ "ror    $dst, $src, $shift" %}
13006 
13007   ins_encode %{
13008      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13009   %}
13010   ins_pipe(ialu_reg_reg_vshift);
13011 %}
13012 
13013 // This pattern is automatically generated from aarch64_ad.m4
13014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13015 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13016 %{
13017   match(Set dst (RotateRight src shift));
13018 
13019   ins_cost(INSN_COST);
13020   format %{ "ror    $dst, $src, $shift" %}
13021 
13022   ins_encode %{
13023      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13024   %}
13025   ins_pipe(ialu_reg_reg_vshift);
13026 %}
13027 
13028 // This pattern is automatically generated from aarch64_ad.m4
13029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13030 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13031 %{
13032   match(Set dst (RotateLeft src shift));
13033 
13034   ins_cost(INSN_COST);
13035   format %{ "rol    $dst, $src, $shift" %}
13036 
13037   ins_encode %{
13038      __ subw(rscratch1, zr, as_Register($shift$$reg));
13039      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13040   %}
13041   ins_pipe(ialu_reg_reg_vshift);
13042 %}
13043 
13044 // This pattern is automatically generated from aarch64_ad.m4
13045 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13046 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13047 %{
13048   match(Set dst (RotateLeft src shift));
13049 
13050   ins_cost(INSN_COST);
13051   format %{ "rol    $dst, $src, $shift" %}
13052 
13053   ins_encode %{
13054      __ subw(rscratch1, zr, as_Register($shift$$reg));
13055      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13056   %}
13057   ins_pipe(ialu_reg_reg_vshift);
13058 %}
13059 
13060 
13061 // Add/subtract (extended)
13062 
13063 // This pattern is automatically generated from aarch64_ad.m4
13064 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13065 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13066 %{
13067   match(Set dst (AddL src1 (ConvI2L src2)));
13068   ins_cost(INSN_COST);
13069   format %{ "add  $dst, $src1, $src2, sxtw" %}
13070 
13071    ins_encode %{
13072      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13073             as_Register($src2$$reg), ext::sxtw);
13074    %}
13075   ins_pipe(ialu_reg_reg);
13076 %}
13077 
13078 // This pattern is automatically generated from aarch64_ad.m4
13079 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13080 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13081 %{
13082   match(Set dst (SubL src1 (ConvI2L src2)));
13083   ins_cost(INSN_COST);
13084   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13085 
13086    ins_encode %{
13087      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13088             as_Register($src2$$reg), ext::sxtw);
13089    %}
13090   ins_pipe(ialu_reg_reg);
13091 %}
13092 
13093 // This pattern is automatically generated from aarch64_ad.m4
13094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13095 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13096 %{
13097   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13098   ins_cost(INSN_COST);
13099   format %{ "add  $dst, $src1, $src2, sxth" %}
13100 
13101    ins_encode %{
13102      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13103             as_Register($src2$$reg), ext::sxth);
13104    %}
13105   ins_pipe(ialu_reg_reg);
13106 %}
13107 
13108 // This pattern is automatically generated from aarch64_ad.m4
13109 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13110 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13111 %{
13112   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13113   ins_cost(INSN_COST);
13114   format %{ "add  $dst, $src1, $src2, sxtb" %}
13115 
13116    ins_encode %{
13117      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13118             as_Register($src2$$reg), ext::sxtb);
13119    %}
13120   ins_pipe(ialu_reg_reg);
13121 %}
13122 
13123 // This pattern is automatically generated from aarch64_ad.m4
13124 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13125 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13126 %{
13127   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13128   ins_cost(INSN_COST);
13129   format %{ "add  $dst, $src1, $src2, uxtb" %}
13130 
13131    ins_encode %{
13132      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13133             as_Register($src2$$reg), ext::uxtb);
13134    %}
13135   ins_pipe(ialu_reg_reg);
13136 %}
13137 
13138 // This pattern is automatically generated from aarch64_ad.m4
13139 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13140 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13141 %{
13142   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13143   ins_cost(INSN_COST);
13144   format %{ "add  $dst, $src1, $src2, sxth" %}
13145 
13146    ins_encode %{
13147      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13148             as_Register($src2$$reg), ext::sxth);
13149    %}
13150   ins_pipe(ialu_reg_reg);
13151 %}
13152 
13153 // This pattern is automatically generated from aarch64_ad.m4
13154 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13155 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13156 %{
13157   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13158   ins_cost(INSN_COST);
13159   format %{ "add  $dst, $src1, $src2, sxtw" %}
13160 
13161    ins_encode %{
13162      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13163             as_Register($src2$$reg), ext::sxtw);
13164    %}
13165   ins_pipe(ialu_reg_reg);
13166 %}
13167 
13168 // This pattern is automatically generated from aarch64_ad.m4
13169 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13170 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13171 %{
13172   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13173   ins_cost(INSN_COST);
13174   format %{ "add  $dst, $src1, $src2, sxtb" %}
13175 
13176    ins_encode %{
13177      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13178             as_Register($src2$$reg), ext::sxtb);
13179    %}
13180   ins_pipe(ialu_reg_reg);
13181 %}
13182 
13183 // This pattern is automatically generated from aarch64_ad.m4
13184 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13185 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13186 %{
13187   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13188   ins_cost(INSN_COST);
13189   format %{ "add  $dst, $src1, $src2, uxtb" %}
13190 
13191    ins_encode %{
13192      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13193             as_Register($src2$$reg), ext::uxtb);
13194    %}
13195   ins_pipe(ialu_reg_reg);
13196 %}
13197 
13198 // This pattern is automatically generated from aarch64_ad.m4
13199 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13200 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13201 %{
13202   match(Set dst (AddI src1 (AndI src2 mask)));
13203   ins_cost(INSN_COST);
13204   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13205 
13206    ins_encode %{
13207      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13208             as_Register($src2$$reg), ext::uxtb);
13209    %}
13210   ins_pipe(ialu_reg_reg);
13211 %}
13212 
13213 // This pattern is automatically generated from aarch64_ad.m4
13214 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13215 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13216 %{
13217   match(Set dst (AddI src1 (AndI src2 mask)));
13218   ins_cost(INSN_COST);
13219   format %{ "addw  $dst, $src1, $src2, uxth" %}
13220 
13221    ins_encode %{
13222      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13223             as_Register($src2$$reg), ext::uxth);
13224    %}
13225   ins_pipe(ialu_reg_reg);
13226 %}
13227 
13228 // This pattern is automatically generated from aarch64_ad.m4
13229 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13230 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13231 %{
13232   match(Set dst (AddL src1 (AndL src2 mask)));
13233   ins_cost(INSN_COST);
13234   format %{ "add  $dst, $src1, $src2, uxtb" %}
13235 
13236    ins_encode %{
13237      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13238             as_Register($src2$$reg), ext::uxtb);
13239    %}
13240   ins_pipe(ialu_reg_reg);
13241 %}
13242 
13243 // This pattern is automatically generated from aarch64_ad.m4
13244 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13245 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13246 %{
13247   match(Set dst (AddL src1 (AndL src2 mask)));
13248   ins_cost(INSN_COST);
13249   format %{ "add  $dst, $src1, $src2, uxth" %}
13250 
13251    ins_encode %{
13252      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13253             as_Register($src2$$reg), ext::uxth);
13254    %}
13255   ins_pipe(ialu_reg_reg);
13256 %}
13257 
13258 // This pattern is automatically generated from aarch64_ad.m4
13259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13260 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13261 %{
13262   match(Set dst (AddL src1 (AndL src2 mask)));
13263   ins_cost(INSN_COST);
13264   format %{ "add  $dst, $src1, $src2, uxtw" %}
13265 
13266    ins_encode %{
13267      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13268             as_Register($src2$$reg), ext::uxtw);
13269    %}
13270   ins_pipe(ialu_reg_reg);
13271 %}
13272 
13273 // This pattern is automatically generated from aarch64_ad.m4
13274 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13275 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13276 %{
13277   match(Set dst (SubI src1 (AndI src2 mask)));
13278   ins_cost(INSN_COST);
13279   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13280 
13281    ins_encode %{
13282      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13283             as_Register($src2$$reg), ext::uxtb);
13284    %}
13285   ins_pipe(ialu_reg_reg);
13286 %}
13287 
13288 // This pattern is automatically generated from aarch64_ad.m4
13289 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13290 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13291 %{
13292   match(Set dst (SubI src1 (AndI src2 mask)));
13293   ins_cost(INSN_COST);
13294   format %{ "subw  $dst, $src1, $src2, uxth" %}
13295 
13296    ins_encode %{
13297      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13298             as_Register($src2$$reg), ext::uxth);
13299    %}
13300   ins_pipe(ialu_reg_reg);
13301 %}
13302 
13303 // This pattern is automatically generated from aarch64_ad.m4
13304 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13305 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13306 %{
13307   match(Set dst (SubL src1 (AndL src2 mask)));
13308   ins_cost(INSN_COST);
13309   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13310 
13311    ins_encode %{
13312      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13313             as_Register($src2$$reg), ext::uxtb);
13314    %}
13315   ins_pipe(ialu_reg_reg);
13316 %}
13317 
13318 // This pattern is automatically generated from aarch64_ad.m4
13319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13320 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13321 %{
13322   match(Set dst (SubL src1 (AndL src2 mask)));
13323   ins_cost(INSN_COST);
13324   format %{ "sub  $dst, $src1, $src2, uxth" %}
13325 
13326    ins_encode %{
13327      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13328             as_Register($src2$$reg), ext::uxth);
13329    %}
13330   ins_pipe(ialu_reg_reg);
13331 %}
13332 
13333 // This pattern is automatically generated from aarch64_ad.m4
13334 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13335 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13336 %{
13337   match(Set dst (SubL src1 (AndL src2 mask)));
13338   ins_cost(INSN_COST);
13339   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13340 
13341    ins_encode %{
13342      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13343             as_Register($src2$$reg), ext::uxtw);
13344    %}
13345   ins_pipe(ialu_reg_reg);
13346 %}
13347 
13348 
13349 // This pattern is automatically generated from aarch64_ad.m4
13350 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13351 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13352 %{
13353   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13354   ins_cost(1.9 * INSN_COST);
13355   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13356 
13357    ins_encode %{
13358      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13359             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13360    %}
13361   ins_pipe(ialu_reg_reg_shift);
13362 %}
13363 
13364 // This pattern is automatically generated from aarch64_ad.m4
13365 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13366 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13367 %{
13368   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13369   ins_cost(1.9 * INSN_COST);
13370   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13371 
13372    ins_encode %{
13373      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13374             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13375    %}
13376   ins_pipe(ialu_reg_reg_shift);
13377 %}
13378 
13379 // This pattern is automatically generated from aarch64_ad.m4
13380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13381 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13382 %{
13383   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13384   ins_cost(1.9 * INSN_COST);
13385   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13386 
13387    ins_encode %{
13388      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13389             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13390    %}
13391   ins_pipe(ialu_reg_reg_shift);
13392 %}
13393 
13394 // This pattern is automatically generated from aarch64_ad.m4
13395 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13396 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13397 %{
13398   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13399   ins_cost(1.9 * INSN_COST);
13400   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13401 
13402    ins_encode %{
13403      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13404             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13405    %}
13406   ins_pipe(ialu_reg_reg_shift);
13407 %}
13408 
13409 // This pattern is automatically generated from aarch64_ad.m4
13410 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13411 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13412 %{
13413   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13414   ins_cost(1.9 * INSN_COST);
13415   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13416 
13417    ins_encode %{
13418      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13419             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13420    %}
13421   ins_pipe(ialu_reg_reg_shift);
13422 %}
13423 
13424 // This pattern is automatically generated from aarch64_ad.m4
13425 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13426 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13427 %{
13428   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13429   ins_cost(1.9 * INSN_COST);
13430   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13431 
13432    ins_encode %{
13433      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13434             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13435    %}
13436   ins_pipe(ialu_reg_reg_shift);
13437 %}
13438 
13439 // This pattern is automatically generated from aarch64_ad.m4
13440 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13441 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13442 %{
13443   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13444   ins_cost(1.9 * INSN_COST);
13445   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13446 
13447    ins_encode %{
13448      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13449             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13450    %}
13451   ins_pipe(ialu_reg_reg_shift);
13452 %}
13453 
13454 // This pattern is automatically generated from aarch64_ad.m4
13455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13456 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13457 %{
13458   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13459   ins_cost(1.9 * INSN_COST);
13460   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13461 
13462    ins_encode %{
13463      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13464             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13465    %}
13466   ins_pipe(ialu_reg_reg_shift);
13467 %}
13468 
13469 // This pattern is automatically generated from aarch64_ad.m4
13470 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13471 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13472 %{
13473   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13474   ins_cost(1.9 * INSN_COST);
13475   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13476 
13477    ins_encode %{
13478      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13479             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13480    %}
13481   ins_pipe(ialu_reg_reg_shift);
13482 %}
13483 
13484 // This pattern is automatically generated from aarch64_ad.m4
13485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13486 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13487 %{
13488   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13489   ins_cost(1.9 * INSN_COST);
13490   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13491 
13492    ins_encode %{
13493      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13494             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13495    %}
13496   ins_pipe(ialu_reg_reg_shift);
13497 %}
13498 
13499 // This pattern is automatically generated from aarch64_ad.m4
13500 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13501 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13502 %{
13503   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13504   ins_cost(1.9 * INSN_COST);
13505   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13506 
13507    ins_encode %{
13508      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13509             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13510    %}
13511   ins_pipe(ialu_reg_reg_shift);
13512 %}
13513 
13514 // This pattern is automatically generated from aarch64_ad.m4
13515 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13516 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13517 %{
13518   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13519   ins_cost(1.9 * INSN_COST);
13520   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13521 
13522    ins_encode %{
13523      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13524             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13525    %}
13526   ins_pipe(ialu_reg_reg_shift);
13527 %}
13528 
13529 // This pattern is automatically generated from aarch64_ad.m4
13530 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13531 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13532 %{
13533   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13534   ins_cost(1.9 * INSN_COST);
13535   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13536 
13537    ins_encode %{
13538      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13539             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13540    %}
13541   ins_pipe(ialu_reg_reg_shift);
13542 %}
13543 
13544 // This pattern is automatically generated from aarch64_ad.m4
13545 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13546 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13547 %{
13548   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13549   ins_cost(1.9 * INSN_COST);
13550   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13551 
13552    ins_encode %{
13553      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13554             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13555    %}
13556   ins_pipe(ialu_reg_reg_shift);
13557 %}
13558 
13559 // This pattern is automatically generated from aarch64_ad.m4
13560 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13561 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13562 %{
13563   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13564   ins_cost(1.9 * INSN_COST);
13565   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13566 
13567    ins_encode %{
13568      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13569             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13570    %}
13571   ins_pipe(ialu_reg_reg_shift);
13572 %}
13573 
13574 // This pattern is automatically generated from aarch64_ad.m4
13575 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13576 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13577 %{
13578   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13579   ins_cost(1.9 * INSN_COST);
13580   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13581 
13582    ins_encode %{
13583      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13584             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13585    %}
13586   ins_pipe(ialu_reg_reg_shift);
13587 %}
13588 
13589 // This pattern is automatically generated from aarch64_ad.m4
13590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13591 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13592 %{
13593   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13594   ins_cost(1.9 * INSN_COST);
13595   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13596 
13597    ins_encode %{
13598      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13599             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13600    %}
13601   ins_pipe(ialu_reg_reg_shift);
13602 %}
13603 
13604 // This pattern is automatically generated from aarch64_ad.m4
13605 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13606 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13607 %{
13608   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13609   ins_cost(1.9 * INSN_COST);
13610   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13611 
13612    ins_encode %{
13613      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13614             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13615    %}
13616   ins_pipe(ialu_reg_reg_shift);
13617 %}
13618 
13619 // This pattern is automatically generated from aarch64_ad.m4
13620 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13621 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13622 %{
13623   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13624   ins_cost(1.9 * INSN_COST);
13625   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13626 
13627    ins_encode %{
13628      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13629             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13630    %}
13631   ins_pipe(ialu_reg_reg_shift);
13632 %}
13633 
13634 // This pattern is automatically generated from aarch64_ad.m4
13635 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13636 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13637 %{
13638   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13639   ins_cost(1.9 * INSN_COST);
13640   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13641 
13642    ins_encode %{
13643      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13644             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13645    %}
13646   ins_pipe(ialu_reg_reg_shift);
13647 %}
13648 
13649 // This pattern is automatically generated from aarch64_ad.m4
13650 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13651 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13652 %{
13653   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13654   ins_cost(1.9 * INSN_COST);
13655   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13656 
13657    ins_encode %{
13658      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13659             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13660    %}
13661   ins_pipe(ialu_reg_reg_shift);
13662 %}
13663 
13664 // This pattern is automatically generated from aarch64_ad.m4
13665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13666 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13667 %{
13668   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13669   ins_cost(1.9 * INSN_COST);
13670   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13671 
13672    ins_encode %{
13673      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13674             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13675    %}
13676   ins_pipe(ialu_reg_reg_shift);
13677 %}
13678 
13679 
13680 
13681 // END This section of the file is automatically generated. Do not edit --------------
13682 
13683 
13684 // ============================================================================
13685 // Floating Point Arithmetic Instructions
13686 
13687 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13688   match(Set dst (AddF src1 src2));
13689 
13690   ins_cost(INSN_COST * 5);
13691   format %{ "fadds   $dst, $src1, $src2" %}
13692 
13693   ins_encode %{
13694     __ fadds(as_FloatRegister($dst$$reg),
13695              as_FloatRegister($src1$$reg),
13696              as_FloatRegister($src2$$reg));
13697   %}
13698 
13699   ins_pipe(fp_dop_reg_reg_s);
13700 %}
13701 
13702 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13703   match(Set dst (AddD src1 src2));
13704 
13705   ins_cost(INSN_COST * 5);
13706   format %{ "faddd   $dst, $src1, $src2" %}
13707 
13708   ins_encode %{
13709     __ faddd(as_FloatRegister($dst$$reg),
13710              as_FloatRegister($src1$$reg),
13711              as_FloatRegister($src2$$reg));
13712   %}
13713 
13714   ins_pipe(fp_dop_reg_reg_d);
13715 %}
13716 
13717 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13718   match(Set dst (SubF src1 src2));
13719 
13720   ins_cost(INSN_COST * 5);
13721   format %{ "fsubs   $dst, $src1, $src2" %}
13722 
13723   ins_encode %{
13724     __ fsubs(as_FloatRegister($dst$$reg),
13725              as_FloatRegister($src1$$reg),
13726              as_FloatRegister($src2$$reg));
13727   %}
13728 
13729   ins_pipe(fp_dop_reg_reg_s);
13730 %}
13731 
13732 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13733   match(Set dst (SubD src1 src2));
13734 
13735   ins_cost(INSN_COST * 5);
13736   format %{ "fsubd   $dst, $src1, $src2" %}
13737 
13738   ins_encode %{
13739     __ fsubd(as_FloatRegister($dst$$reg),
13740              as_FloatRegister($src1$$reg),
13741              as_FloatRegister($src2$$reg));
13742   %}
13743 
13744   ins_pipe(fp_dop_reg_reg_d);
13745 %}
13746 
13747 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13748   match(Set dst (MulF src1 src2));
13749 
13750   ins_cost(INSN_COST * 6);
13751   format %{ "fmuls   $dst, $src1, $src2" %}
13752 
13753   ins_encode %{
13754     __ fmuls(as_FloatRegister($dst$$reg),
13755              as_FloatRegister($src1$$reg),
13756              as_FloatRegister($src2$$reg));
13757   %}
13758 
13759   ins_pipe(fp_dop_reg_reg_s);
13760 %}
13761 
13762 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13763   match(Set dst (MulD src1 src2));
13764 
13765   ins_cost(INSN_COST * 6);
13766   format %{ "fmuld   $dst, $src1, $src2" %}
13767 
13768   ins_encode %{
13769     __ fmuld(as_FloatRegister($dst$$reg),
13770              as_FloatRegister($src1$$reg),
13771              as_FloatRegister($src2$$reg));
13772   %}
13773 
13774   ins_pipe(fp_dop_reg_reg_d);
13775 %}
13776 
13777 // src1 * src2 + src3
13778 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13779   predicate(UseFMA);
13780   match(Set dst (FmaF src3 (Binary src1 src2)));
13781 
13782   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
13783 
13784   ins_encode %{
13785     __ fmadds(as_FloatRegister($dst$$reg),
13786              as_FloatRegister($src1$$reg),
13787              as_FloatRegister($src2$$reg),
13788              as_FloatRegister($src3$$reg));
13789   %}
13790 
13791   ins_pipe(pipe_class_default);
13792 %}
13793 
13794 // src1 * src2 + src3
13795 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13796   predicate(UseFMA);
13797   match(Set dst (FmaD src3 (Binary src1 src2)));
13798 
13799   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
13800 
13801   ins_encode %{
13802     __ fmaddd(as_FloatRegister($dst$$reg),
13803              as_FloatRegister($src1$$reg),
13804              as_FloatRegister($src2$$reg),
13805              as_FloatRegister($src3$$reg));
13806   %}
13807 
13808   ins_pipe(pipe_class_default);
13809 %}
13810 
13811 // -src1 * src2 + src3
13812 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13813   predicate(UseFMA);
13814   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
13815   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13816 
13817   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
13818 
13819   ins_encode %{
13820     __ fmsubs(as_FloatRegister($dst$$reg),
13821               as_FloatRegister($src1$$reg),
13822               as_FloatRegister($src2$$reg),
13823               as_FloatRegister($src3$$reg));
13824   %}
13825 
13826   ins_pipe(pipe_class_default);
13827 %}
13828 
13829 // -src1 * src2 + src3
13830 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13831   predicate(UseFMA);
13832   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
13833   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13834 
13835   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
13836 
13837   ins_encode %{
13838     __ fmsubd(as_FloatRegister($dst$$reg),
13839               as_FloatRegister($src1$$reg),
13840               as_FloatRegister($src2$$reg),
13841               as_FloatRegister($src3$$reg));
13842   %}
13843 
13844   ins_pipe(pipe_class_default);
13845 %}
13846 
13847 // -src1 * src2 - src3
13848 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13849   predicate(UseFMA);
13850   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
13851   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13852 
13853   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
13854 
13855   ins_encode %{
13856     __ fnmadds(as_FloatRegister($dst$$reg),
13857                as_FloatRegister($src1$$reg),
13858                as_FloatRegister($src2$$reg),
13859                as_FloatRegister($src3$$reg));
13860   %}
13861 
13862   ins_pipe(pipe_class_default);
13863 %}
13864 
13865 // -src1 * src2 - src3
13866 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13867   predicate(UseFMA);
13868   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
13869   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13870 
13871   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
13872 
13873   ins_encode %{
13874     __ fnmaddd(as_FloatRegister($dst$$reg),
13875                as_FloatRegister($src1$$reg),
13876                as_FloatRegister($src2$$reg),
13877                as_FloatRegister($src3$$reg));
13878   %}
13879 
13880   ins_pipe(pipe_class_default);
13881 %}
13882 
13883 // src1 * src2 - src3
13884 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13885   predicate(UseFMA);
13886   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13887 
13888   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
13889 
13890   ins_encode %{
13891     __ fnmsubs(as_FloatRegister($dst$$reg),
13892                as_FloatRegister($src1$$reg),
13893                as_FloatRegister($src2$$reg),
13894                as_FloatRegister($src3$$reg));
13895   %}
13896 
13897   ins_pipe(pipe_class_default);
13898 %}
13899 
13900 // src1 * src2 - src3
13901 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13902   predicate(UseFMA);
13903   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13904 
13905   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
13906 
13907   ins_encode %{
13908   // n.b. insn name should be fnmsubd
13909     __ fnmsub(as_FloatRegister($dst$$reg),
13910               as_FloatRegister($src1$$reg),
13911               as_FloatRegister($src2$$reg),
13912               as_FloatRegister($src3$$reg));
13913   %}
13914 
13915   ins_pipe(pipe_class_default);
13916 %}
13917 
13918 
13919 // Math.max(FF)F
13920 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13921   match(Set dst (MaxF src1 src2));
13922 
13923   format %{ "fmaxs   $dst, $src1, $src2" %}
13924   ins_encode %{
13925     __ fmaxs(as_FloatRegister($dst$$reg),
13926              as_FloatRegister($src1$$reg),
13927              as_FloatRegister($src2$$reg));
13928   %}
13929 
13930   ins_pipe(fp_dop_reg_reg_s);
13931 %}
13932 
13933 // Math.min(FF)F
13934 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13935   match(Set dst (MinF src1 src2));
13936 
13937   format %{ "fmins   $dst, $src1, $src2" %}
13938   ins_encode %{
13939     __ fmins(as_FloatRegister($dst$$reg),
13940              as_FloatRegister($src1$$reg),
13941              as_FloatRegister($src2$$reg));
13942   %}
13943 
13944   ins_pipe(fp_dop_reg_reg_s);
13945 %}
13946 
13947 // Math.max(DD)D
13948 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13949   match(Set dst (MaxD src1 src2));
13950 
13951   format %{ "fmaxd   $dst, $src1, $src2" %}
13952   ins_encode %{
13953     __ fmaxd(as_FloatRegister($dst$$reg),
13954              as_FloatRegister($src1$$reg),
13955              as_FloatRegister($src2$$reg));
13956   %}
13957 
13958   ins_pipe(fp_dop_reg_reg_d);
13959 %}
13960 
13961 // Math.min(DD)D
13962 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13963   match(Set dst (MinD src1 src2));
13964 
13965   format %{ "fmind   $dst, $src1, $src2" %}
13966   ins_encode %{
13967     __ fmind(as_FloatRegister($dst$$reg),
13968              as_FloatRegister($src1$$reg),
13969              as_FloatRegister($src2$$reg));
13970   %}
13971 
13972   ins_pipe(fp_dop_reg_reg_d);
13973 %}
13974 
13975 
13976 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13977   match(Set dst (DivF src1  src2));
13978 
13979   ins_cost(INSN_COST * 18);
13980   format %{ "fdivs   $dst, $src1, $src2" %}
13981 
13982   ins_encode %{
13983     __ fdivs(as_FloatRegister($dst$$reg),
13984              as_FloatRegister($src1$$reg),
13985              as_FloatRegister($src2$$reg));
13986   %}
13987 
13988   ins_pipe(fp_div_s);
13989 %}
13990 
13991 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13992   match(Set dst (DivD src1  src2));
13993 
13994   ins_cost(INSN_COST * 32);
13995   format %{ "fdivd   $dst, $src1, $src2" %}
13996 
13997   ins_encode %{
13998     __ fdivd(as_FloatRegister($dst$$reg),
13999              as_FloatRegister($src1$$reg),
14000              as_FloatRegister($src2$$reg));
14001   %}
14002 
14003   ins_pipe(fp_div_d);
14004 %}
14005 
14006 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14007   match(Set dst (NegF src));
14008 
14009   ins_cost(INSN_COST * 3);
14010   format %{ "fneg   $dst, $src" %}
14011 
14012   ins_encode %{
14013     __ fnegs(as_FloatRegister($dst$$reg),
14014              as_FloatRegister($src$$reg));
14015   %}
14016 
14017   ins_pipe(fp_uop_s);
14018 %}
14019 
14020 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14021   match(Set dst (NegD src));
14022 
14023   ins_cost(INSN_COST * 3);
14024   format %{ "fnegd   $dst, $src" %}
14025 
14026   ins_encode %{
14027     __ fnegd(as_FloatRegister($dst$$reg),
14028              as_FloatRegister($src$$reg));
14029   %}
14030 
14031   ins_pipe(fp_uop_d);
14032 %}
14033 
14034 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14035 %{
14036   match(Set dst (AbsI src));
14037 
14038   effect(KILL cr);
14039   ins_cost(INSN_COST * 2);
14040   format %{ "cmpw  $src, zr\n\t"
14041             "cnegw $dst, $src, Assembler::LT\t# int abs"
14042   %}
14043 
14044   ins_encode %{
14045     __ cmpw(as_Register($src$$reg), zr);
14046     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14047   %}
14048   ins_pipe(pipe_class_default);
14049 %}
14050 
14051 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14052 %{
14053   match(Set dst (AbsL src));
14054 
14055   effect(KILL cr);
14056   ins_cost(INSN_COST * 2);
14057   format %{ "cmp  $src, zr\n\t"
14058             "cneg $dst, $src, Assembler::LT\t# long abs"
14059   %}
14060 
14061   ins_encode %{
14062     __ cmp(as_Register($src$$reg), zr);
14063     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14064   %}
14065   ins_pipe(pipe_class_default);
14066 %}
14067 
14068 instruct absF_reg(vRegF dst, vRegF src) %{
14069   match(Set dst (AbsF src));
14070 
14071   ins_cost(INSN_COST * 3);
14072   format %{ "fabss   $dst, $src" %}
14073   ins_encode %{
14074     __ fabss(as_FloatRegister($dst$$reg),
14075              as_FloatRegister($src$$reg));
14076   %}
14077 
14078   ins_pipe(fp_uop_s);
14079 %}
14080 
14081 instruct absD_reg(vRegD dst, vRegD src) %{
14082   match(Set dst (AbsD src));
14083 
14084   ins_cost(INSN_COST * 3);
14085   format %{ "fabsd   $dst, $src" %}
14086   ins_encode %{
14087     __ fabsd(as_FloatRegister($dst$$reg),
14088              as_FloatRegister($src$$reg));
14089   %}
14090 
14091   ins_pipe(fp_uop_d);
14092 %}
14093 
14094 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14095   match(Set dst (AbsF (SubF src1 src2)));
14096 
14097   ins_cost(INSN_COST * 3);
14098   format %{ "fabds   $dst, $src1, $src2" %}
14099   ins_encode %{
14100     __ fabds(as_FloatRegister($dst$$reg),
14101              as_FloatRegister($src1$$reg),
14102              as_FloatRegister($src2$$reg));
14103   %}
14104 
14105   ins_pipe(fp_uop_s);
14106 %}
14107 
14108 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14109   match(Set dst (AbsD (SubD src1 src2)));
14110 
14111   ins_cost(INSN_COST * 3);
14112   format %{ "fabdd   $dst, $src1, $src2" %}
14113   ins_encode %{
14114     __ fabdd(as_FloatRegister($dst$$reg),
14115              as_FloatRegister($src1$$reg),
14116              as_FloatRegister($src2$$reg));
14117   %}
14118 
14119   ins_pipe(fp_uop_d);
14120 %}
14121 
14122 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14123   match(Set dst (SqrtD src));
14124 
14125   ins_cost(INSN_COST * 50);
14126   format %{ "fsqrtd  $dst, $src" %}
14127   ins_encode %{
14128     __ fsqrtd(as_FloatRegister($dst$$reg),
14129              as_FloatRegister($src$$reg));
14130   %}
14131 
14132   ins_pipe(fp_div_s);
14133 %}
14134 
14135 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14136   match(Set dst (SqrtF src));
14137 
14138   ins_cost(INSN_COST * 50);
14139   format %{ "fsqrts  $dst, $src" %}
14140   ins_encode %{
14141     __ fsqrts(as_FloatRegister($dst$$reg),
14142              as_FloatRegister($src$$reg));
14143   %}
14144 
14145   ins_pipe(fp_div_d);
14146 %}
14147 
14148 // Math.rint, floor, ceil
14149 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14150   match(Set dst (RoundDoubleMode src rmode));
14151   format %{ "frint  $dst, $src, $rmode" %}
14152   ins_encode %{
14153     switch ($rmode$$constant) {
14154       case RoundDoubleModeNode::rmode_rint:
14155         __ frintnd(as_FloatRegister($dst$$reg),
14156                    as_FloatRegister($src$$reg));
14157         break;
14158       case RoundDoubleModeNode::rmode_floor:
14159         __ frintmd(as_FloatRegister($dst$$reg),
14160                    as_FloatRegister($src$$reg));
14161         break;
14162       case RoundDoubleModeNode::rmode_ceil:
14163         __ frintpd(as_FloatRegister($dst$$reg),
14164                    as_FloatRegister($src$$reg));
14165         break;
14166     }
14167   %}
14168   ins_pipe(fp_uop_d);
14169 %}
14170 
14171 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14172   match(Set dst (CopySignD src1 (Binary src2 zero)));
14173   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14174   format %{ "CopySignD  $dst $src1 $src2" %}
14175   ins_encode %{
14176     FloatRegister dst = as_FloatRegister($dst$$reg),
14177                   src1 = as_FloatRegister($src1$$reg),
14178                   src2 = as_FloatRegister($src2$$reg),
14179                   zero = as_FloatRegister($zero$$reg);
14180     __ fnegd(dst, zero);
14181     __ bsl(dst, __ T8B, src2, src1);
14182   %}
14183   ins_pipe(fp_uop_d);
14184 %}
14185 
14186 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14187   match(Set dst (CopySignF src1 src2));
14188   effect(TEMP_DEF dst, USE src1, USE src2);
14189   format %{ "CopySignF  $dst $src1 $src2" %}
14190   ins_encode %{
14191     FloatRegister dst = as_FloatRegister($dst$$reg),
14192                   src1 = as_FloatRegister($src1$$reg),
14193                   src2 = as_FloatRegister($src2$$reg);
14194     __ movi(dst, __ T2S, 0x80, 24);
14195     __ bsl(dst, __ T8B, src2, src1);
14196   %}
14197   ins_pipe(fp_uop_d);
14198 %}
14199 
14200 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14201   match(Set dst (SignumD src (Binary zero one)));
14202   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14203   format %{ "signumD  $dst, $src" %}
14204   ins_encode %{
14205     FloatRegister src = as_FloatRegister($src$$reg),
14206                   dst = as_FloatRegister($dst$$reg),
14207                   zero = as_FloatRegister($zero$$reg),
14208                   one = as_FloatRegister($one$$reg);
14209     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14210     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14211     // Bit selection instruction gets bit from "one" for each enabled bit in
14212     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14213     // NaN the whole "src" will be copied because "dst" is zero. For all other
14214     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14215     // from "src", and all other bits are copied from 1.0.
14216     __ bsl(dst, __ T8B, one, src);
14217   %}
14218   ins_pipe(fp_uop_d);
14219 %}
14220 
14221 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14222   match(Set dst (SignumF src (Binary zero one)));
14223   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14224   format %{ "signumF  $dst, $src" %}
14225   ins_encode %{
14226     FloatRegister src = as_FloatRegister($src$$reg),
14227                   dst = as_FloatRegister($dst$$reg),
14228                   zero = as_FloatRegister($zero$$reg),
14229                   one = as_FloatRegister($one$$reg);
14230     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14231     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14232     // Bit selection instruction gets bit from "one" for each enabled bit in
14233     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14234     // NaN the whole "src" will be copied because "dst" is zero. For all other
14235     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14236     // from "src", and all other bits are copied from 1.0.
14237     __ bsl(dst, __ T8B, one, src);
14238   %}
14239   ins_pipe(fp_uop_d);
14240 %}
14241 
14242 instruct onspinwait() %{
14243   match(OnSpinWait);
14244   ins_cost(INSN_COST);
14245 
14246   format %{ "onspinwait" %}
14247 
14248   ins_encode %{
14249     __ spin_wait();
14250   %}
14251   ins_pipe(pipe_class_empty);
14252 %}
14253 
14254 // ============================================================================
14255 // Logical Instructions
14256 
14257 // Integer Logical Instructions
14258 
14259 // And Instructions
14260 
14261 
14262 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14263   match(Set dst (AndI src1 src2));
14264 
14265   format %{ "andw  $dst, $src1, $src2\t# int" %}
14266 
14267   ins_cost(INSN_COST);
14268   ins_encode %{
14269     __ andw(as_Register($dst$$reg),
14270             as_Register($src1$$reg),
14271             as_Register($src2$$reg));
14272   %}
14273 
14274   ins_pipe(ialu_reg_reg);
14275 %}
14276 
14277 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14278   match(Set dst (AndI src1 src2));
14279 
14280   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14281 
14282   ins_cost(INSN_COST);
14283   ins_encode %{
14284     __ andw(as_Register($dst$$reg),
14285             as_Register($src1$$reg),
14286             (uint64_t)($src2$$constant));
14287   %}
14288 
14289   ins_pipe(ialu_reg_imm);
14290 %}
14291 
14292 // Or Instructions
14293 
14294 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14295   match(Set dst (OrI src1 src2));
14296 
14297   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14298 
14299   ins_cost(INSN_COST);
14300   ins_encode %{
14301     __ orrw(as_Register($dst$$reg),
14302             as_Register($src1$$reg),
14303             as_Register($src2$$reg));
14304   %}
14305 
14306   ins_pipe(ialu_reg_reg);
14307 %}
14308 
14309 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14310   match(Set dst (OrI src1 src2));
14311 
14312   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14313 
14314   ins_cost(INSN_COST);
14315   ins_encode %{
14316     __ orrw(as_Register($dst$$reg),
14317             as_Register($src1$$reg),
14318             (uint64_t)($src2$$constant));
14319   %}
14320 
14321   ins_pipe(ialu_reg_imm);
14322 %}
14323 
14324 // Xor Instructions
14325 
14326 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14327   match(Set dst (XorI src1 src2));
14328 
14329   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14330 
14331   ins_cost(INSN_COST);
14332   ins_encode %{
14333     __ eorw(as_Register($dst$$reg),
14334             as_Register($src1$$reg),
14335             as_Register($src2$$reg));
14336   %}
14337 
14338   ins_pipe(ialu_reg_reg);
14339 %}
14340 
14341 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14342   match(Set dst (XorI src1 src2));
14343 
14344   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14345 
14346   ins_cost(INSN_COST);
14347   ins_encode %{
14348     __ eorw(as_Register($dst$$reg),
14349             as_Register($src1$$reg),
14350             (uint64_t)($src2$$constant));
14351   %}
14352 
14353   ins_pipe(ialu_reg_imm);
14354 %}
14355 
14356 // Long Logical Instructions
14357 // TODO
14358 
14359 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14360   match(Set dst (AndL src1 src2));
14361 
14362   format %{ "and  $dst, $src1, $src2\t# int" %}
14363 
14364   ins_cost(INSN_COST);
14365   ins_encode %{
14366     __ andr(as_Register($dst$$reg),
14367             as_Register($src1$$reg),
14368             as_Register($src2$$reg));
14369   %}
14370 
14371   ins_pipe(ialu_reg_reg);
14372 %}
14373 
14374 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14375   match(Set dst (AndL src1 src2));
14376 
14377   format %{ "and  $dst, $src1, $src2\t# int" %}
14378 
14379   ins_cost(INSN_COST);
14380   ins_encode %{
14381     __ andr(as_Register($dst$$reg),
14382             as_Register($src1$$reg),
14383             (uint64_t)($src2$$constant));
14384   %}
14385 
14386   ins_pipe(ialu_reg_imm);
14387 %}
14388 
14389 // Or Instructions
14390 
14391 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14392   match(Set dst (OrL src1 src2));
14393 
14394   format %{ "orr  $dst, $src1, $src2\t# int" %}
14395 
14396   ins_cost(INSN_COST);
14397   ins_encode %{
14398     __ orr(as_Register($dst$$reg),
14399            as_Register($src1$$reg),
14400            as_Register($src2$$reg));
14401   %}
14402 
14403   ins_pipe(ialu_reg_reg);
14404 %}
14405 
14406 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14407   match(Set dst (OrL src1 src2));
14408 
14409   format %{ "orr  $dst, $src1, $src2\t# int" %}
14410 
14411   ins_cost(INSN_COST);
14412   ins_encode %{
14413     __ orr(as_Register($dst$$reg),
14414            as_Register($src1$$reg),
14415            (uint64_t)($src2$$constant));
14416   %}
14417 
14418   ins_pipe(ialu_reg_imm);
14419 %}
14420 
14421 // Xor Instructions
14422 
14423 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14424   match(Set dst (XorL src1 src2));
14425 
14426   format %{ "eor  $dst, $src1, $src2\t# int" %}
14427 
14428   ins_cost(INSN_COST);
14429   ins_encode %{
14430     __ eor(as_Register($dst$$reg),
14431            as_Register($src1$$reg),
14432            as_Register($src2$$reg));
14433   %}
14434 
14435   ins_pipe(ialu_reg_reg);
14436 %}
14437 
14438 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14439   match(Set dst (XorL src1 src2));
14440 
14441   ins_cost(INSN_COST);
14442   format %{ "eor  $dst, $src1, $src2\t# int" %}
14443 
14444   ins_encode %{
14445     __ eor(as_Register($dst$$reg),
14446            as_Register($src1$$reg),
14447            (uint64_t)($src2$$constant));
14448   %}
14449 
14450   ins_pipe(ialu_reg_imm);
14451 %}
14452 
14453 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14454 %{
14455   match(Set dst (ConvI2L src));
14456 
14457   ins_cost(INSN_COST);
14458   format %{ "sxtw  $dst, $src\t# i2l" %}
14459   ins_encode %{
14460     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14461   %}
14462   ins_pipe(ialu_reg_shift);
14463 %}
14464 
14465 // this pattern occurs in bigmath arithmetic
14466 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14467 %{
14468   match(Set dst (AndL (ConvI2L src) mask));
14469 
14470   ins_cost(INSN_COST);
14471   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14472   ins_encode %{
14473     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14474   %}
14475 
14476   ins_pipe(ialu_reg_shift);
14477 %}
14478 
14479 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14480   match(Set dst (ConvL2I src));
14481 
14482   ins_cost(INSN_COST);
14483   format %{ "movw  $dst, $src \t// l2i" %}
14484 
14485   ins_encode %{
14486     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14487   %}
14488 
14489   ins_pipe(ialu_reg);
14490 %}
14491 
14492 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14493 %{
14494   match(Set dst (Conv2B src));
14495   effect(KILL cr);
14496 
14497   format %{
14498     "cmpw $src, zr\n\t"
14499     "cset $dst, ne"
14500   %}
14501 
14502   ins_encode %{
14503     __ cmpw(as_Register($src$$reg), zr);
14504     __ cset(as_Register($dst$$reg), Assembler::NE);
14505   %}
14506 
14507   ins_pipe(ialu_reg);
14508 %}
14509 
14510 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14511 %{
14512   match(Set dst (Conv2B src));
14513   effect(KILL cr);
14514 
14515   format %{
14516     "cmp  $src, zr\n\t"
14517     "cset $dst, ne"
14518   %}
14519 
14520   ins_encode %{
14521     __ cmp(as_Register($src$$reg), zr);
14522     __ cset(as_Register($dst$$reg), Assembler::NE);
14523   %}
14524 
14525   ins_pipe(ialu_reg);
14526 %}
14527 
14528 instruct convD2F_reg(vRegF dst, vRegD src) %{
14529   match(Set dst (ConvD2F src));
14530 
14531   ins_cost(INSN_COST * 5);
14532   format %{ "fcvtd  $dst, $src \t// d2f" %}
14533 
14534   ins_encode %{
14535     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14536   %}
14537 
14538   ins_pipe(fp_d2f);
14539 %}
14540 
14541 instruct convF2D_reg(vRegD dst, vRegF src) %{
14542   match(Set dst (ConvF2D src));
14543 
14544   ins_cost(INSN_COST * 5);
14545   format %{ "fcvts  $dst, $src \t// f2d" %}
14546 
14547   ins_encode %{
14548     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14549   %}
14550 
14551   ins_pipe(fp_f2d);
14552 %}
14553 
14554 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14555   match(Set dst (ConvF2I src));
14556 
14557   ins_cost(INSN_COST * 5);
14558   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14559 
14560   ins_encode %{
14561     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14562   %}
14563 
14564   ins_pipe(fp_f2i);
14565 %}
14566 
14567 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14568   match(Set dst (ConvF2L src));
14569 
14570   ins_cost(INSN_COST * 5);
14571   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14572 
14573   ins_encode %{
14574     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14575   %}
14576 
14577   ins_pipe(fp_f2l);
14578 %}
14579 
14580 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14581   match(Set dst (ConvF2HF src));
14582   format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14583             "smov $dst, $tmp\t# move result from $tmp to $dst"
14584   %}
14585   effect(TEMP tmp);
14586   ins_encode %{
14587       __ fcvtsh($tmp$$FloatRegister, $src$$FloatRegister);
14588       __ smov($dst$$Register, $tmp$$FloatRegister, __ H, 0);
14589   %}
14590   ins_pipe(pipe_slow);
14591 %}
14592 
14593 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14594   match(Set dst (ConvHF2F src));
14595   format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14596             "fcvt $dst, $tmp\t# convert half to single precision"
14597   %}
14598   effect(TEMP tmp);
14599   ins_encode %{
14600       __ mov($tmp$$FloatRegister, __ H, 0, $src$$Register);
14601       __ fcvths($dst$$FloatRegister, $tmp$$FloatRegister);
14602   %}
14603   ins_pipe(pipe_slow);
14604 %}
14605 
14606 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14607   match(Set dst (ConvI2F src));
14608 
14609   ins_cost(INSN_COST * 5);
14610   format %{ "scvtfws  $dst, $src \t// i2f" %}
14611 
14612   ins_encode %{
14613     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14614   %}
14615 
14616   ins_pipe(fp_i2f);
14617 %}
14618 
14619 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14620   match(Set dst (ConvL2F src));
14621 
14622   ins_cost(INSN_COST * 5);
14623   format %{ "scvtfs  $dst, $src \t// l2f" %}
14624 
14625   ins_encode %{
14626     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14627   %}
14628 
14629   ins_pipe(fp_l2f);
14630 %}
14631 
14632 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14633   match(Set dst (ConvD2I src));
14634 
14635   ins_cost(INSN_COST * 5);
14636   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14637 
14638   ins_encode %{
14639     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14640   %}
14641 
14642   ins_pipe(fp_d2i);
14643 %}
14644 
14645 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14646   match(Set dst (ConvD2L src));
14647 
14648   ins_cost(INSN_COST * 5);
14649   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14650 
14651   ins_encode %{
14652     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14653   %}
14654 
14655   ins_pipe(fp_d2l);
14656 %}
14657 
14658 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14659   match(Set dst (ConvI2D src));
14660 
14661   ins_cost(INSN_COST * 5);
14662   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14663 
14664   ins_encode %{
14665     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14666   %}
14667 
14668   ins_pipe(fp_i2d);
14669 %}
14670 
14671 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14672   match(Set dst (ConvL2D src));
14673 
14674   ins_cost(INSN_COST * 5);
14675   format %{ "scvtfd  $dst, $src \t// l2d" %}
14676 
14677   ins_encode %{
14678     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14679   %}
14680 
14681   ins_pipe(fp_l2d);
14682 %}
14683 
14684 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14685 %{
14686   match(Set dst (RoundD src));
14687   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14688   format %{ "java_round_double $dst,$src"%}
14689   ins_encode %{
14690     __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14691                          as_FloatRegister($ftmp$$reg));
14692   %}
14693   ins_pipe(pipe_slow);
14694 %}
14695 
14696 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14697 %{
14698   match(Set dst (RoundF src));
14699   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14700   format %{ "java_round_float $dst,$src"%}
14701   ins_encode %{
14702     __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14703                         as_FloatRegister($ftmp$$reg));
14704   %}
14705   ins_pipe(pipe_slow);
14706 %}
14707 
14708 // stack <-> reg and reg <-> reg shuffles with no conversion
14709 
14710 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14711 
14712   match(Set dst (MoveF2I src));
14713 
14714   effect(DEF dst, USE src);
14715 
14716   ins_cost(4 * INSN_COST);
14717 
14718   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14719 
14720   ins_encode %{
14721     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14722   %}
14723 
14724   ins_pipe(iload_reg_reg);
14725 
14726 %}
14727 
14728 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14729 
14730   match(Set dst (MoveI2F src));
14731 
14732   effect(DEF dst, USE src);
14733 
14734   ins_cost(4 * INSN_COST);
14735 
14736   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14737 
14738   ins_encode %{
14739     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14740   %}
14741 
14742   ins_pipe(pipe_class_memory);
14743 
14744 %}
14745 
14746 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14747 
14748   match(Set dst (MoveD2L src));
14749 
14750   effect(DEF dst, USE src);
14751 
14752   ins_cost(4 * INSN_COST);
14753 
14754   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14755 
14756   ins_encode %{
14757     __ ldr($dst$$Register, Address(sp, $src$$disp));
14758   %}
14759 
14760   ins_pipe(iload_reg_reg);
14761 
14762 %}
14763 
14764 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14765 
14766   match(Set dst (MoveL2D src));
14767 
14768   effect(DEF dst, USE src);
14769 
14770   ins_cost(4 * INSN_COST);
14771 
14772   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14773 
14774   ins_encode %{
14775     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14776   %}
14777 
14778   ins_pipe(pipe_class_memory);
14779 
14780 %}
14781 
14782 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14783 
14784   match(Set dst (MoveF2I src));
14785 
14786   effect(DEF dst, USE src);
14787 
14788   ins_cost(INSN_COST);
14789 
14790   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14791 
14792   ins_encode %{
14793     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14794   %}
14795 
14796   ins_pipe(pipe_class_memory);
14797 
14798 %}
14799 
14800 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14801 
14802   match(Set dst (MoveI2F src));
14803 
14804   effect(DEF dst, USE src);
14805 
14806   ins_cost(INSN_COST);
14807 
14808   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14809 
14810   ins_encode %{
14811     __ strw($src$$Register, Address(sp, $dst$$disp));
14812   %}
14813 
14814   ins_pipe(istore_reg_reg);
14815 
14816 %}
14817 
14818 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14819 
14820   match(Set dst (MoveD2L src));
14821 
14822   effect(DEF dst, USE src);
14823 
14824   ins_cost(INSN_COST);
14825 
14826   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14827 
14828   ins_encode %{
14829     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14830   %}
14831 
14832   ins_pipe(pipe_class_memory);
14833 
14834 %}
14835 
14836 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14837 
14838   match(Set dst (MoveL2D src));
14839 
14840   effect(DEF dst, USE src);
14841 
14842   ins_cost(INSN_COST);
14843 
14844   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14845 
14846   ins_encode %{
14847     __ str($src$$Register, Address(sp, $dst$$disp));
14848   %}
14849 
14850   ins_pipe(istore_reg_reg);
14851 
14852 %}
14853 
14854 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14855 
14856   match(Set dst (MoveF2I src));
14857 
14858   effect(DEF dst, USE src);
14859 
14860   ins_cost(INSN_COST);
14861 
14862   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14863 
14864   ins_encode %{
14865     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14866   %}
14867 
14868   ins_pipe(fp_f2i);
14869 
14870 %}
14871 
14872 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14873 
14874   match(Set dst (MoveI2F src));
14875 
14876   effect(DEF dst, USE src);
14877 
14878   ins_cost(INSN_COST);
14879 
14880   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14881 
14882   ins_encode %{
14883     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14884   %}
14885 
14886   ins_pipe(fp_i2f);
14887 
14888 %}
14889 
14890 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14891 
14892   match(Set dst (MoveD2L src));
14893 
14894   effect(DEF dst, USE src);
14895 
14896   ins_cost(INSN_COST);
14897 
14898   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14899 
14900   ins_encode %{
14901     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14902   %}
14903 
14904   ins_pipe(fp_d2l);
14905 
14906 %}
14907 
14908 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14909 
14910   match(Set dst (MoveL2D src));
14911 
14912   effect(DEF dst, USE src);
14913 
14914   ins_cost(INSN_COST);
14915 
14916   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14917 
14918   ins_encode %{
14919     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14920   %}
14921 
14922   ins_pipe(fp_l2d);
14923 
14924 %}
14925 
14926 // ============================================================================
14927 // clearing of an array
14928 
14929 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14930 %{
14931   match(Set dummy (ClearArray cnt base));
14932   effect(USE_KILL cnt, USE_KILL base, KILL cr);
14933 
14934   ins_cost(4 * INSN_COST);
14935   format %{ "ClearArray $cnt, $base" %}
14936 
14937   ins_encode %{
14938     address tpc = __ zero_words($base$$Register, $cnt$$Register);
14939     if (tpc == NULL) {
14940       ciEnv::current()->record_failure("CodeCache is full");
14941       return;
14942     }
14943   %}
14944 
14945   ins_pipe(pipe_class_memory);
14946 %}
14947 
14948 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14949 %{
14950   predicate((uint64_t)n->in(2)->get_long()
14951             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14952   match(Set dummy (ClearArray cnt base));
14953   effect(TEMP temp, USE_KILL base, KILL cr);
14954 
14955   ins_cost(4 * INSN_COST);
14956   format %{ "ClearArray $cnt, $base" %}
14957 
14958   ins_encode %{
14959     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14960     if (tpc == NULL) {
14961       ciEnv::current()->record_failure("CodeCache is full");
14962       return;
14963     }
14964   %}
14965 
14966   ins_pipe(pipe_class_memory);
14967 %}
14968 
14969 // ============================================================================
14970 // Overflow Math Instructions
14971 
14972 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14973 %{
14974   match(Set cr (OverflowAddI op1 op2));
14975 
14976   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
14977   ins_cost(INSN_COST);
14978   ins_encode %{
14979     __ cmnw($op1$$Register, $op2$$Register);
14980   %}
14981 
14982   ins_pipe(icmp_reg_reg);
14983 %}
14984 
14985 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14986 %{
14987   match(Set cr (OverflowAddI op1 op2));
14988 
14989   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
14990   ins_cost(INSN_COST);
14991   ins_encode %{
14992     __ cmnw($op1$$Register, $op2$$constant);
14993   %}
14994 
14995   ins_pipe(icmp_reg_imm);
14996 %}
14997 
14998 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14999 %{
15000   match(Set cr (OverflowAddL op1 op2));
15001 
15002   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15003   ins_cost(INSN_COST);
15004   ins_encode %{
15005     __ cmn($op1$$Register, $op2$$Register);
15006   %}
15007 
15008   ins_pipe(icmp_reg_reg);
15009 %}
15010 
15011 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15012 %{
15013   match(Set cr (OverflowAddL op1 op2));
15014 
15015   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
15016   ins_cost(INSN_COST);
15017   ins_encode %{
15018     __ adds(zr, $op1$$Register, $op2$$constant);
15019   %}
15020 
15021   ins_pipe(icmp_reg_imm);
15022 %}
15023 
15024 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15025 %{
15026   match(Set cr (OverflowSubI op1 op2));
15027 
15028   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15029   ins_cost(INSN_COST);
15030   ins_encode %{
15031     __ cmpw($op1$$Register, $op2$$Register);
15032   %}
15033 
15034   ins_pipe(icmp_reg_reg);
15035 %}
15036 
15037 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15038 %{
15039   match(Set cr (OverflowSubI op1 op2));
15040 
15041   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15042   ins_cost(INSN_COST);
15043   ins_encode %{
15044     __ cmpw($op1$$Register, $op2$$constant);
15045   %}
15046 
15047   ins_pipe(icmp_reg_imm);
15048 %}
15049 
15050 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15051 %{
15052   match(Set cr (OverflowSubL op1 op2));
15053 
15054   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15055   ins_cost(INSN_COST);
15056   ins_encode %{
15057     __ cmp($op1$$Register, $op2$$Register);
15058   %}
15059 
15060   ins_pipe(icmp_reg_reg);
15061 %}
15062 
15063 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15064 %{
15065   match(Set cr (OverflowSubL op1 op2));
15066 
15067   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15068   ins_cost(INSN_COST);
15069   ins_encode %{
15070     __ subs(zr, $op1$$Register, $op2$$constant);
15071   %}
15072 
15073   ins_pipe(icmp_reg_imm);
15074 %}
15075 
15076 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15077 %{
15078   match(Set cr (OverflowSubI zero op1));
15079 
15080   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15081   ins_cost(INSN_COST);
15082   ins_encode %{
15083     __ cmpw(zr, $op1$$Register);
15084   %}
15085 
15086   ins_pipe(icmp_reg_imm);
15087 %}
15088 
15089 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15090 %{
15091   match(Set cr (OverflowSubL zero op1));
15092 
15093   format %{ "cmp   zr, $op1\t# overflow check long" %}
15094   ins_cost(INSN_COST);
15095   ins_encode %{
15096     __ cmp(zr, $op1$$Register);
15097   %}
15098 
15099   ins_pipe(icmp_reg_imm);
15100 %}
15101 
15102 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15103 %{
15104   match(Set cr (OverflowMulI op1 op2));
15105 
15106   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15107             "cmp   rscratch1, rscratch1, sxtw\n\t"
15108             "movw  rscratch1, #0x80000000\n\t"
15109             "cselw rscratch1, rscratch1, zr, NE\n\t"
15110             "cmpw  rscratch1, #1" %}
15111   ins_cost(5 * INSN_COST);
15112   ins_encode %{
15113     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15114     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15115     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15116     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15117     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15118   %}
15119 
15120   ins_pipe(pipe_slow);
15121 %}
15122 
15123 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15124 %{
15125   match(If cmp (OverflowMulI op1 op2));
15126   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15127             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15128   effect(USE labl, KILL cr);
15129 
15130   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15131             "cmp   rscratch1, rscratch1, sxtw\n\t"
15132             "b$cmp   $labl" %}
15133   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15134   ins_encode %{
15135     Label* L = $labl$$label;
15136     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15137     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15138     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15139     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15140   %}
15141 
15142   ins_pipe(pipe_serial);
15143 %}
15144 
15145 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15146 %{
15147   match(Set cr (OverflowMulL op1 op2));
15148 
15149   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15150             "smulh rscratch2, $op1, $op2\n\t"
15151             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15152             "movw  rscratch1, #0x80000000\n\t"
15153             "cselw rscratch1, rscratch1, zr, NE\n\t"
15154             "cmpw  rscratch1, #1" %}
15155   ins_cost(6 * INSN_COST);
15156   ins_encode %{
15157     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15158     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15159     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15160     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15161     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15162     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15163   %}
15164 
15165   ins_pipe(pipe_slow);
15166 %}
15167 
15168 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15169 %{
15170   match(If cmp (OverflowMulL op1 op2));
15171   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15172             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15173   effect(USE labl, KILL cr);
15174 
15175   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15176             "smulh rscratch2, $op1, $op2\n\t"
15177             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15178             "b$cmp $labl" %}
15179   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15180   ins_encode %{
15181     Label* L = $labl$$label;
15182     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15183     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15184     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15185     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15186     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15187   %}
15188 
15189   ins_pipe(pipe_serial);
15190 %}
15191 
15192 // ============================================================================
15193 // Compare Instructions
15194 
15195 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15196 %{
15197   match(Set cr (CmpI op1 op2));
15198 
15199   effect(DEF cr, USE op1, USE op2);
15200 
15201   ins_cost(INSN_COST);
15202   format %{ "cmpw  $op1, $op2" %}
15203 
15204   ins_encode(aarch64_enc_cmpw(op1, op2));
15205 
15206   ins_pipe(icmp_reg_reg);
15207 %}
15208 
15209 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15210 %{
15211   match(Set cr (CmpI op1 zero));
15212 
15213   effect(DEF cr, USE op1);
15214 
15215   ins_cost(INSN_COST);
15216   format %{ "cmpw $op1, 0" %}
15217 
15218   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15219 
15220   ins_pipe(icmp_reg_imm);
15221 %}
15222 
15223 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15224 %{
15225   match(Set cr (CmpI op1 op2));
15226 
15227   effect(DEF cr, USE op1);
15228 
15229   ins_cost(INSN_COST);
15230   format %{ "cmpw  $op1, $op2" %}
15231 
15232   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15233 
15234   ins_pipe(icmp_reg_imm);
15235 %}
15236 
15237 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15238 %{
15239   match(Set cr (CmpI op1 op2));
15240 
15241   effect(DEF cr, USE op1);
15242 
15243   ins_cost(INSN_COST * 2);
15244   format %{ "cmpw  $op1, $op2" %}
15245 
15246   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15247 
15248   ins_pipe(icmp_reg_imm);
15249 %}
15250 
15251 // Unsigned compare Instructions; really, same as signed compare
15252 // except it should only be used to feed an If or a CMovI which takes a
15253 // cmpOpU.
15254 
15255 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15256 %{
15257   match(Set cr (CmpU op1 op2));
15258 
15259   effect(DEF cr, USE op1, USE op2);
15260 
15261   ins_cost(INSN_COST);
15262   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15263 
15264   ins_encode(aarch64_enc_cmpw(op1, op2));
15265 
15266   ins_pipe(icmp_reg_reg);
15267 %}
15268 
15269 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15270 %{
15271   match(Set cr (CmpU op1 zero));
15272 
15273   effect(DEF cr, USE op1);
15274 
15275   ins_cost(INSN_COST);
15276   format %{ "cmpw $op1, #0\t# unsigned" %}
15277 
15278   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15279 
15280   ins_pipe(icmp_reg_imm);
15281 %}
15282 
15283 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15284 %{
15285   match(Set cr (CmpU op1 op2));
15286 
15287   effect(DEF cr, USE op1);
15288 
15289   ins_cost(INSN_COST);
15290   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15291 
15292   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15293 
15294   ins_pipe(icmp_reg_imm);
15295 %}
15296 
15297 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15298 %{
15299   match(Set cr (CmpU op1 op2));
15300 
15301   effect(DEF cr, USE op1);
15302 
15303   ins_cost(INSN_COST * 2);
15304   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15305 
15306   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15307 
15308   ins_pipe(icmp_reg_imm);
15309 %}
15310 
15311 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15312 %{
15313   match(Set cr (CmpL op1 op2));
15314 
15315   effect(DEF cr, USE op1, USE op2);
15316 
15317   ins_cost(INSN_COST);
15318   format %{ "cmp  $op1, $op2" %}
15319 
15320   ins_encode(aarch64_enc_cmp(op1, op2));
15321 
15322   ins_pipe(icmp_reg_reg);
15323 %}
15324 
15325 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15326 %{
15327   match(Set cr (CmpL op1 zero));
15328 
15329   effect(DEF cr, USE op1);
15330 
15331   ins_cost(INSN_COST);
15332   format %{ "tst  $op1" %}
15333 
15334   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15335 
15336   ins_pipe(icmp_reg_imm);
15337 %}
15338 
15339 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15340 %{
15341   match(Set cr (CmpL op1 op2));
15342 
15343   effect(DEF cr, USE op1);
15344 
15345   ins_cost(INSN_COST);
15346   format %{ "cmp  $op1, $op2" %}
15347 
15348   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15349 
15350   ins_pipe(icmp_reg_imm);
15351 %}
15352 
15353 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15354 %{
15355   match(Set cr (CmpL op1 op2));
15356 
15357   effect(DEF cr, USE op1);
15358 
15359   ins_cost(INSN_COST * 2);
15360   format %{ "cmp  $op1, $op2" %}
15361 
15362   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15363 
15364   ins_pipe(icmp_reg_imm);
15365 %}
15366 
15367 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15368 %{
15369   match(Set cr (CmpUL op1 op2));
15370 
15371   effect(DEF cr, USE op1, USE op2);
15372 
15373   ins_cost(INSN_COST);
15374   format %{ "cmp  $op1, $op2" %}
15375 
15376   ins_encode(aarch64_enc_cmp(op1, op2));
15377 
15378   ins_pipe(icmp_reg_reg);
15379 %}
15380 
15381 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15382 %{
15383   match(Set cr (CmpUL op1 zero));
15384 
15385   effect(DEF cr, USE op1);
15386 
15387   ins_cost(INSN_COST);
15388   format %{ "tst  $op1" %}
15389 
15390   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15391 
15392   ins_pipe(icmp_reg_imm);
15393 %}
15394 
15395 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15396 %{
15397   match(Set cr (CmpUL op1 op2));
15398 
15399   effect(DEF cr, USE op1);
15400 
15401   ins_cost(INSN_COST);
15402   format %{ "cmp  $op1, $op2" %}
15403 
15404   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15405 
15406   ins_pipe(icmp_reg_imm);
15407 %}
15408 
15409 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15410 %{
15411   match(Set cr (CmpUL op1 op2));
15412 
15413   effect(DEF cr, USE op1);
15414 
15415   ins_cost(INSN_COST * 2);
15416   format %{ "cmp  $op1, $op2" %}
15417 
15418   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15419 
15420   ins_pipe(icmp_reg_imm);
15421 %}
15422 
15423 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15424 %{
15425   match(Set cr (CmpP op1 op2));
15426 
15427   effect(DEF cr, USE op1, USE op2);
15428 
15429   ins_cost(INSN_COST);
15430   format %{ "cmp  $op1, $op2\t // ptr" %}
15431 
15432   ins_encode(aarch64_enc_cmpp(op1, op2));
15433 
15434   ins_pipe(icmp_reg_reg);
15435 %}
15436 
15437 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15438 %{
15439   match(Set cr (CmpN op1 op2));
15440 
15441   effect(DEF cr, USE op1, USE op2);
15442 
15443   ins_cost(INSN_COST);
15444   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15445 
15446   ins_encode(aarch64_enc_cmpn(op1, op2));
15447 
15448   ins_pipe(icmp_reg_reg);
15449 %}
15450 
15451 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15452 %{
15453   match(Set cr (CmpP op1 zero));
15454 
15455   effect(DEF cr, USE op1, USE zero);
15456 
15457   ins_cost(INSN_COST);
15458   format %{ "cmp  $op1, 0\t // ptr" %}
15459 
15460   ins_encode(aarch64_enc_testp(op1));
15461 
15462   ins_pipe(icmp_reg_imm);
15463 %}
15464 
15465 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15466 %{
15467   match(Set cr (CmpN op1 zero));
15468 
15469   effect(DEF cr, USE op1, USE zero);
15470 
15471   ins_cost(INSN_COST);
15472   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15473 
15474   ins_encode(aarch64_enc_testn(op1));
15475 
15476   ins_pipe(icmp_reg_imm);
15477 %}
15478 
15479 // FP comparisons
15480 //
15481 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15482 // using normal cmpOp. See declaration of rFlagsReg for details.
15483 
15484 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15485 %{
15486   match(Set cr (CmpF src1 src2));
15487 
15488   ins_cost(3 * INSN_COST);
15489   format %{ "fcmps $src1, $src2" %}
15490 
15491   ins_encode %{
15492     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15493   %}
15494 
15495   ins_pipe(pipe_class_compare);
15496 %}
15497 
15498 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15499 %{
15500   match(Set cr (CmpF src1 src2));
15501 
15502   ins_cost(3 * INSN_COST);
15503   format %{ "fcmps $src1, 0.0" %}
15504 
15505   ins_encode %{
15506     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15507   %}
15508 
15509   ins_pipe(pipe_class_compare);
15510 %}
15511 // FROM HERE
15512 
15513 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15514 %{
15515   match(Set cr (CmpD src1 src2));
15516 
15517   ins_cost(3 * INSN_COST);
15518   format %{ "fcmpd $src1, $src2" %}
15519 
15520   ins_encode %{
15521     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15522   %}
15523 
15524   ins_pipe(pipe_class_compare);
15525 %}
15526 
15527 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15528 %{
15529   match(Set cr (CmpD src1 src2));
15530 
15531   ins_cost(3 * INSN_COST);
15532   format %{ "fcmpd $src1, 0.0" %}
15533 
15534   ins_encode %{
15535     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15536   %}
15537 
15538   ins_pipe(pipe_class_compare);
15539 %}
15540 
15541 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15542 %{
15543   match(Set dst (CmpF3 src1 src2));
15544   effect(KILL cr);
15545 
15546   ins_cost(5 * INSN_COST);
15547   format %{ "fcmps $src1, $src2\n\t"
15548             "csinvw($dst, zr, zr, eq\n\t"
15549             "csnegw($dst, $dst, $dst, lt)"
15550   %}
15551 
15552   ins_encode %{
15553     Label done;
15554     FloatRegister s1 = as_FloatRegister($src1$$reg);
15555     FloatRegister s2 = as_FloatRegister($src2$$reg);
15556     Register d = as_Register($dst$$reg);
15557     __ fcmps(s1, s2);
15558     // installs 0 if EQ else -1
15559     __ csinvw(d, zr, zr, Assembler::EQ);
15560     // keeps -1 if less or unordered else installs 1
15561     __ csnegw(d, d, d, Assembler::LT);
15562     __ bind(done);
15563   %}
15564 
15565   ins_pipe(pipe_class_default);
15566 
15567 %}
15568 
15569 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15570 %{
15571   match(Set dst (CmpD3 src1 src2));
15572   effect(KILL cr);
15573 
15574   ins_cost(5 * INSN_COST);
15575   format %{ "fcmpd $src1, $src2\n\t"
15576             "csinvw($dst, zr, zr, eq\n\t"
15577             "csnegw($dst, $dst, $dst, lt)"
15578   %}
15579 
15580   ins_encode %{
15581     Label done;
15582     FloatRegister s1 = as_FloatRegister($src1$$reg);
15583     FloatRegister s2 = as_FloatRegister($src2$$reg);
15584     Register d = as_Register($dst$$reg);
15585     __ fcmpd(s1, s2);
15586     // installs 0 if EQ else -1
15587     __ csinvw(d, zr, zr, Assembler::EQ);
15588     // keeps -1 if less or unordered else installs 1
15589     __ csnegw(d, d, d, Assembler::LT);
15590     __ bind(done);
15591   %}
15592   ins_pipe(pipe_class_default);
15593 
15594 %}
15595 
15596 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15597 %{
15598   match(Set dst (CmpF3 src1 zero));
15599   effect(KILL cr);
15600 
15601   ins_cost(5 * INSN_COST);
15602   format %{ "fcmps $src1, 0.0\n\t"
15603             "csinvw($dst, zr, zr, eq\n\t"
15604             "csnegw($dst, $dst, $dst, lt)"
15605   %}
15606 
15607   ins_encode %{
15608     Label done;
15609     FloatRegister s1 = as_FloatRegister($src1$$reg);
15610     Register d = as_Register($dst$$reg);
15611     __ fcmps(s1, 0.0);
15612     // installs 0 if EQ else -1
15613     __ csinvw(d, zr, zr, Assembler::EQ);
15614     // keeps -1 if less or unordered else installs 1
15615     __ csnegw(d, d, d, Assembler::LT);
15616     __ bind(done);
15617   %}
15618 
15619   ins_pipe(pipe_class_default);
15620 
15621 %}
15622 
15623 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15624 %{
15625   match(Set dst (CmpD3 src1 zero));
15626   effect(KILL cr);
15627 
15628   ins_cost(5 * INSN_COST);
15629   format %{ "fcmpd $src1, 0.0\n\t"
15630             "csinvw($dst, zr, zr, eq\n\t"
15631             "csnegw($dst, $dst, $dst, lt)"
15632   %}
15633 
15634   ins_encode %{
15635     Label done;
15636     FloatRegister s1 = as_FloatRegister($src1$$reg);
15637     Register d = as_Register($dst$$reg);
15638     __ fcmpd(s1, 0.0);
15639     // installs 0 if EQ else -1
15640     __ csinvw(d, zr, zr, Assembler::EQ);
15641     // keeps -1 if less or unordered else installs 1
15642     __ csnegw(d, d, d, Assembler::LT);
15643     __ bind(done);
15644   %}
15645   ins_pipe(pipe_class_default);
15646 
15647 %}
15648 
15649 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15650 %{
15651   match(Set dst (CmpLTMask p q));
15652   effect(KILL cr);
15653 
15654   ins_cost(3 * INSN_COST);
15655 
15656   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15657             "csetw $dst, lt\n\t"
15658             "subw $dst, zr, $dst"
15659   %}
15660 
15661   ins_encode %{
15662     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15663     __ csetw(as_Register($dst$$reg), Assembler::LT);
15664     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15665   %}
15666 
15667   ins_pipe(ialu_reg_reg);
15668 %}
15669 
15670 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15671 %{
15672   match(Set dst (CmpLTMask src zero));
15673   effect(KILL cr);
15674 
15675   ins_cost(INSN_COST);
15676 
15677   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15678 
15679   ins_encode %{
15680     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15681   %}
15682 
15683   ins_pipe(ialu_reg_shift);
15684 %}
15685 
15686 // ============================================================================
15687 // Max and Min
15688 
15689 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15690 %{
15691   effect( DEF dst, USE src1, USE src2, USE cr );
15692 
15693   ins_cost(INSN_COST * 2);
15694   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
15695 
15696   ins_encode %{
15697     __ cselw(as_Register($dst$$reg),
15698              as_Register($src1$$reg),
15699              as_Register($src2$$reg),
15700              Assembler::LT);
15701   %}
15702 
15703   ins_pipe(icond_reg_reg);
15704 %}
15705 
15706 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15707 %{
15708   match(Set dst (MinI src1 src2));
15709   ins_cost(INSN_COST * 3);
15710 
15711   expand %{
15712     rFlagsReg cr;
15713     compI_reg_reg(cr, src1, src2);
15714     cmovI_reg_reg_lt(dst, src1, src2, cr);
15715   %}
15716 
15717 %}
15718 // FROM HERE
15719 
15720 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15721 %{
15722   effect( DEF dst, USE src1, USE src2, USE cr );
15723 
15724   ins_cost(INSN_COST * 2);
15725   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
15726 
15727   ins_encode %{
15728     __ cselw(as_Register($dst$$reg),
15729              as_Register($src1$$reg),
15730              as_Register($src2$$reg),
15731              Assembler::GT);
15732   %}
15733 
15734   ins_pipe(icond_reg_reg);
15735 %}
15736 
15737 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15738 %{
15739   match(Set dst (MaxI src1 src2));
15740   ins_cost(INSN_COST * 3);
15741   expand %{
15742     rFlagsReg cr;
15743     compI_reg_reg(cr, src1, src2);
15744     cmovI_reg_reg_gt(dst, src1, src2, cr);
15745   %}
15746 %}
15747 
15748 // ============================================================================
15749 // Branch Instructions
15750 
15751 // Direct Branch.
15752 instruct branch(label lbl)
15753 %{
15754   match(Goto);
15755 
15756   effect(USE lbl);
15757 
15758   ins_cost(BRANCH_COST);
15759   format %{ "b  $lbl" %}
15760 
15761   ins_encode(aarch64_enc_b(lbl));
15762 
15763   ins_pipe(pipe_branch);
15764 %}
15765 
15766 // Conditional Near Branch
15767 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15768 %{
15769   // Same match rule as `branchConFar'.
15770   match(If cmp cr);
15771 
15772   effect(USE lbl);
15773 
15774   ins_cost(BRANCH_COST);
15775   // If set to 1 this indicates that the current instruction is a
15776   // short variant of a long branch. This avoids using this
15777   // instruction in first-pass matching. It will then only be used in
15778   // the `Shorten_branches' pass.
15779   // ins_short_branch(1);
15780   format %{ "b$cmp  $lbl" %}
15781 
15782   ins_encode(aarch64_enc_br_con(cmp, lbl));
15783 
15784   ins_pipe(pipe_branch_cond);
15785 %}
15786 
15787 // Conditional Near Branch Unsigned
15788 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15789 %{
15790   // Same match rule as `branchConFar'.
15791   match(If cmp cr);
15792 
15793   effect(USE lbl);
15794 
15795   ins_cost(BRANCH_COST);
15796   // If set to 1 this indicates that the current instruction is a
15797   // short variant of a long branch. This avoids using this
15798   // instruction in first-pass matching. It will then only be used in
15799   // the `Shorten_branches' pass.
15800   // ins_short_branch(1);
15801   format %{ "b$cmp  $lbl\t# unsigned" %}
15802 
15803   ins_encode(aarch64_enc_br_conU(cmp, lbl));
15804 
15805   ins_pipe(pipe_branch_cond);
15806 %}
15807 
15808 // Make use of CBZ and CBNZ.  These instructions, as well as being
15809 // shorter than (cmp; branch), have the additional benefit of not
15810 // killing the flags.
15811 
15812 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15813   match(If cmp (CmpI op1 op2));
15814   effect(USE labl);
15815 
15816   ins_cost(BRANCH_COST);
15817   format %{ "cbw$cmp   $op1, $labl" %}
15818   ins_encode %{
15819     Label* L = $labl$$label;
15820     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15821     if (cond == Assembler::EQ)
15822       __ cbzw($op1$$Register, *L);
15823     else
15824       __ cbnzw($op1$$Register, *L);
15825   %}
15826   ins_pipe(pipe_cmp_branch);
15827 %}
15828 
15829 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15830   match(If cmp (CmpL op1 op2));
15831   effect(USE labl);
15832 
15833   ins_cost(BRANCH_COST);
15834   format %{ "cb$cmp   $op1, $labl" %}
15835   ins_encode %{
15836     Label* L = $labl$$label;
15837     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15838     if (cond == Assembler::EQ)
15839       __ cbz($op1$$Register, *L);
15840     else
15841       __ cbnz($op1$$Register, *L);
15842   %}
15843   ins_pipe(pipe_cmp_branch);
15844 %}
15845 
15846 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15847   match(If cmp (CmpP op1 op2));
15848   effect(USE labl);
15849 
15850   ins_cost(BRANCH_COST);
15851   format %{ "cb$cmp   $op1, $labl" %}
15852   ins_encode %{
15853     Label* L = $labl$$label;
15854     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15855     if (cond == Assembler::EQ)
15856       __ cbz($op1$$Register, *L);
15857     else
15858       __ cbnz($op1$$Register, *L);
15859   %}
15860   ins_pipe(pipe_cmp_branch);
15861 %}
15862 
15863 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15864   match(If cmp (CmpN op1 op2));
15865   effect(USE labl);
15866 
15867   ins_cost(BRANCH_COST);
15868   format %{ "cbw$cmp   $op1, $labl" %}
15869   ins_encode %{
15870     Label* L = $labl$$label;
15871     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15872     if (cond == Assembler::EQ)
15873       __ cbzw($op1$$Register, *L);
15874     else
15875       __ cbnzw($op1$$Register, *L);
15876   %}
15877   ins_pipe(pipe_cmp_branch);
15878 %}
15879 
15880 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15881   match(If cmp (CmpP (DecodeN oop) zero));
15882   effect(USE labl);
15883 
15884   ins_cost(BRANCH_COST);
15885   format %{ "cb$cmp   $oop, $labl" %}
15886   ins_encode %{
15887     Label* L = $labl$$label;
15888     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15889     if (cond == Assembler::EQ)
15890       __ cbzw($oop$$Register, *L);
15891     else
15892       __ cbnzw($oop$$Register, *L);
15893   %}
15894   ins_pipe(pipe_cmp_branch);
15895 %}
15896 
15897 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
15898   match(If cmp (CmpU op1 op2));
15899   effect(USE labl);
15900 
15901   ins_cost(BRANCH_COST);
15902   format %{ "cbw$cmp   $op1, $labl" %}
15903   ins_encode %{
15904     Label* L = $labl$$label;
15905     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15906     if (cond == Assembler::EQ || cond == Assembler::LS)
15907       __ cbzw($op1$$Register, *L);
15908     else
15909       __ cbnzw($op1$$Register, *L);
15910   %}
15911   ins_pipe(pipe_cmp_branch);
15912 %}
15913 
15914 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
15915   match(If cmp (CmpUL op1 op2));
15916   effect(USE labl);
15917 
15918   ins_cost(BRANCH_COST);
15919   format %{ "cb$cmp   $op1, $labl" %}
15920   ins_encode %{
15921     Label* L = $labl$$label;
15922     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15923     if (cond == Assembler::EQ || cond == Assembler::LS)
15924       __ cbz($op1$$Register, *L);
15925     else
15926       __ cbnz($op1$$Register, *L);
15927   %}
15928   ins_pipe(pipe_cmp_branch);
15929 %}
15930 
15931 // Test bit and Branch
15932 
15933 // Patterns for short (< 32KiB) variants
15934 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15935   match(If cmp (CmpL op1 op2));
15936   effect(USE labl);
15937 
15938   ins_cost(BRANCH_COST);
15939   format %{ "cb$cmp   $op1, $labl # long" %}
15940   ins_encode %{
15941     Label* L = $labl$$label;
15942     Assembler::Condition cond =
15943       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15944     __ tbr(cond, $op1$$Register, 63, *L);
15945   %}
15946   ins_pipe(pipe_cmp_branch);
15947   ins_short_branch(1);
15948 %}
15949 
15950 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15951   match(If cmp (CmpI op1 op2));
15952   effect(USE labl);
15953 
15954   ins_cost(BRANCH_COST);
15955   format %{ "cb$cmp   $op1, $labl # int" %}
15956   ins_encode %{
15957     Label* L = $labl$$label;
15958     Assembler::Condition cond =
15959       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15960     __ tbr(cond, $op1$$Register, 31, *L);
15961   %}
15962   ins_pipe(pipe_cmp_branch);
15963   ins_short_branch(1);
15964 %}
15965 
15966 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15967   match(If cmp (CmpL (AndL op1 op2) op3));
15968   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15969   effect(USE labl);
15970 
15971   ins_cost(BRANCH_COST);
15972   format %{ "tb$cmp   $op1, $op2, $labl" %}
15973   ins_encode %{
15974     Label* L = $labl$$label;
15975     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15976     int bit = exact_log2_long($op2$$constant);
15977     __ tbr(cond, $op1$$Register, bit, *L);
15978   %}
15979   ins_pipe(pipe_cmp_branch);
15980   ins_short_branch(1);
15981 %}
15982 
15983 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15984   match(If cmp (CmpI (AndI op1 op2) op3));
15985   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15986   effect(USE labl);
15987 
15988   ins_cost(BRANCH_COST);
15989   format %{ "tb$cmp   $op1, $op2, $labl" %}
15990   ins_encode %{
15991     Label* L = $labl$$label;
15992     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15993     int bit = exact_log2((juint)$op2$$constant);
15994     __ tbr(cond, $op1$$Register, bit, *L);
15995   %}
15996   ins_pipe(pipe_cmp_branch);
15997   ins_short_branch(1);
15998 %}
15999 
16000 // And far variants
16001 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16002   match(If cmp (CmpL op1 op2));
16003   effect(USE labl);
16004 
16005   ins_cost(BRANCH_COST);
16006   format %{ "cb$cmp   $op1, $labl # long" %}
16007   ins_encode %{
16008     Label* L = $labl$$label;
16009     Assembler::Condition cond =
16010       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16011     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16012   %}
16013   ins_pipe(pipe_cmp_branch);
16014 %}
16015 
16016 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16017   match(If cmp (CmpI op1 op2));
16018   effect(USE labl);
16019 
16020   ins_cost(BRANCH_COST);
16021   format %{ "cb$cmp   $op1, $labl # int" %}
16022   ins_encode %{
16023     Label* L = $labl$$label;
16024     Assembler::Condition cond =
16025       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16026     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16027   %}
16028   ins_pipe(pipe_cmp_branch);
16029 %}
16030 
16031 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16032   match(If cmp (CmpL (AndL op1 op2) op3));
16033   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16034   effect(USE labl);
16035 
16036   ins_cost(BRANCH_COST);
16037   format %{ "tb$cmp   $op1, $op2, $labl" %}
16038   ins_encode %{
16039     Label* L = $labl$$label;
16040     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16041     int bit = exact_log2_long($op2$$constant);
16042     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16043   %}
16044   ins_pipe(pipe_cmp_branch);
16045 %}
16046 
16047 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16048   match(If cmp (CmpI (AndI op1 op2) op3));
16049   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16050   effect(USE labl);
16051 
16052   ins_cost(BRANCH_COST);
16053   format %{ "tb$cmp   $op1, $op2, $labl" %}
16054   ins_encode %{
16055     Label* L = $labl$$label;
16056     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16057     int bit = exact_log2((juint)$op2$$constant);
16058     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16059   %}
16060   ins_pipe(pipe_cmp_branch);
16061 %}
16062 
16063 // Test bits
16064 
16065 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16066   match(Set cr (CmpL (AndL op1 op2) op3));
16067   predicate(Assembler::operand_valid_for_logical_immediate
16068             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16069 
16070   ins_cost(INSN_COST);
16071   format %{ "tst $op1, $op2 # long" %}
16072   ins_encode %{
16073     __ tst($op1$$Register, $op2$$constant);
16074   %}
16075   ins_pipe(ialu_reg_reg);
16076 %}
16077 
16078 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16079   match(Set cr (CmpI (AndI op1 op2) op3));
16080   predicate(Assembler::operand_valid_for_logical_immediate
16081             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16082 
16083   ins_cost(INSN_COST);
16084   format %{ "tst $op1, $op2 # int" %}
16085   ins_encode %{
16086     __ tstw($op1$$Register, $op2$$constant);
16087   %}
16088   ins_pipe(ialu_reg_reg);
16089 %}
16090 
16091 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16092   match(Set cr (CmpL (AndL op1 op2) op3));
16093 
16094   ins_cost(INSN_COST);
16095   format %{ "tst $op1, $op2 # long" %}
16096   ins_encode %{
16097     __ tst($op1$$Register, $op2$$Register);
16098   %}
16099   ins_pipe(ialu_reg_reg);
16100 %}
16101 
16102 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16103   match(Set cr (CmpI (AndI op1 op2) op3));
16104 
16105   ins_cost(INSN_COST);
16106   format %{ "tstw $op1, $op2 # int" %}
16107   ins_encode %{
16108     __ tstw($op1$$Register, $op2$$Register);
16109   %}
16110   ins_pipe(ialu_reg_reg);
16111 %}
16112 
16113 
16114 // Conditional Far Branch
16115 // Conditional Far Branch Unsigned
16116 // TODO: fixme
16117 
16118 // counted loop end branch near
16119 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16120 %{
16121   match(CountedLoopEnd cmp cr);
16122 
16123   effect(USE lbl);
16124 
16125   ins_cost(BRANCH_COST);
16126   // short variant.
16127   // ins_short_branch(1);
16128   format %{ "b$cmp $lbl \t// counted loop end" %}
16129 
16130   ins_encode(aarch64_enc_br_con(cmp, lbl));
16131 
16132   ins_pipe(pipe_branch);
16133 %}
16134 
16135 // counted loop end branch near Unsigned
16136 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16137 %{
16138   match(CountedLoopEnd cmp cr);
16139 
16140   effect(USE lbl);
16141 
16142   ins_cost(BRANCH_COST);
16143   // short variant.
16144   // ins_short_branch(1);
16145   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16146 
16147   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16148 
16149   ins_pipe(pipe_branch);
16150 %}
16151 
16152 // counted loop end branch far
16153 // counted loop end branch far unsigned
16154 // TODO: fixme
16155 
16156 // ============================================================================
16157 // inlined locking and unlocking
16158 
16159 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16160 %{
16161   match(Set cr (FastLock object box));
16162   effect(TEMP tmp, TEMP tmp2);
16163 
16164   // TODO
16165   // identify correct cost
16166   ins_cost(5 * INSN_COST);
16167   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16168 
16169   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16170 
16171   ins_pipe(pipe_serial);
16172 %}
16173 
16174 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16175 %{
16176   match(Set cr (FastUnlock object box));
16177   effect(TEMP tmp, TEMP tmp2);
16178 
16179   ins_cost(5 * INSN_COST);
16180   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16181 
16182   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16183 
16184   ins_pipe(pipe_serial);
16185 %}
16186 
16187 
16188 // ============================================================================
16189 // Safepoint Instructions
16190 
16191 // TODO
16192 // provide a near and far version of this code
16193 
16194 instruct safePoint(rFlagsReg cr, iRegP poll)
16195 %{
16196   match(SafePoint poll);
16197   effect(KILL cr);
16198 
16199   format %{
16200     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16201   %}
16202   ins_encode %{
16203     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16204   %}
16205   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16206 %}
16207 
16208 
16209 // ============================================================================
16210 // Procedure Call/Return Instructions
16211 
16212 // Call Java Static Instruction
16213 
16214 instruct CallStaticJavaDirect(method meth)
16215 %{
16216   match(CallStaticJava);
16217 
16218   effect(USE meth);
16219 
16220   ins_cost(CALL_COST);
16221 
16222   format %{ "call,static $meth \t// ==> " %}
16223 
16224   ins_encode(aarch64_enc_java_static_call(meth),
16225              aarch64_enc_call_epilog);
16226 
16227   ins_pipe(pipe_class_call);
16228 %}
16229 
16230 // TO HERE
16231 
16232 // Call Java Dynamic Instruction
16233 instruct CallDynamicJavaDirect(method meth)
16234 %{
16235   match(CallDynamicJava);
16236 
16237   effect(USE meth);
16238 
16239   ins_cost(CALL_COST);
16240 
16241   format %{ "CALL,dynamic $meth \t// ==> " %}
16242 
16243   ins_encode(aarch64_enc_java_dynamic_call(meth),
16244              aarch64_enc_call_epilog);
16245 
16246   ins_pipe(pipe_class_call);
16247 %}
16248 
16249 // Call Runtime Instruction
16250 
16251 instruct CallRuntimeDirect(method meth)
16252 %{
16253   match(CallRuntime);
16254 
16255   effect(USE meth);
16256 
16257   ins_cost(CALL_COST);
16258 
16259   format %{ "CALL, runtime $meth" %}
16260 
16261   ins_encode( aarch64_enc_java_to_runtime(meth) );
16262 
16263   ins_pipe(pipe_class_call);
16264 %}
16265 
16266 // Call Runtime Instruction
16267 
16268 instruct CallLeafDirect(method meth)
16269 %{
16270   match(CallLeaf);
16271 
16272   effect(USE meth);
16273 
16274   ins_cost(CALL_COST);
16275 
16276   format %{ "CALL, runtime leaf $meth" %}
16277 
16278   ins_encode( aarch64_enc_java_to_runtime(meth) );
16279 
16280   ins_pipe(pipe_class_call);
16281 %}
16282 
16283 // Call Runtime Instruction
16284 
16285 instruct CallLeafNoFPDirect(method meth)
16286 %{
16287   match(CallLeafNoFP);
16288 
16289   effect(USE meth);
16290 
16291   ins_cost(CALL_COST);
16292 
16293   format %{ "CALL, runtime leaf nofp $meth" %}
16294 
16295   ins_encode( aarch64_enc_java_to_runtime(meth) );
16296 
16297   ins_pipe(pipe_class_call);
16298 %}
16299 
16300 // Tail Call; Jump from runtime stub to Java code.
16301 // Also known as an 'interprocedural jump'.
16302 // Target of jump will eventually return to caller.
16303 // TailJump below removes the return address.
16304 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16305 %{
16306   match(TailCall jump_target method_ptr);
16307 
16308   ins_cost(CALL_COST);
16309 
16310   format %{ "br $jump_target\t# $method_ptr holds method" %}
16311 
16312   ins_encode(aarch64_enc_tail_call(jump_target));
16313 
16314   ins_pipe(pipe_class_call);
16315 %}
16316 
16317 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16318 %{
16319   match(TailJump jump_target ex_oop);
16320 
16321   ins_cost(CALL_COST);
16322 
16323   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16324 
16325   ins_encode(aarch64_enc_tail_jmp(jump_target));
16326 
16327   ins_pipe(pipe_class_call);
16328 %}
16329 
16330 // Create exception oop: created by stack-crawling runtime code.
16331 // Created exception is now available to this handler, and is setup
16332 // just prior to jumping to this handler. No code emitted.
16333 // TODO check
16334 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16335 instruct CreateException(iRegP_R0 ex_oop)
16336 %{
16337   match(Set ex_oop (CreateEx));
16338 
16339   format %{ " -- \t// exception oop; no code emitted" %}
16340 
16341   size(0);
16342 
16343   ins_encode( /*empty*/ );
16344 
16345   ins_pipe(pipe_class_empty);
16346 %}
16347 
16348 // Rethrow exception: The exception oop will come in the first
16349 // argument position. Then JUMP (not call) to the rethrow stub code.
16350 instruct RethrowException() %{
16351   match(Rethrow);
16352   ins_cost(CALL_COST);
16353 
16354   format %{ "b rethrow_stub" %}
16355 
16356   ins_encode( aarch64_enc_rethrow() );
16357 
16358   ins_pipe(pipe_class_call);
16359 %}
16360 
16361 
16362 // Return Instruction
16363 // epilog node loads ret address into lr as part of frame pop
16364 instruct Ret()
16365 %{
16366   match(Return);
16367 
16368   format %{ "ret\t// return register" %}
16369 
16370   ins_encode( aarch64_enc_ret() );
16371 
16372   ins_pipe(pipe_branch);
16373 %}
16374 
16375 // Die now.
16376 instruct ShouldNotReachHere() %{
16377   match(Halt);
16378 
16379   ins_cost(CALL_COST);
16380   format %{ "ShouldNotReachHere" %}
16381 
16382   ins_encode %{
16383     if (is_reachable()) {
16384       __ stop(_halt_reason);
16385     }
16386   %}
16387 
16388   ins_pipe(pipe_class_default);
16389 %}
16390 
16391 // ============================================================================
16392 // Partial Subtype Check
16393 //
16394 // superklass array for an instance of the superklass.  Set a hidden
16395 // internal cache on a hit (cache is checked with exposed code in
16396 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16397 // encoding ALSO sets flags.
16398 
16399 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16400 %{
16401   match(Set result (PartialSubtypeCheck sub super));
16402   effect(KILL cr, KILL temp);
16403 
16404   ins_cost(1100);  // slightly larger than the next version
16405   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16406 
16407   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16408 
16409   opcode(0x1); // Force zero of result reg on hit
16410 
16411   ins_pipe(pipe_class_memory);
16412 %}
16413 
16414 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16415 %{
16416   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16417   effect(KILL temp, KILL result);
16418 
16419   ins_cost(1100);  // slightly larger than the next version
16420   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16421 
16422   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16423 
16424   opcode(0x0); // Don't zero result reg on hit
16425 
16426   ins_pipe(pipe_class_memory);
16427 %}
16428 
16429 // Intrisics for String.compareTo()
16430 
16431 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16432                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16433 %{
16434   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16435   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16436   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16437 
16438   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16439   ins_encode %{
16440     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16441     __ string_compare($str1$$Register, $str2$$Register,
16442                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16443                       $tmp1$$Register, $tmp2$$Register,
16444                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16445   %}
16446   ins_pipe(pipe_class_memory);
16447 %}
16448 
16449 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16450                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16451 %{
16452   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16453   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16454   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16455 
16456   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16457   ins_encode %{
16458     __ string_compare($str1$$Register, $str2$$Register,
16459                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16460                       $tmp1$$Register, $tmp2$$Register,
16461                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16462   %}
16463   ins_pipe(pipe_class_memory);
16464 %}
16465 
16466 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16467                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16468                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16469 %{
16470   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16471   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16472   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16473          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16474 
16475   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16476   ins_encode %{
16477     __ string_compare($str1$$Register, $str2$$Register,
16478                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16479                       $tmp1$$Register, $tmp2$$Register,
16480                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16481                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16482   %}
16483   ins_pipe(pipe_class_memory);
16484 %}
16485 
16486 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16487                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16488                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16489 %{
16490   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16491   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16492   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16493          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16494 
16495   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16496   ins_encode %{
16497     __ string_compare($str1$$Register, $str2$$Register,
16498                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16499                       $tmp1$$Register, $tmp2$$Register,
16500                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16501                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16502   %}
16503   ins_pipe(pipe_class_memory);
16504 %}
16505 
16506 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16507 // these string_compare variants as NEON register type for convenience so that the prototype of
16508 // string_compare can be shared with all variants.
16509 
16510 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16511                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16512                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16513                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16514 %{
16515   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16516   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16517   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16518          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16519 
16520   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16521   ins_encode %{
16522     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16523     __ string_compare($str1$$Register, $str2$$Register,
16524                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16525                       $tmp1$$Register, $tmp2$$Register,
16526                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16527                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16528                       StrIntrinsicNode::LL);
16529   %}
16530   ins_pipe(pipe_class_memory);
16531 %}
16532 
16533 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16534                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16535                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16536                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16537 %{
16538   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16539   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16540   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16541          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16542 
16543   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16544   ins_encode %{
16545     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16546     __ string_compare($str1$$Register, $str2$$Register,
16547                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16548                       $tmp1$$Register, $tmp2$$Register,
16549                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16550                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16551                       StrIntrinsicNode::LU);
16552   %}
16553   ins_pipe(pipe_class_memory);
16554 %}
16555 
16556 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16557                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16558                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16559                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16560 %{
16561   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16562   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16563   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16564          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16565 
16566   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16567   ins_encode %{
16568     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16569     __ string_compare($str1$$Register, $str2$$Register,
16570                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16571                       $tmp1$$Register, $tmp2$$Register,
16572                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16573                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16574                       StrIntrinsicNode::UL);
16575   %}
16576   ins_pipe(pipe_class_memory);
16577 %}
16578 
16579 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16580                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16581                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16582                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16583 %{
16584   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16585   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16586   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16587          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16588 
16589   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16590   ins_encode %{
16591     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16592     __ string_compare($str1$$Register, $str2$$Register,
16593                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16594                       $tmp1$$Register, $tmp2$$Register,
16595                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16596                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16597                       StrIntrinsicNode::UU);
16598   %}
16599   ins_pipe(pipe_class_memory);
16600 %}
16601 
16602 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16603        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16604        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16605 %{
16606   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16607   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16608   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16609          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16610   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16611 
16612   ins_encode %{
16613     __ string_indexof($str1$$Register, $str2$$Register,
16614                       $cnt1$$Register, $cnt2$$Register,
16615                       $tmp1$$Register, $tmp2$$Register,
16616                       $tmp3$$Register, $tmp4$$Register,
16617                       $tmp5$$Register, $tmp6$$Register,
16618                       -1, $result$$Register, StrIntrinsicNode::UU);
16619   %}
16620   ins_pipe(pipe_class_memory);
16621 %}
16622 
16623 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16624        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16625        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16626 %{
16627   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16628   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16629   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16630          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16631   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16632 
16633   ins_encode %{
16634     __ string_indexof($str1$$Register, $str2$$Register,
16635                       $cnt1$$Register, $cnt2$$Register,
16636                       $tmp1$$Register, $tmp2$$Register,
16637                       $tmp3$$Register, $tmp4$$Register,
16638                       $tmp5$$Register, $tmp6$$Register,
16639                       -1, $result$$Register, StrIntrinsicNode::LL);
16640   %}
16641   ins_pipe(pipe_class_memory);
16642 %}
16643 
16644 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16645        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16646        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16647 %{
16648   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16649   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16650   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16651          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16652   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
16653 
16654   ins_encode %{
16655     __ string_indexof($str1$$Register, $str2$$Register,
16656                       $cnt1$$Register, $cnt2$$Register,
16657                       $tmp1$$Register, $tmp2$$Register,
16658                       $tmp3$$Register, $tmp4$$Register,
16659                       $tmp5$$Register, $tmp6$$Register,
16660                       -1, $result$$Register, StrIntrinsicNode::UL);
16661   %}
16662   ins_pipe(pipe_class_memory);
16663 %}
16664 
16665 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16666                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16667                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16668 %{
16669   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16670   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16671   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16672          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16673   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
16674 
16675   ins_encode %{
16676     int icnt2 = (int)$int_cnt2$$constant;
16677     __ string_indexof($str1$$Register, $str2$$Register,
16678                       $cnt1$$Register, zr,
16679                       $tmp1$$Register, $tmp2$$Register,
16680                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16681                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16682   %}
16683   ins_pipe(pipe_class_memory);
16684 %}
16685 
16686 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16687                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16688                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16689 %{
16690   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16691   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16692   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16693          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16694   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
16695 
16696   ins_encode %{
16697     int icnt2 = (int)$int_cnt2$$constant;
16698     __ string_indexof($str1$$Register, $str2$$Register,
16699                       $cnt1$$Register, zr,
16700                       $tmp1$$Register, $tmp2$$Register,
16701                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16702                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16703   %}
16704   ins_pipe(pipe_class_memory);
16705 %}
16706 
16707 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16708                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16709                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16710 %{
16711   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16712   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16713   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16714          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16715   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
16716 
16717   ins_encode %{
16718     int icnt2 = (int)$int_cnt2$$constant;
16719     __ string_indexof($str1$$Register, $str2$$Register,
16720                       $cnt1$$Register, zr,
16721                       $tmp1$$Register, $tmp2$$Register,
16722                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16723                       icnt2, $result$$Register, StrIntrinsicNode::UL);
16724   %}
16725   ins_pipe(pipe_class_memory);
16726 %}
16727 
16728 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16729                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16730                              iRegINoSp tmp3, rFlagsReg cr)
16731 %{
16732   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16733   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16734   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16735          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16736 
16737   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16738 
16739   ins_encode %{
16740     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16741                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16742                            $tmp3$$Register);
16743   %}
16744   ins_pipe(pipe_class_memory);
16745 %}
16746 
16747 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16748                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16749                               iRegINoSp tmp3, rFlagsReg cr)
16750 %{
16751   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16752   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16753   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16754          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16755 
16756   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16757 
16758   ins_encode %{
16759     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16760                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
16761                             $tmp3$$Register);
16762   %}
16763   ins_pipe(pipe_class_memory);
16764 %}
16765 
16766 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16767                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16768                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16769   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16770   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16771   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16772   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16773   ins_encode %{
16774     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16775                                $result$$Register, $ztmp1$$FloatRegister,
16776                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16777                                $ptmp$$PRegister, true /* isL */);
16778   %}
16779   ins_pipe(pipe_class_memory);
16780 %}
16781 
16782 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16783                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16784                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16785   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16786   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16787   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16788   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16789   ins_encode %{
16790     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16791                                $result$$Register, $ztmp1$$FloatRegister,
16792                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16793                                $ptmp$$PRegister, false /* isL */);
16794   %}
16795   ins_pipe(pipe_class_memory);
16796 %}
16797 
16798 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16799                         iRegI_R0 result, rFlagsReg cr)
16800 %{
16801   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16802   match(Set result (StrEquals (Binary str1 str2) cnt));
16803   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16804 
16805   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16806   ins_encode %{
16807     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16808     __ string_equals($str1$$Register, $str2$$Register,
16809                      $result$$Register, $cnt$$Register, 1);
16810   %}
16811   ins_pipe(pipe_class_memory);
16812 %}
16813 
16814 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16815                         iRegI_R0 result, rFlagsReg cr)
16816 %{
16817   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
16818   match(Set result (StrEquals (Binary str1 str2) cnt));
16819   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16820 
16821   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16822   ins_encode %{
16823     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16824     __ string_equals($str1$$Register, $str2$$Register,
16825                      $result$$Register, $cnt$$Register, 2);
16826   %}
16827   ins_pipe(pipe_class_memory);
16828 %}
16829 
16830 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16831                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16832                        iRegP_R10 tmp, rFlagsReg cr)
16833 %{
16834   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16835   match(Set result (AryEq ary1 ary2));
16836   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16837 
16838   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16839   ins_encode %{
16840     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16841                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16842                                    $result$$Register, $tmp$$Register, 1);
16843     if (tpc == NULL) {
16844       ciEnv::current()->record_failure("CodeCache is full");
16845       return;
16846     }
16847   %}
16848   ins_pipe(pipe_class_memory);
16849 %}
16850 
16851 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16852                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16853                        iRegP_R10 tmp, rFlagsReg cr)
16854 %{
16855   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16856   match(Set result (AryEq ary1 ary2));
16857   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16858 
16859   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16860   ins_encode %{
16861     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16862                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16863                                    $result$$Register, $tmp$$Register, 2);
16864     if (tpc == NULL) {
16865       ciEnv::current()->record_failure("CodeCache is full");
16866       return;
16867     }
16868   %}
16869   ins_pipe(pipe_class_memory);
16870 %}
16871 
16872 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16873 %{
16874   match(Set result (CountPositives ary1 len));
16875   effect(USE_KILL ary1, USE_KILL len, KILL cr);
16876   format %{ "count positives byte[] $ary1,$len -> $result" %}
16877   ins_encode %{
16878     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16879     if (tpc == NULL) {
16880       ciEnv::current()->record_failure("CodeCache is full");
16881       return;
16882     }
16883   %}
16884   ins_pipe( pipe_slow );
16885 %}
16886 
16887 // fast char[] to byte[] compression
16888 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16889                          vRegD_V0 tmp1, vRegD_V1 tmp2,
16890                          vRegD_V2 tmp3, vRegD_V3 tmp4,
16891                          iRegI_R0 result, rFlagsReg cr)
16892 %{
16893   match(Set result (StrCompressedCopy src (Binary dst len)));
16894   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4,
16895          USE_KILL src, USE_KILL dst, USE len, KILL cr);
16896 
16897   format %{ "String Compress $src,$dst,$len -> $result  // KILL $src,$dst" %}
16898   ins_encode %{
16899     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16900                            $result$$Register,
16901                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16902                            $tmp3$$FloatRegister, $tmp4$$FloatRegister);
16903   %}
16904   ins_pipe(pipe_slow);
16905 %}
16906 
16907 // fast byte[] to char[] inflation
16908 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
16909                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
16910 %{
16911   match(Set dummy (StrInflatedCopy src (Binary dst len)));
16912   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16913 
16914   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
16915   ins_encode %{
16916     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16917                                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16918                                         $tmp3$$FloatRegister, $tmp4$$Register);
16919     if (tpc == NULL) {
16920       ciEnv::current()->record_failure("CodeCache is full");
16921       return;
16922     }
16923   %}
16924   ins_pipe(pipe_class_memory);
16925 %}
16926 
16927 // encode char[] to byte[] in ISO_8859_1
16928 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16929                           vRegD_V0 vtmp0, vRegD_V1 vtmp1,
16930                           vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16931                           iRegI_R0 result, rFlagsReg cr)
16932 %{
16933   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16934   match(Set result (EncodeISOArray src (Binary dst len)));
16935   effect(USE_KILL src, USE_KILL dst, USE len,
16936          KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
16937 
16938   format %{ "Encode ISO array $src,$dst,$len -> $result" %}
16939   ins_encode %{
16940     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16941                         $result$$Register, false,
16942                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16943                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
16944   %}
16945   ins_pipe(pipe_class_memory);
16946 %}
16947 
16948 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16949                             vRegD_V0 vtmp0, vRegD_V1 vtmp1,
16950                             vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16951                             iRegI_R0 result, rFlagsReg cr)
16952 %{
16953   predicate(((EncodeISOArrayNode*)n)->is_ascii());
16954   match(Set result (EncodeISOArray src (Binary dst len)));
16955   effect(USE_KILL src, USE_KILL dst, USE len,
16956          KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
16957 
16958   format %{ "Encode ASCII array $src,$dst,$len -> $result" %}
16959   ins_encode %{
16960     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16961                         $result$$Register, true,
16962                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16963                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
16964   %}
16965   ins_pipe(pipe_class_memory);
16966 %}
16967 
16968 // ============================================================================
16969 // This name is KNOWN by the ADLC and cannot be changed.
16970 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16971 // for this guy.
16972 instruct tlsLoadP(thread_RegP dst)
16973 %{
16974   match(Set dst (ThreadLocal));
16975 
16976   ins_cost(0);
16977 
16978   format %{ " -- \t// $dst=Thread::current(), empty" %}
16979 
16980   size(0);
16981 
16982   ins_encode( /*empty*/ );
16983 
16984   ins_pipe(pipe_class_empty);
16985 %}
16986 
16987 //----------PEEPHOLE RULES-----------------------------------------------------
16988 // These must follow all instruction definitions as they use the names
16989 // defined in the instructions definitions.
16990 //
16991 // peepmatch ( root_instr_name [preceding_instruction]* );
16992 //
16993 // peepconstraint %{
16994 // (instruction_number.operand_name relational_op instruction_number.operand_name
16995 //  [, ...] );
16996 // // instruction numbers are zero-based using left to right order in peepmatch
16997 //
16998 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
16999 // // provide an instruction_number.operand_name for each operand that appears
17000 // // in the replacement instruction's match rule
17001 //
17002 // ---------VM FLAGS---------------------------------------------------------
17003 //
17004 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17005 //
17006 // Each peephole rule is given an identifying number starting with zero and
17007 // increasing by one in the order seen by the parser.  An individual peephole
17008 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17009 // on the command-line.
17010 //
17011 // ---------CURRENT LIMITATIONS----------------------------------------------
17012 //
17013 // Only match adjacent instructions in same basic block
17014 // Only equality constraints
17015 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17016 // Only one replacement instruction
17017 //
17018 // ---------EXAMPLE----------------------------------------------------------
17019 //
17020 // // pertinent parts of existing instructions in architecture description
17021 // instruct movI(iRegINoSp dst, iRegI src)
17022 // %{
17023 //   match(Set dst (CopyI src));
17024 // %}
17025 //
17026 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17027 // %{
17028 //   match(Set dst (AddI dst src));
17029 //   effect(KILL cr);
17030 // %}
17031 //
17032 // // Change (inc mov) to lea
17033 // peephole %{
17034 //   // increment preceded by register-register move
17035 //   peepmatch ( incI_iReg movI );
17036 //   // require that the destination register of the increment
17037 //   // match the destination register of the move
17038 //   peepconstraint ( 0.dst == 1.dst );
17039 //   // construct a replacement instruction that sets
17040 //   // the destination to ( move's source register + one )
17041 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17042 // %}
17043 //
17044 
17045 // Implementation no longer uses movX instructions since
17046 // machine-independent system no longer uses CopyX nodes.
17047 //
17048 // peephole
17049 // %{
17050 //   peepmatch (incI_iReg movI);
17051 //   peepconstraint (0.dst == 1.dst);
17052 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17053 // %}
17054 
17055 // peephole
17056 // %{
17057 //   peepmatch (decI_iReg movI);
17058 //   peepconstraint (0.dst == 1.dst);
17059 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17060 // %}
17061 
17062 // peephole
17063 // %{
17064 //   peepmatch (addI_iReg_imm movI);
17065 //   peepconstraint (0.dst == 1.dst);
17066 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17067 // %}
17068 
17069 // peephole
17070 // %{
17071 //   peepmatch (incL_iReg movL);
17072 //   peepconstraint (0.dst == 1.dst);
17073 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17074 // %}
17075 
17076 // peephole
17077 // %{
17078 //   peepmatch (decL_iReg movL);
17079 //   peepconstraint (0.dst == 1.dst);
17080 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17081 // %}
17082 
17083 // peephole
17084 // %{
17085 //   peepmatch (addL_iReg_imm movL);
17086 //   peepconstraint (0.dst == 1.dst);
17087 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17088 // %}
17089 
17090 // peephole
17091 // %{
17092 //   peepmatch (addP_iReg_imm movP);
17093 //   peepconstraint (0.dst == 1.dst);
17094 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17095 // %}
17096 
17097 // // Change load of spilled value to only a spill
17098 // instruct storeI(memory mem, iRegI src)
17099 // %{
17100 //   match(Set mem (StoreI mem src));
17101 // %}
17102 //
17103 // instruct loadI(iRegINoSp dst, memory mem)
17104 // %{
17105 //   match(Set dst (LoadI mem));
17106 // %}
17107 //
17108 
17109 //----------SMARTSPILL RULES---------------------------------------------------
17110 // These must follow all instruction definitions as they use the names
17111 // defined in the instructions definitions.
17112 
17113 // Local Variables:
17114 // mode: c++
17115 // End: