1 //
    2 // Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // architecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182 
  183   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  184   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  185   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  186   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  187 
  188   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  189   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  190   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  191   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  192 
  193   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  194   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  195   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  196   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  197 
  198   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  199   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  200   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  201   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  202 
  203   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  204   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  205   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  206   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  207 
  208   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  209   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  210   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  211   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  212 
  213   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  214   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  215   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  216   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  217 
  218   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  219   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  220   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  221   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  222 
  223   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  224   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  225   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  226   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  227 
  228   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  229   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  230   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  231   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  232 
  233   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  234   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  235   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  236   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  237 
  238   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  239   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  240   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  241   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  242 
  243   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  244   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  245   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  246   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  247 
  248   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  249   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  250   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  251   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  252 
  253   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  254   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  255   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  256   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  257 
  258   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  259   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  260   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  261   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  262 
  263   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  264   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  265   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  266   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  267 
  268   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  269   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  270   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  271   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  272 
  273   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  274   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  275   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  276   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  277 
  278   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  279   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  280   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  281   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  282 
  283   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  284   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  285   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  286   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  287 
  288   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  289   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  290   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  291   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  292 
  293   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  294   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  295   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  296   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  297 
  298   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  299   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  300   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  301   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  302 
  303   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  304   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  305   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  306   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  307 
  308   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  309   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  310   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  311   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  312 
  313   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  314   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  315   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  316   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  317 
  318   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  319   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  320   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  321   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  322 
  323   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  324   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  325   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  326   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  327 
  328   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  329   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  330   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  331   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  332 
  333   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  334   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  335   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  336   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  337 
  338 // ----------------------------
  339 // SVE Predicate Registers
  340 // ----------------------------
  341   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  342   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  343   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  344   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  345   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  346   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  347   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  348   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  349   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  350   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  351   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  352   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  353   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  354   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  355   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  356   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  357 
  358 // ----------------------------
  359 // Special Registers
  360 // ----------------------------
  361 
  362 // the AArch64 CSPR status flag register is not directly accessible as
  363 // instruction operand. the FPSR status flag register is a system
  364 // register which can be written/read using MSR/MRS but again does not
  365 // appear as an operand (a code identifying the FSPR occurs as an
  366 // immediate value in the instruction).
  367 
  368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  369 
  370 // Specify priority of register selection within phases of register
  371 // allocation.  Highest priority is first.  A useful heuristic is to
  372 // give registers a low priority when they are required by machine
  373 // instructions, like EAX and EDX on I486, and choose no-save registers
  374 // before save-on-call, & save-on-call before save-on-entry.  Registers
  375 // which participate in fixed calling sequences should come last.
  376 // Registers which are used as pairs must fall on an even boundary.
  377 
  378 alloc_class chunk0(
  379     // volatiles
  380     R10, R10_H,
  381     R11, R11_H,
  382     R12, R12_H,
  383     R13, R13_H,
  384     R14, R14_H,
  385     R15, R15_H,
  386     R16, R16_H,
  387     R17, R17_H,
  388     R18, R18_H,
  389 
  390     // arg registers
  391     R0, R0_H,
  392     R1, R1_H,
  393     R2, R2_H,
  394     R3, R3_H,
  395     R4, R4_H,
  396     R5, R5_H,
  397     R6, R6_H,
  398     R7, R7_H,
  399 
  400     // non-volatiles
  401     R19, R19_H,
  402     R20, R20_H,
  403     R21, R21_H,
  404     R22, R22_H,
  405     R23, R23_H,
  406     R24, R24_H,
  407     R25, R25_H,
  408     R26, R26_H,
  409 
  410     // non-allocatable registers
  411 
  412     R27, R27_H, // heapbase
  413     R28, R28_H, // thread
  414     R29, R29_H, // fp
  415     R30, R30_H, // lr
  416     R31, R31_H, // sp
  417     R8, R8_H,   // rscratch1
  418     R9, R9_H,   // rscratch2
  419 );
  420 
  421 alloc_class chunk1(
  422 
  423     // no save
  424     V16, V16_H, V16_J, V16_K,
  425     V17, V17_H, V17_J, V17_K,
  426     V18, V18_H, V18_J, V18_K,
  427     V19, V19_H, V19_J, V19_K,
  428     V20, V20_H, V20_J, V20_K,
  429     V21, V21_H, V21_J, V21_K,
  430     V22, V22_H, V22_J, V22_K,
  431     V23, V23_H, V23_J, V23_K,
  432     V24, V24_H, V24_J, V24_K,
  433     V25, V25_H, V25_J, V25_K,
  434     V26, V26_H, V26_J, V26_K,
  435     V27, V27_H, V27_J, V27_K,
  436     V28, V28_H, V28_J, V28_K,
  437     V29, V29_H, V29_J, V29_K,
  438     V30, V30_H, V30_J, V30_K,
  439     V31, V31_H, V31_J, V31_K,
  440 
  441     // arg registers
  442     V0, V0_H, V0_J, V0_K,
  443     V1, V1_H, V1_J, V1_K,
  444     V2, V2_H, V2_J, V2_K,
  445     V3, V3_H, V3_J, V3_K,
  446     V4, V4_H, V4_J, V4_K,
  447     V5, V5_H, V5_J, V5_K,
  448     V6, V6_H, V6_J, V6_K,
  449     V7, V7_H, V7_J, V7_K,
  450 
  451     // non-volatiles
  452     V8, V8_H, V8_J, V8_K,
  453     V9, V9_H, V9_J, V9_K,
  454     V10, V10_H, V10_J, V10_K,
  455     V11, V11_H, V11_J, V11_K,
  456     V12, V12_H, V12_J, V12_K,
  457     V13, V13_H, V13_J, V13_K,
  458     V14, V14_H, V14_J, V14_K,
  459     V15, V15_H, V15_J, V15_K,
  460 );
  461 
  462 alloc_class chunk2 (
  463     P0,
  464     P1,
  465     P2,
  466     P3,
  467     P4,
  468     P5,
  469     P6,
  470     P7,
  471 
  472     P8,
  473     P9,
  474     P10,
  475     P11,
  476     P12,
  477     P13,
  478     P14,
  479     P15,
  480 );
  481 
  482 alloc_class chunk3(RFLAGS);
  483 
  484 //----------Architecture Description Register Classes--------------------------
  485 // Several register classes are automatically defined based upon information in
  486 // this architecture description.
  487 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  488 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  489 //
  490 
  491 // Class for all 32 bit general purpose registers
  492 reg_class all_reg32(
  493     R0,
  494     R1,
  495     R2,
  496     R3,
  497     R4,
  498     R5,
  499     R6,
  500     R7,
  501     R10,
  502     R11,
  503     R12,
  504     R13,
  505     R14,
  506     R15,
  507     R16,
  508     R17,
  509     R18,
  510     R19,
  511     R20,
  512     R21,
  513     R22,
  514     R23,
  515     R24,
  516     R25,
  517     R26,
  518     R27,
  519     R28,
  520     R29,
  521     R30,
  522     R31
  523 );
  524 
  525 
  526 // Class for all 32 bit integer registers (excluding SP which
  527 // will never be used as an integer register)
  528 reg_class any_reg32 %{
  529   return _ANY_REG32_mask;
  530 %}
  531 
  532 // Singleton class for R0 int register
  533 reg_class int_r0_reg(R0);
  534 
  535 // Singleton class for R2 int register
  536 reg_class int_r2_reg(R2);
  537 
  538 // Singleton class for R3 int register
  539 reg_class int_r3_reg(R3);
  540 
  541 // Singleton class for R4 int register
  542 reg_class int_r4_reg(R4);
  543 
  544 // Singleton class for R31 int register
  545 reg_class int_r31_reg(R31);
  546 
  547 // Class for all 64 bit general purpose registers
  548 reg_class all_reg(
  549     R0, R0_H,
  550     R1, R1_H,
  551     R2, R2_H,
  552     R3, R3_H,
  553     R4, R4_H,
  554     R5, R5_H,
  555     R6, R6_H,
  556     R7, R7_H,
  557     R10, R10_H,
  558     R11, R11_H,
  559     R12, R12_H,
  560     R13, R13_H,
  561     R14, R14_H,
  562     R15, R15_H,
  563     R16, R16_H,
  564     R17, R17_H,
  565     R18, R18_H,
  566     R19, R19_H,
  567     R20, R20_H,
  568     R21, R21_H,
  569     R22, R22_H,
  570     R23, R23_H,
  571     R24, R24_H,
  572     R25, R25_H,
  573     R26, R26_H,
  574     R27, R27_H,
  575     R28, R28_H,
  576     R29, R29_H,
  577     R30, R30_H,
  578     R31, R31_H
  579 );
  580 
  581 // Class for all long integer registers (including SP)
  582 reg_class any_reg %{
  583   return _ANY_REG_mask;
  584 %}
  585 
  586 // Class for non-allocatable 32 bit registers
  587 reg_class non_allocatable_reg32(
  588 #ifdef R18_RESERVED
  589     // See comment in register_aarch64.hpp
  590     R18,                        // tls on Windows
  591 #endif
  592     R28,                        // thread
  593     R30,                        // lr
  594     R31                         // sp
  595 );
  596 
  597 // Class for non-allocatable 64 bit registers
  598 reg_class non_allocatable_reg(
  599 #ifdef R18_RESERVED
  600     // See comment in register_aarch64.hpp
  601     R18, R18_H,                 // tls on Windows, platform register on macOS
  602 #endif
  603     R28, R28_H,                 // thread
  604     R30, R30_H,                 // lr
  605     R31, R31_H                  // sp
  606 );
  607 
  608 // Class for all non-special integer registers
  609 reg_class no_special_reg32 %{
  610   return _NO_SPECIAL_REG32_mask;
  611 %}
  612 
  613 // Class for all non-special long integer registers
  614 reg_class no_special_reg %{
  615   return _NO_SPECIAL_REG_mask;
  616 %}
  617 
  618 // Class for 64 bit register r0
  619 reg_class r0_reg(
  620     R0, R0_H
  621 );
  622 
  623 // Class for 64 bit register r1
  624 reg_class r1_reg(
  625     R1, R1_H
  626 );
  627 
  628 // Class for 64 bit register r2
  629 reg_class r2_reg(
  630     R2, R2_H
  631 );
  632 
  633 // Class for 64 bit register r3
  634 reg_class r3_reg(
  635     R3, R3_H
  636 );
  637 
  638 // Class for 64 bit register r4
  639 reg_class r4_reg(
  640     R4, R4_H
  641 );
  642 
  643 // Class for 64 bit register r5
  644 reg_class r5_reg(
  645     R5, R5_H
  646 );
  647 
  648 // Class for 64 bit register r10
  649 reg_class r10_reg(
  650     R10, R10_H
  651 );
  652 
  653 // Class for 64 bit register r11
  654 reg_class r11_reg(
  655     R11, R11_H
  656 );
  657 
  658 // Class for method register
  659 reg_class method_reg(
  660     R12, R12_H
  661 );
  662 
  663 // Class for heapbase register
  664 reg_class heapbase_reg(
  665     R27, R27_H
  666 );
  667 
  668 // Class for thread register
  669 reg_class thread_reg(
  670     R28, R28_H
  671 );
  672 
  673 // Class for frame pointer register
  674 reg_class fp_reg(
  675     R29, R29_H
  676 );
  677 
  678 // Class for link register
  679 reg_class lr_reg(
  680     R30, R30_H
  681 );
  682 
  683 // Class for long sp register
  684 reg_class sp_reg(
  685   R31, R31_H
  686 );
  687 
  688 // Class for all pointer registers
  689 reg_class ptr_reg %{
  690   return _PTR_REG_mask;
  691 %}
  692 
  693 // Class for all non_special pointer registers
  694 reg_class no_special_ptr_reg %{
  695   return _NO_SPECIAL_PTR_REG_mask;
  696 %}
  697 
  698 // Class for all float registers
  699 reg_class float_reg(
  700     V0,
  701     V1,
  702     V2,
  703     V3,
  704     V4,
  705     V5,
  706     V6,
  707     V7,
  708     V8,
  709     V9,
  710     V10,
  711     V11,
  712     V12,
  713     V13,
  714     V14,
  715     V15,
  716     V16,
  717     V17,
  718     V18,
  719     V19,
  720     V20,
  721     V21,
  722     V22,
  723     V23,
  724     V24,
  725     V25,
  726     V26,
  727     V27,
  728     V28,
  729     V29,
  730     V30,
  731     V31
  732 );
  733 
  734 // Double precision float registers have virtual `high halves' that
  735 // are needed by the allocator.
  736 // Class for all double registers
  737 reg_class double_reg(
  738     V0, V0_H,
  739     V1, V1_H,
  740     V2, V2_H,
  741     V3, V3_H,
  742     V4, V4_H,
  743     V5, V5_H,
  744     V6, V6_H,
  745     V7, V7_H,
  746     V8, V8_H,
  747     V9, V9_H,
  748     V10, V10_H,
  749     V11, V11_H,
  750     V12, V12_H,
  751     V13, V13_H,
  752     V14, V14_H,
  753     V15, V15_H,
  754     V16, V16_H,
  755     V17, V17_H,
  756     V18, V18_H,
  757     V19, V19_H,
  758     V20, V20_H,
  759     V21, V21_H,
  760     V22, V22_H,
  761     V23, V23_H,
  762     V24, V24_H,
  763     V25, V25_H,
  764     V26, V26_H,
  765     V27, V27_H,
  766     V28, V28_H,
  767     V29, V29_H,
  768     V30, V30_H,
  769     V31, V31_H
  770 );
  771 
  772 // Class for all SVE vector registers.
  773 reg_class vectora_reg (
  774     V0, V0_H, V0_J, V0_K,
  775     V1, V1_H, V1_J, V1_K,
  776     V2, V2_H, V2_J, V2_K,
  777     V3, V3_H, V3_J, V3_K,
  778     V4, V4_H, V4_J, V4_K,
  779     V5, V5_H, V5_J, V5_K,
  780     V6, V6_H, V6_J, V6_K,
  781     V7, V7_H, V7_J, V7_K,
  782     V8, V8_H, V8_J, V8_K,
  783     V9, V9_H, V9_J, V9_K,
  784     V10, V10_H, V10_J, V10_K,
  785     V11, V11_H, V11_J, V11_K,
  786     V12, V12_H, V12_J, V12_K,
  787     V13, V13_H, V13_J, V13_K,
  788     V14, V14_H, V14_J, V14_K,
  789     V15, V15_H, V15_J, V15_K,
  790     V16, V16_H, V16_J, V16_K,
  791     V17, V17_H, V17_J, V17_K,
  792     V18, V18_H, V18_J, V18_K,
  793     V19, V19_H, V19_J, V19_K,
  794     V20, V20_H, V20_J, V20_K,
  795     V21, V21_H, V21_J, V21_K,
  796     V22, V22_H, V22_J, V22_K,
  797     V23, V23_H, V23_J, V23_K,
  798     V24, V24_H, V24_J, V24_K,
  799     V25, V25_H, V25_J, V25_K,
  800     V26, V26_H, V26_J, V26_K,
  801     V27, V27_H, V27_J, V27_K,
  802     V28, V28_H, V28_J, V28_K,
  803     V29, V29_H, V29_J, V29_K,
  804     V30, V30_H, V30_J, V30_K,
  805     V31, V31_H, V31_J, V31_K,
  806 );
  807 
  808 // Class for all 64bit vector registers
  809 reg_class vectord_reg(
  810     V0, V0_H,
  811     V1, V1_H,
  812     V2, V2_H,
  813     V3, V3_H,
  814     V4, V4_H,
  815     V5, V5_H,
  816     V6, V6_H,
  817     V7, V7_H,
  818     V8, V8_H,
  819     V9, V9_H,
  820     V10, V10_H,
  821     V11, V11_H,
  822     V12, V12_H,
  823     V13, V13_H,
  824     V14, V14_H,
  825     V15, V15_H,
  826     V16, V16_H,
  827     V17, V17_H,
  828     V18, V18_H,
  829     V19, V19_H,
  830     V20, V20_H,
  831     V21, V21_H,
  832     V22, V22_H,
  833     V23, V23_H,
  834     V24, V24_H,
  835     V25, V25_H,
  836     V26, V26_H,
  837     V27, V27_H,
  838     V28, V28_H,
  839     V29, V29_H,
  840     V30, V30_H,
  841     V31, V31_H
  842 );
  843 
  844 // Class for all 128bit vector registers
  845 reg_class vectorx_reg(
  846     V0, V0_H, V0_J, V0_K,
  847     V1, V1_H, V1_J, V1_K,
  848     V2, V2_H, V2_J, V2_K,
  849     V3, V3_H, V3_J, V3_K,
  850     V4, V4_H, V4_J, V4_K,
  851     V5, V5_H, V5_J, V5_K,
  852     V6, V6_H, V6_J, V6_K,
  853     V7, V7_H, V7_J, V7_K,
  854     V8, V8_H, V8_J, V8_K,
  855     V9, V9_H, V9_J, V9_K,
  856     V10, V10_H, V10_J, V10_K,
  857     V11, V11_H, V11_J, V11_K,
  858     V12, V12_H, V12_J, V12_K,
  859     V13, V13_H, V13_J, V13_K,
  860     V14, V14_H, V14_J, V14_K,
  861     V15, V15_H, V15_J, V15_K,
  862     V16, V16_H, V16_J, V16_K,
  863     V17, V17_H, V17_J, V17_K,
  864     V18, V18_H, V18_J, V18_K,
  865     V19, V19_H, V19_J, V19_K,
  866     V20, V20_H, V20_J, V20_K,
  867     V21, V21_H, V21_J, V21_K,
  868     V22, V22_H, V22_J, V22_K,
  869     V23, V23_H, V23_J, V23_K,
  870     V24, V24_H, V24_J, V24_K,
  871     V25, V25_H, V25_J, V25_K,
  872     V26, V26_H, V26_J, V26_K,
  873     V27, V27_H, V27_J, V27_K,
  874     V28, V28_H, V28_J, V28_K,
  875     V29, V29_H, V29_J, V29_K,
  876     V30, V30_H, V30_J, V30_K,
  877     V31, V31_H, V31_J, V31_K
  878 );
  879 
  880 // Class for 128 bit register v0
  881 reg_class v0_reg(
  882     V0, V0_H
  883 );
  884 
  885 // Class for 128 bit register v1
  886 reg_class v1_reg(
  887     V1, V1_H
  888 );
  889 
  890 // Class for 128 bit register v2
  891 reg_class v2_reg(
  892     V2, V2_H
  893 );
  894 
  895 // Class for 128 bit register v3
  896 reg_class v3_reg(
  897     V3, V3_H
  898 );
  899 
  900 // Class for 128 bit register v4
  901 reg_class v4_reg(
  902     V4, V4_H
  903 );
  904 
  905 // Class for 128 bit register v5
  906 reg_class v5_reg(
  907     V5, V5_H
  908 );
  909 
  910 // Class for 128 bit register v6
  911 reg_class v6_reg(
  912     V6, V6_H
  913 );
  914 
  915 // Class for 128 bit register v7
  916 reg_class v7_reg(
  917     V7, V7_H
  918 );
  919 
  920 // Class for 128 bit register v8
  921 reg_class v8_reg(
  922     V8, V8_H
  923 );
  924 
  925 // Class for 128 bit register v9
  926 reg_class v9_reg(
  927     V9, V9_H
  928 );
  929 
  930 // Class for 128 bit register v10
  931 reg_class v10_reg(
  932     V10, V10_H
  933 );
  934 
  935 // Class for 128 bit register v11
  936 reg_class v11_reg(
  937     V11, V11_H
  938 );
  939 
  940 // Class for 128 bit register v12
  941 reg_class v12_reg(
  942     V12, V12_H
  943 );
  944 
  945 // Class for 128 bit register v13
  946 reg_class v13_reg(
  947     V13, V13_H
  948 );
  949 
  950 // Class for 128 bit register v14
  951 reg_class v14_reg(
  952     V14, V14_H
  953 );
  954 
  955 // Class for 128 bit register v15
  956 reg_class v15_reg(
  957     V15, V15_H
  958 );
  959 
  960 // Class for 128 bit register v16
  961 reg_class v16_reg(
  962     V16, V16_H
  963 );
  964 
  965 // Class for 128 bit register v17
  966 reg_class v17_reg(
  967     V17, V17_H
  968 );
  969 
  970 // Class for 128 bit register v18
  971 reg_class v18_reg(
  972     V18, V18_H
  973 );
  974 
  975 // Class for 128 bit register v19
  976 reg_class v19_reg(
  977     V19, V19_H
  978 );
  979 
  980 // Class for 128 bit register v20
  981 reg_class v20_reg(
  982     V20, V20_H
  983 );
  984 
  985 // Class for 128 bit register v21
  986 reg_class v21_reg(
  987     V21, V21_H
  988 );
  989 
  990 // Class for 128 bit register v22
  991 reg_class v22_reg(
  992     V22, V22_H
  993 );
  994 
  995 // Class for 128 bit register v23
  996 reg_class v23_reg(
  997     V23, V23_H
  998 );
  999 
 1000 // Class for 128 bit register v24
 1001 reg_class v24_reg(
 1002     V24, V24_H
 1003 );
 1004 
 1005 // Class for 128 bit register v25
 1006 reg_class v25_reg(
 1007     V25, V25_H
 1008 );
 1009 
 1010 // Class for 128 bit register v26
 1011 reg_class v26_reg(
 1012     V26, V26_H
 1013 );
 1014 
 1015 // Class for 128 bit register v27
 1016 reg_class v27_reg(
 1017     V27, V27_H
 1018 );
 1019 
 1020 // Class for 128 bit register v28
 1021 reg_class v28_reg(
 1022     V28, V28_H
 1023 );
 1024 
 1025 // Class for 128 bit register v29
 1026 reg_class v29_reg(
 1027     V29, V29_H
 1028 );
 1029 
 1030 // Class for 128 bit register v30
 1031 reg_class v30_reg(
 1032     V30, V30_H
 1033 );
 1034 
 1035 // Class for 128 bit register v31
 1036 reg_class v31_reg(
 1037     V31, V31_H
 1038 );
 1039 
 1040 // Class for all SVE predicate registers.
 1041 reg_class pr_reg (
 1042     P0,
 1043     P1,
 1044     P2,
 1045     P3,
 1046     P4,
 1047     P5,
 1048     P6,
 1049     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1050     P8,
 1051     P9,
 1052     P10,
 1053     P11,
 1054     P12,
 1055     P13,
 1056     P14,
 1057     P15
 1058 );
 1059 
 1060 // Class for SVE governing predicate registers, which are used
 1061 // to determine the active elements of a predicated instruction.
 1062 reg_class gov_pr (
 1063     P0,
 1064     P1,
 1065     P2,
 1066     P3,
 1067     P4,
 1068     P5,
 1069     P6,
 1070     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1071 );
 1072 
 1073 reg_class p0_reg(P0);
 1074 reg_class p1_reg(P1);
 1075 
 1076 // Singleton class for condition codes
 1077 reg_class int_flags(RFLAGS);
 1078 
 1079 %}
 1080 
 1081 //----------DEFINITION BLOCK---------------------------------------------------
 1082 // Define name --> value mappings to inform the ADLC of an integer valued name
 1083 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1084 // Format:
 1085 //        int_def  <name>         ( <int_value>, <expression>);
 1086 // Generated Code in ad_<arch>.hpp
 1087 //        #define  <name>   (<expression>)
 1088 //        // value == <int_value>
 1089 // Generated code in ad_<arch>.cpp adlc_verification()
 1090 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1091 //
 1092 
 1093 // we follow the ppc-aix port in using a simple cost model which ranks
 1094 // register operations as cheap, memory ops as more expensive and
 1095 // branches as most expensive. the first two have a low as well as a
 1096 // normal cost. huge cost appears to be a way of saying don't do
 1097 // something
 1098 
 1099 definitions %{
 1100   // The default cost (of a register move instruction).
 1101   int_def INSN_COST            (    100,     100);
 1102   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1103   int_def CALL_COST            (    200,     2 * INSN_COST);
 1104   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1105 %}
 1106 
 1107 
 1108 //----------SOURCE BLOCK-------------------------------------------------------
 1109 // This is a block of C++ code which provides values, functions, and
 1110 // definitions necessary in the rest of the architecture description
 1111 
 1112 source_hpp %{
 1113 
 1114 #include "asm/macroAssembler.hpp"
 1115 #include "gc/shared/barrierSetAssembler.hpp"
 1116 #include "gc/shared/cardTable.hpp"
 1117 #include "gc/shared/cardTableBarrierSet.hpp"
 1118 #include "gc/shared/collectedHeap.hpp"
 1119 #include "opto/addnode.hpp"
 1120 #include "opto/convertnode.hpp"
 1121 #include "runtime/objectMonitor.hpp"
 1122 
 1123 extern RegMask _ANY_REG32_mask;
 1124 extern RegMask _ANY_REG_mask;
 1125 extern RegMask _PTR_REG_mask;
 1126 extern RegMask _NO_SPECIAL_REG32_mask;
 1127 extern RegMask _NO_SPECIAL_REG_mask;
 1128 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1129 
 1130 class CallStubImpl {
 1131 
 1132   //--------------------------------------------------------------
 1133   //---<  Used for optimization in Compile::shorten_branches  >---
 1134   //--------------------------------------------------------------
 1135 
 1136  public:
 1137   // Size of call trampoline stub.
 1138   static uint size_call_trampoline() {
 1139     return 0; // no call trampolines on this platform
 1140   }
 1141 
 1142   // number of relocations needed by a call trampoline stub
 1143   static uint reloc_call_trampoline() {
 1144     return 0; // no call trampolines on this platform
 1145   }
 1146 };
 1147 
 1148 class HandlerImpl {
 1149 
 1150  public:
 1151 
 1152   static int emit_exception_handler(CodeBuffer &cbuf);
 1153   static int emit_deopt_handler(CodeBuffer& cbuf);
 1154 
 1155   static uint size_exception_handler() {
 1156     return MacroAssembler::far_codestub_branch_size();
 1157   }
 1158 
 1159   static uint size_deopt_handler() {
 1160     // count one adr and one far branch instruction
 1161     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1162   }
 1163 };
 1164 
 1165 class Node::PD {
 1166 public:
 1167   enum NodeFlags {
 1168     _last_flag = Node::_last_flag
 1169   };
 1170 };
 1171 
 1172   bool is_CAS(int opcode, bool maybe_volatile);
 1173 
 1174   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1175 
 1176   bool unnecessary_acquire(const Node *barrier);
 1177   bool needs_acquiring_load(const Node *load);
 1178 
 1179   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1180 
 1181   bool unnecessary_release(const Node *barrier);
 1182   bool unnecessary_volatile(const Node *barrier);
 1183   bool needs_releasing_store(const Node *store);
 1184 
 1185   // predicate controlling translation of CompareAndSwapX
 1186   bool needs_acquiring_load_exclusive(const Node *load);
 1187 
 1188   // predicate controlling addressing modes
 1189   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1190 %}
 1191 
 1192 source %{
 1193 
 1194   // Derived RegMask with conditionally allocatable registers
 1195 
 1196   void PhaseOutput::pd_perform_mach_node_analysis() {
 1197   }
 1198 
 1199   int MachNode::pd_alignment_required() const {
 1200     return 1;
 1201   }
 1202 
 1203   int MachNode::compute_padding(int current_offset) const {
 1204     return 0;
 1205   }
 1206 
 1207   RegMask _ANY_REG32_mask;
 1208   RegMask _ANY_REG_mask;
 1209   RegMask _PTR_REG_mask;
 1210   RegMask _NO_SPECIAL_REG32_mask;
 1211   RegMask _NO_SPECIAL_REG_mask;
 1212   RegMask _NO_SPECIAL_PTR_REG_mask;
 1213 
 1214   void reg_mask_init() {
 1215     // We derive below RegMask(s) from the ones which are auto-generated from
 1216     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1217     // registers conditionally reserved.
 1218 
 1219     _ANY_REG32_mask = _ALL_REG32_mask;
 1220     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1221 
 1222     _ANY_REG_mask = _ALL_REG_mask;
 1223 
 1224     _PTR_REG_mask = _ALL_REG_mask;
 1225 
 1226     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1227     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1228 
 1229     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1230     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1231 
 1232     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1233     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1234 
 1235     // r27 is not allocatable when compressed oops is on and heapbase is not
 1236     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1237     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1238       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1239       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1240       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1241     }
 1242 
 1243     // r29 is not allocatable when PreserveFramePointer is on
 1244     if (PreserveFramePointer) {
 1245       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1246       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1247       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1248     }
 1249   }
 1250 
 1251   // Optimizaton of volatile gets and puts
 1252   // -------------------------------------
 1253   //
 1254   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1255   // use to implement volatile reads and writes. For a volatile read
 1256   // we simply need
 1257   //
 1258   //   ldar<x>
 1259   //
 1260   // and for a volatile write we need
 1261   //
 1262   //   stlr<x>
 1263   //
 1264   // Alternatively, we can implement them by pairing a normal
 1265   // load/store with a memory barrier. For a volatile read we need
 1266   //
 1267   //   ldr<x>
 1268   //   dmb ishld
 1269   //
 1270   // for a volatile write
 1271   //
 1272   //   dmb ish
 1273   //   str<x>
 1274   //   dmb ish
 1275   //
 1276   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1277   // sequences. These are normally translated to an instruction
 1278   // sequence like the following
 1279   //
 1280   //   dmb      ish
 1281   // retry:
 1282   //   ldxr<x>   rval raddr
 1283   //   cmp       rval rold
 1284   //   b.ne done
 1285   //   stlxr<x>  rval, rnew, rold
 1286   //   cbnz      rval retry
 1287   // done:
 1288   //   cset      r0, eq
 1289   //   dmb ishld
 1290   //
 1291   // Note that the exclusive store is already using an stlxr
 1292   // instruction. That is required to ensure visibility to other
 1293   // threads of the exclusive write (assuming it succeeds) before that
 1294   // of any subsequent writes.
 1295   //
 1296   // The following instruction sequence is an improvement on the above
 1297   //
 1298   // retry:
 1299   //   ldaxr<x>  rval raddr
 1300   //   cmp       rval rold
 1301   //   b.ne done
 1302   //   stlxr<x>  rval, rnew, rold
 1303   //   cbnz      rval retry
 1304   // done:
 1305   //   cset      r0, eq
 1306   //
 1307   // We don't need the leading dmb ish since the stlxr guarantees
 1308   // visibility of prior writes in the case that the swap is
 1309   // successful. Crucially we don't have to worry about the case where
 1310   // the swap is not successful since no valid program should be
 1311   // relying on visibility of prior changes by the attempting thread
 1312   // in the case where the CAS fails.
 1313   //
 1314   // Similarly, we don't need the trailing dmb ishld if we substitute
 1315   // an ldaxr instruction since that will provide all the guarantees we
 1316   // require regarding observation of changes made by other threads
 1317   // before any change to the CAS address observed by the load.
 1318   //
 1319   // In order to generate the desired instruction sequence we need to
 1320   // be able to identify specific 'signature' ideal graph node
 1321   // sequences which i) occur as a translation of a volatile reads or
 1322   // writes or CAS operations and ii) do not occur through any other
 1323   // translation or graph transformation. We can then provide
 1324   // alternative aldc matching rules which translate these node
 1325   // sequences to the desired machine code sequences. Selection of the
 1326   // alternative rules can be implemented by predicates which identify
 1327   // the relevant node sequences.
 1328   //
 1329   // The ideal graph generator translates a volatile read to the node
 1330   // sequence
 1331   //
 1332   //   LoadX[mo_acquire]
 1333   //   MemBarAcquire
 1334   //
 1335   // As a special case when using the compressed oops optimization we
 1336   // may also see this variant
 1337   //
 1338   //   LoadN[mo_acquire]
 1339   //   DecodeN
 1340   //   MemBarAcquire
 1341   //
 1342   // A volatile write is translated to the node sequence
 1343   //
 1344   //   MemBarRelease
 1345   //   StoreX[mo_release] {CardMark}-optional
 1346   //   MemBarVolatile
 1347   //
 1348   // n.b. the above node patterns are generated with a strict
 1349   // 'signature' configuration of input and output dependencies (see
 1350   // the predicates below for exact details). The card mark may be as
 1351   // simple as a few extra nodes or, in a few GC configurations, may
 1352   // include more complex control flow between the leading and
 1353   // trailing memory barriers. However, whatever the card mark
 1354   // configuration these signatures are unique to translated volatile
 1355   // reads/stores -- they will not appear as a result of any other
 1356   // bytecode translation or inlining nor as a consequence of
 1357   // optimizing transforms.
 1358   //
 1359   // We also want to catch inlined unsafe volatile gets and puts and
 1360   // be able to implement them using either ldar<x>/stlr<x> or some
 1361   // combination of ldr<x>/stlr<x> and dmb instructions.
 1362   //
 1363   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1364   // normal volatile put node sequence containing an extra cpuorder
 1365   // membar
 1366   //
 1367   //   MemBarRelease
 1368   //   MemBarCPUOrder
 1369   //   StoreX[mo_release] {CardMark}-optional
 1370   //   MemBarCPUOrder
 1371   //   MemBarVolatile
 1372   //
 1373   // n.b. as an aside, a cpuorder membar is not itself subject to
 1374   // matching and translation by adlc rules.  However, the rule
 1375   // predicates need to detect its presence in order to correctly
 1376   // select the desired adlc rules.
 1377   //
 1378   // Inlined unsafe volatile gets manifest as a slightly different
 1379   // node sequence to a normal volatile get because of the
 1380   // introduction of some CPUOrder memory barriers to bracket the
 1381   // Load. However, but the same basic skeleton of a LoadX feeding a
 1382   // MemBarAcquire, possibly through an optional DecodeN, is still
 1383   // present
 1384   //
 1385   //   MemBarCPUOrder
 1386   //        ||       \\
 1387   //   MemBarCPUOrder LoadX[mo_acquire]
 1388   //        ||            |
 1389   //        ||       {DecodeN} optional
 1390   //        ||       /
 1391   //     MemBarAcquire
 1392   //
 1393   // In this case the acquire membar does not directly depend on the
 1394   // load. However, we can be sure that the load is generated from an
 1395   // inlined unsafe volatile get if we see it dependent on this unique
 1396   // sequence of membar nodes. Similarly, given an acquire membar we
 1397   // can know that it was added because of an inlined unsafe volatile
 1398   // get if it is fed and feeds a cpuorder membar and if its feed
 1399   // membar also feeds an acquiring load.
 1400   //
 1401   // Finally an inlined (Unsafe) CAS operation is translated to the
 1402   // following ideal graph
 1403   //
 1404   //   MemBarRelease
 1405   //   MemBarCPUOrder
 1406   //   CompareAndSwapX {CardMark}-optional
 1407   //   MemBarCPUOrder
 1408   //   MemBarAcquire
 1409   //
 1410   // So, where we can identify these volatile read and write
 1411   // signatures we can choose to plant either of the above two code
 1412   // sequences. For a volatile read we can simply plant a normal
 1413   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1414   // also choose to inhibit translation of the MemBarAcquire and
 1415   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1416   //
 1417   // When we recognise a volatile store signature we can choose to
 1418   // plant at a dmb ish as a translation for the MemBarRelease, a
 1419   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1420   // Alternatively, we can inhibit translation of the MemBarRelease
 1421   // and MemBarVolatile and instead plant a simple stlr<x>
 1422   // instruction.
 1423   //
 1424   // when we recognise a CAS signature we can choose to plant a dmb
 1425   // ish as a translation for the MemBarRelease, the conventional
 1426   // macro-instruction sequence for the CompareAndSwap node (which
 1427   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1428   // Alternatively, we can elide generation of the dmb instructions
 1429   // and plant the alternative CompareAndSwap macro-instruction
 1430   // sequence (which uses ldaxr<x>).
 1431   //
 1432   // Of course, the above only applies when we see these signature
 1433   // configurations. We still want to plant dmb instructions in any
 1434   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1435   // MemBarVolatile. For example, at the end of a constructor which
 1436   // writes final/volatile fields we will see a MemBarRelease
 1437   // instruction and this needs a 'dmb ish' lest we risk the
 1438   // constructed object being visible without making the
 1439   // final/volatile field writes visible.
 1440   //
 1441   // n.b. the translation rules below which rely on detection of the
 1442   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1443   // If we see anything other than the signature configurations we
 1444   // always just translate the loads and stores to ldr<x> and str<x>
 1445   // and translate acquire, release and volatile membars to the
 1446   // relevant dmb instructions.
 1447   //
 1448 
 1449   // is_CAS(int opcode, bool maybe_volatile)
 1450   //
 1451   // return true if opcode is one of the possible CompareAndSwapX
 1452   // values otherwise false.
 1453 
 1454   bool is_CAS(int opcode, bool maybe_volatile)
 1455   {
 1456     switch(opcode) {
 1457       // We handle these
 1458     case Op_CompareAndSwapI:
 1459     case Op_CompareAndSwapL:
 1460     case Op_CompareAndSwapP:
 1461     case Op_CompareAndSwapN:
 1462     case Op_ShenandoahCompareAndSwapP:
 1463     case Op_ShenandoahCompareAndSwapN:
 1464     case Op_CompareAndSwapB:
 1465     case Op_CompareAndSwapS:
 1466     case Op_GetAndSetI:
 1467     case Op_GetAndSetL:
 1468     case Op_GetAndSetP:
 1469     case Op_GetAndSetN:
 1470     case Op_GetAndAddI:
 1471     case Op_GetAndAddL:
 1472       return true;
 1473     case Op_CompareAndExchangeI:
 1474     case Op_CompareAndExchangeN:
 1475     case Op_CompareAndExchangeB:
 1476     case Op_CompareAndExchangeS:
 1477     case Op_CompareAndExchangeL:
 1478     case Op_CompareAndExchangeP:
 1479     case Op_WeakCompareAndSwapB:
 1480     case Op_WeakCompareAndSwapS:
 1481     case Op_WeakCompareAndSwapI:
 1482     case Op_WeakCompareAndSwapL:
 1483     case Op_WeakCompareAndSwapP:
 1484     case Op_WeakCompareAndSwapN:
 1485     case Op_ShenandoahWeakCompareAndSwapP:
 1486     case Op_ShenandoahWeakCompareAndSwapN:
 1487     case Op_ShenandoahCompareAndExchangeP:
 1488     case Op_ShenandoahCompareAndExchangeN:
 1489       return maybe_volatile;
 1490     default:
 1491       return false;
 1492     }
 1493   }
 1494 
 1495   // helper to determine the maximum number of Phi nodes we may need to
 1496   // traverse when searching from a card mark membar for the merge mem
 1497   // feeding a trailing membar or vice versa
 1498 
 1499 // predicates controlling emit of ldr<x>/ldar<x>
 1500 
 1501 bool unnecessary_acquire(const Node *barrier)
 1502 {
 1503   assert(barrier->is_MemBar(), "expecting a membar");
 1504 
 1505   MemBarNode* mb = barrier->as_MemBar();
 1506 
 1507   if (mb->trailing_load()) {
 1508     return true;
 1509   }
 1510 
 1511   if (mb->trailing_load_store()) {
 1512     Node* load_store = mb->in(MemBarNode::Precedent);
 1513     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1514     return is_CAS(load_store->Opcode(), true);
 1515   }
 1516 
 1517   return false;
 1518 }
 1519 
 1520 bool needs_acquiring_load(const Node *n)
 1521 {
 1522   assert(n->is_Load(), "expecting a load");
 1523   LoadNode *ld = n->as_Load();
 1524   return ld->is_acquire();
 1525 }
 1526 
 1527 bool unnecessary_release(const Node *n)
 1528 {
 1529   assert((n->is_MemBar() &&
 1530           n->Opcode() == Op_MemBarRelease),
 1531          "expecting a release membar");
 1532 
 1533   MemBarNode *barrier = n->as_MemBar();
 1534   if (!barrier->leading()) {
 1535     return false;
 1536   } else {
 1537     Node* trailing = barrier->trailing_membar();
 1538     MemBarNode* trailing_mb = trailing->as_MemBar();
 1539     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1540     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1541 
 1542     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1543     if (mem->is_Store()) {
 1544       assert(mem->as_Store()->is_release(), "");
 1545       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1546       return true;
 1547     } else {
 1548       assert(mem->is_LoadStore(), "");
 1549       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1550       return is_CAS(mem->Opcode(), true);
 1551     }
 1552   }
 1553   return false;
 1554 }
 1555 
 1556 bool unnecessary_volatile(const Node *n)
 1557 {
 1558   // assert n->is_MemBar();
 1559   MemBarNode *mbvol = n->as_MemBar();
 1560 
 1561   bool release = mbvol->trailing_store();
 1562   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1563 #ifdef ASSERT
 1564   if (release) {
 1565     Node* leading = mbvol->leading_membar();
 1566     assert(leading->Opcode() == Op_MemBarRelease, "");
 1567     assert(leading->as_MemBar()->leading_store(), "");
 1568     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1569   }
 1570 #endif
 1571 
 1572   return release;
 1573 }
 1574 
 1575 // predicates controlling emit of str<x>/stlr<x>
 1576 
 1577 bool needs_releasing_store(const Node *n)
 1578 {
 1579   // assert n->is_Store();
 1580   StoreNode *st = n->as_Store();
 1581   return st->trailing_membar() != NULL;
 1582 }
 1583 
 1584 // predicate controlling translation of CAS
 1585 //
 1586 // returns true if CAS needs to use an acquiring load otherwise false
 1587 
 1588 bool needs_acquiring_load_exclusive(const Node *n)
 1589 {
 1590   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1591   LoadStoreNode* ldst = n->as_LoadStore();
 1592   if (is_CAS(n->Opcode(), false)) {
 1593     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1594   } else {
 1595     return ldst->trailing_membar() != NULL;
 1596   }
 1597 
 1598   // so we can just return true here
 1599   return true;
 1600 }
 1601 
 1602 #define __ _masm.
 1603 
 1604 // advance declarations for helper functions to convert register
 1605 // indices to register objects
 1606 
 1607 // the ad file has to provide implementations of certain methods
 1608 // expected by the generic code
 1609 //
 1610 // REQUIRED FUNCTIONALITY
 1611 
 1612 //=============================================================================
 1613 
 1614 // !!!!! Special hack to get all types of calls to specify the byte offset
 1615 //       from the start of the call to the point where the return address
 1616 //       will point.
 1617 
 1618 int MachCallStaticJavaNode::ret_addr_offset()
 1619 {
 1620   // call should be a simple bl
 1621   int off = 4;
 1622   return off;
 1623 }
 1624 
 1625 int MachCallDynamicJavaNode::ret_addr_offset()
 1626 {
 1627   return 16; // movz, movk, movk, bl
 1628 }
 1629 
 1630 int MachCallRuntimeNode::ret_addr_offset() {
 1631   // for generated stubs the call will be
 1632   //   bl(addr)
 1633   // or with far branches
 1634   //   bl(trampoline_stub)
 1635   // for real runtime callouts it will be six instructions
 1636   // see aarch64_enc_java_to_runtime
 1637   //   adr(rscratch2, retaddr)
 1638   //   lea(rscratch1, RuntimeAddress(addr)
 1639   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1640   //   blr(rscratch1)
 1641   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1642   if (cb) {
 1643     return 1 * NativeInstruction::instruction_size;
 1644   } else if (_entry_point == NULL) {
 1645     // See CallLeafNoFPIndirect
 1646     return 1 * NativeInstruction::instruction_size;
 1647   } else {
 1648     return 6 * NativeInstruction::instruction_size;
 1649   }
 1650 }
 1651 
 1652 //=============================================================================
 1653 
 1654 #ifndef PRODUCT
 1655 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1656   st->print("BREAKPOINT");
 1657 }
 1658 #endif
 1659 
 1660 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1661   C2_MacroAssembler _masm(&cbuf);
 1662   __ brk(0);
 1663 }
 1664 
 1665 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1666   return MachNode::size(ra_);
 1667 }
 1668 
 1669 //=============================================================================
 1670 
 1671 #ifndef PRODUCT
 1672   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1673     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1674   }
 1675 #endif
 1676 
 1677   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1678     C2_MacroAssembler _masm(&cbuf);
 1679     for (int i = 0; i < _count; i++) {
 1680       __ nop();
 1681     }
 1682   }
 1683 
 1684   uint MachNopNode::size(PhaseRegAlloc*) const {
 1685     return _count * NativeInstruction::instruction_size;
 1686   }
 1687 
 1688 //=============================================================================
 1689 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1690 
 1691 int ConstantTable::calculate_table_base_offset() const {
 1692   return 0;  // absolute addressing, no offset
 1693 }
 1694 
 1695 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1696 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1697   ShouldNotReachHere();
 1698 }
 1699 
 1700 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1701   // Empty encoding
 1702 }
 1703 
 1704 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1705   return 0;
 1706 }
 1707 
 1708 #ifndef PRODUCT
 1709 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1710   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1711 }
 1712 #endif
 1713 
 1714 #ifndef PRODUCT
 1715 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1716   Compile* C = ra_->C;
 1717 
 1718   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1719 
 1720   if (C->output()->need_stack_bang(framesize))
 1721     st->print("# stack bang size=%d\n\t", framesize);
 1722 
 1723   if (VM_Version::use_rop_protection()) {
 1724     st->print("ldr zr, [lr]\n\t");
 1725     st->print("pacia  lr, rfp\n\t");
 1726   }
 1727   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1728     st->print("sub  sp, sp, #%d\n\t", framesize);
 1729     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1730     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1731   } else {
 1732     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1733     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1734     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1735     st->print("sub  sp, sp, rscratch1");
 1736   }
 1737   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1738     st->print("\n\t");
 1739     st->print("ldr  rscratch1, [guard]\n\t");
 1740     st->print("dmb ishld\n\t");
 1741     st->print("ldr  rscratch2, [rthread, #thread_disarmed_offset]\n\t");
 1742     st->print("cmp  rscratch1, rscratch2\n\t");
 1743     st->print("b.eq skip");
 1744     st->print("\n\t");
 1745     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1746     st->print("b skip\n\t");
 1747     st->print("guard: int\n\t");
 1748     st->print("\n\t");
 1749     st->print("skip:\n\t");
 1750   }
 1751 }
 1752 #endif
 1753 
 1754 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1755   Compile* C = ra_->C;
 1756   C2_MacroAssembler _masm(&cbuf);
 1757 
 1758   // insert a nop at the start of the prolog so we can patch in a
 1759   // branch if we need to invalidate the method later
 1760   __ nop();
 1761 
 1762   __ verified_entry(C, 0);
 1763 
 1764   if (C->stub_function() == NULL) {
 1765     __ entry_barrier();
 1766   }
 1767 
 1768   if (!Compile::current()->output()->in_scratch_emit_size()) {
 1769     __ bind(*_verified_entry);
 1770   }
 1771 
 1772   if (VerifyStackAtCalls) {
 1773     Unimplemented();
 1774   }
 1775 
 1776   C->output()->set_frame_complete(cbuf.insts_size());
 1777 
 1778   if (C->has_mach_constant_base_node()) {
 1779     // NOTE: We set the table base offset here because users might be
 1780     // emitted before MachConstantBaseNode.
 1781     ConstantTable& constant_table = C->output()->constant_table();
 1782     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1783   }
 1784 }
 1785 
 1786 int MachPrologNode::reloc() const
 1787 {
 1788   return 0;
 1789 }
 1790 
 1791 //=============================================================================
 1792 
 1793 #ifndef PRODUCT
 1794 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1795   Compile* C = ra_->C;
 1796   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1797 
 1798   st->print("# pop frame %d\n\t",framesize);
 1799 
 1800   if (framesize == 0) {
 1801     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1802   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1803     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1804     st->print("add  sp, sp, #%d\n\t", framesize);
 1805   } else {
 1806     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1807     st->print("add  sp, sp, rscratch1\n\t");
 1808     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1809   }
 1810   if (VM_Version::use_rop_protection()) {
 1811     st->print("autia lr, rfp\n\t");
 1812     st->print("ldr zr, [lr]\n\t");
 1813   }
 1814 
 1815   if (do_polling() && C->is_method_compilation()) {
 1816     st->print("# test polling word\n\t");
 1817     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1818     st->print("cmp  sp, rscratch1\n\t");
 1819     st->print("bhi #slow_path");
 1820   }
 1821 }
 1822 #endif
 1823 
 1824 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1825   Compile* C = ra_->C;
 1826   C2_MacroAssembler _masm(&cbuf);
 1827   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1828 
 1829   __ remove_frame(framesize, C->needs_stack_repair());
 1830 
 1831   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1832     __ reserved_stack_check();
 1833   }
 1834 
 1835   if (do_polling() && C->is_method_compilation()) {
 1836     Label dummy_label;
 1837     Label* code_stub = &dummy_label;
 1838     if (!C->output()->in_scratch_emit_size()) {
 1839       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1840     }
 1841     __ relocate(relocInfo::poll_return_type);
 1842     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1843   }
 1844 }
 1845 
 1846 int MachEpilogNode::reloc() const {
 1847   // Return number of relocatable values contained in this instruction.
 1848   return 1; // 1 for polling page.
 1849 }
 1850 
 1851 const Pipeline * MachEpilogNode::pipeline() const {
 1852   return MachNode::pipeline_class();
 1853 }
 1854 
 1855 //=============================================================================
 1856 
 1857 // Figure out which register class each belongs in: rc_int, rc_float or
 1858 // rc_stack.
 1859 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 1860 
 1861 static enum RC rc_class(OptoReg::Name reg) {
 1862 
 1863   if (reg == OptoReg::Bad) {
 1864     return rc_bad;
 1865   }
 1866 
 1867   // we have 32 int registers * 2 halves
 1868   int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
 1869 
 1870   if (reg < slots_of_int_registers) {
 1871     return rc_int;
 1872   }
 1873 
 1874   // we have 32 float register * 8 halves
 1875   int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
 1876   if (reg < slots_of_int_registers + slots_of_float_registers) {
 1877     return rc_float;
 1878   }
 1879 
 1880   int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
 1881   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 1882     return rc_predicate;
 1883   }
 1884 
 1885   // Between predicate regs & stack is the flags.
 1886   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 1887 
 1888   return rc_stack;
 1889 }
 1890 
 1891 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 1892   Compile* C = ra_->C;
 1893 
 1894   // Get registers to move.
 1895   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 1896   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 1897   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 1898   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 1899 
 1900   enum RC src_hi_rc = rc_class(src_hi);
 1901   enum RC src_lo_rc = rc_class(src_lo);
 1902   enum RC dst_hi_rc = rc_class(dst_hi);
 1903   enum RC dst_lo_rc = rc_class(dst_lo);
 1904 
 1905   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 1906 
 1907   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 1908     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 1909            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 1910            "expected aligned-adjacent pairs");
 1911   }
 1912 
 1913   if (src_lo == dst_lo && src_hi == dst_hi) {
 1914     return 0;            // Self copy, no move.
 1915   }
 1916 
 1917   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 1918               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 1919   int src_offset = ra_->reg2offset(src_lo);
 1920   int dst_offset = ra_->reg2offset(dst_lo);
 1921 
 1922   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 1923     uint ireg = ideal_reg();
 1924     if (ireg == Op_VecA && cbuf) {
 1925       C2_MacroAssembler _masm(cbuf);
 1926       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 1927       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1928         // stack->stack
 1929         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 1930                                                 sve_vector_reg_size_in_bytes);
 1931       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 1932         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 1933                             sve_vector_reg_size_in_bytes);
 1934       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 1935         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 1936                               sve_vector_reg_size_in_bytes);
 1937       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 1938         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1939                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 1940                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 1941       } else {
 1942         ShouldNotReachHere();
 1943       }
 1944     } else if (cbuf) {
 1945       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 1946       C2_MacroAssembler _masm(cbuf);
 1947       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 1948       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1949         // stack->stack
 1950         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 1951         if (ireg == Op_VecD) {
 1952           __ unspill(rscratch1, true, src_offset);
 1953           __ spill(rscratch1, true, dst_offset);
 1954         } else {
 1955           __ spill_copy128(src_offset, dst_offset);
 1956         }
 1957       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 1958         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1959                ireg == Op_VecD ? __ T8B : __ T16B,
 1960                as_FloatRegister(Matcher::_regEncode[src_lo]));
 1961       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 1962         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 1963                  ireg == Op_VecD ? __ D : __ Q,
 1964                  ra_->reg2offset(dst_lo));
 1965       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 1966         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1967                    ireg == Op_VecD ? __ D : __ Q,
 1968                    ra_->reg2offset(src_lo));
 1969       } else {
 1970         ShouldNotReachHere();
 1971       }
 1972     }
 1973   } else if (cbuf) {
 1974     C2_MacroAssembler _masm(cbuf);
 1975     switch (src_lo_rc) {
 1976     case rc_int:
 1977       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 1978         if (is64) {
 1979             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 1980                    as_Register(Matcher::_regEncode[src_lo]));
 1981         } else {
 1982             C2_MacroAssembler _masm(cbuf);
 1983             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 1984                     as_Register(Matcher::_regEncode[src_lo]));
 1985         }
 1986       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 1987         if (is64) {
 1988             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1989                      as_Register(Matcher::_regEncode[src_lo]));
 1990         } else {
 1991             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1992                      as_Register(Matcher::_regEncode[src_lo]));
 1993         }
 1994       } else {                    // gpr --> stack spill
 1995         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 1996         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 1997       }
 1998       break;
 1999     case rc_float:
 2000       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2001         if (is64) {
 2002             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2003                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2004         } else {
 2005             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2006                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2007         }
 2008       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2009         if (is64) {
 2010             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2011                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2012         } else {
 2013             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2014                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2015         }
 2016       } else {                    // fpr --> stack spill
 2017         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2018         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2019                  is64 ? __ D : __ S, dst_offset);
 2020       }
 2021       break;
 2022     case rc_stack:
 2023       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2024         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2025       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2026         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2027                    is64 ? __ D : __ S, src_offset);
 2028       } else if (dst_lo_rc == rc_predicate) {
 2029         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2030                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2031       } else {                    // stack --> stack copy
 2032         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2033         if (ideal_reg() == Op_RegVectMask) {
 2034           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2035                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2036         } else {
 2037           __ unspill(rscratch1, is64, src_offset);
 2038           __ spill(rscratch1, is64, dst_offset);
 2039         }
 2040       }
 2041       break;
 2042     case rc_predicate:
 2043       if (dst_lo_rc == rc_predicate) {
 2044         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2045       } else if (dst_lo_rc == rc_stack) {
 2046         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2047                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2048       } else {
 2049         assert(false, "bad src and dst rc_class combination.");
 2050         ShouldNotReachHere();
 2051       }
 2052       break;
 2053     default:
 2054       assert(false, "bad rc_class for spill");
 2055       ShouldNotReachHere();
 2056     }
 2057   }
 2058 
 2059   if (st) {
 2060     st->print("spill ");
 2061     if (src_lo_rc == rc_stack) {
 2062       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2063     } else {
 2064       st->print("%s -> ", Matcher::regName[src_lo]);
 2065     }
 2066     if (dst_lo_rc == rc_stack) {
 2067       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2068     } else {
 2069       st->print("%s", Matcher::regName[dst_lo]);
 2070     }
 2071     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2072       int vsize = 0;
 2073       switch (ideal_reg()) {
 2074       case Op_VecD:
 2075         vsize = 64;
 2076         break;
 2077       case Op_VecX:
 2078         vsize = 128;
 2079         break;
 2080       case Op_VecA:
 2081         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2082         break;
 2083       default:
 2084         assert(false, "bad register type for spill");
 2085         ShouldNotReachHere();
 2086       }
 2087       st->print("\t# vector spill size = %d", vsize);
 2088     } else if (ideal_reg() == Op_RegVectMask) {
 2089       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2090       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2091       st->print("\t# predicate spill size = %d", vsize);
 2092     } else {
 2093       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2094     }
 2095   }
 2096 
 2097   return 0;
 2098 
 2099 }
 2100 
 2101 #ifndef PRODUCT
 2102 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2103   if (!ra_)
 2104     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2105   else
 2106     implementation(NULL, ra_, false, st);
 2107 }
 2108 #endif
 2109 
 2110 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2111   implementation(&cbuf, ra_, false, NULL);
 2112 }
 2113 
 2114 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2115   return MachNode::size(ra_);
 2116 }
 2117 
 2118 //=============================================================================
 2119 
 2120 #ifndef PRODUCT
 2121 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2122   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2123   int reg = ra_->get_reg_first(this);
 2124   st->print("add %s, rsp, #%d]\t# box lock",
 2125             Matcher::regName[reg], offset);
 2126 }
 2127 #endif
 2128 
 2129 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2130   C2_MacroAssembler _masm(&cbuf);
 2131 
 2132   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2133   int reg    = ra_->get_encode(this);
 2134 
 2135   // This add will handle any 24-bit signed offset. 24 bits allows an
 2136   // 8 megabyte stack frame.
 2137   __ add(as_Register(reg), sp, offset);
 2138 }
 2139 
 2140 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2141   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2142   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2143 
 2144   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2145     return NativeInstruction::instruction_size;
 2146   } else {
 2147     return 2 * NativeInstruction::instruction_size;
 2148   }
 2149 }
 2150 
 2151 ///=============================================================================
 2152 #ifndef PRODUCT
 2153 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2154 {
 2155   st->print_cr("# MachVEPNode");
 2156   if (!_verified) {
 2157     st->print_cr("\t load_class");
 2158   } else {
 2159     st->print_cr("\t unpack_inline_arg");
 2160   }
 2161 }
 2162 #endif
 2163 
 2164 void MachVEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2165 {
 2166   C2_MacroAssembler _masm(&cbuf);
 2167 
 2168   if (!_verified) {
 2169     Label skip;
 2170     __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2171     __ br(Assembler::EQ, skip);
 2172       __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2173     __ bind(skip);
 2174 
 2175   } else {
 2176     // insert a nop at the start of the prolog so we can patch in a
 2177     // branch if we need to invalidate the method later
 2178     __ nop();
 2179 
 2180     // TODO 8284443 Avoid creation of temporary frame
 2181     if (ra_->C->stub_function() == NULL) {
 2182       __ verified_entry(ra_->C, 0);
 2183       __ entry_barrier();
 2184       int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
 2185       __ remove_frame(framesize, false);
 2186     }
 2187     // Unpack inline type args passed as oop and then jump to
 2188     // the verified entry point (skipping the unverified entry).
 2189     int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
 2190     // Emit code for verified entry and save increment for stack repair on return
 2191     __ verified_entry(ra_->C, sp_inc);
 2192     if (Compile::current()->output()->in_scratch_emit_size()) {
 2193       Label dummy_verified_entry;
 2194       __ b(dummy_verified_entry);
 2195     } else {
 2196       __ b(*_verified_entry);
 2197     }
 2198   }
 2199 }
 2200 
 2201 //=============================================================================
 2202 #ifndef PRODUCT
 2203 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2204 {
 2205   st->print_cr("# MachUEPNode");
 2206   if (UseCompressedClassPointers) {
 2207     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2208     if (CompressedKlassPointers::shift() != 0) {
 2209       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2210     }
 2211   } else {
 2212    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2213   }
 2214   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2215   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2216 }
 2217 #endif
 2218 
 2219 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2220 {
 2221   // This is the unverified entry point.
 2222   C2_MacroAssembler _masm(&cbuf);
 2223   Label skip;
 2224 
 2225   // UseCompressedClassPointers logic are inside cmp_klass
 2226   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2227 
 2228   // TODO
 2229   // can we avoid this skip and still use a reloc?
 2230   __ br(Assembler::EQ, skip);
 2231   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2232   __ bind(skip);
 2233 }
 2234 
 2235 // REQUIRED EMIT CODE
 2236 
 2237 //=============================================================================
 2238 
 2239 // Emit exception handler code.
 2240 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2241 {
 2242   // mov rscratch1 #exception_blob_entry_point
 2243   // br rscratch1
 2244   // Note that the code buffer's insts_mark is always relative to insts.
 2245   // That's why we must use the macroassembler to generate a handler.
 2246   C2_MacroAssembler _masm(&cbuf);
 2247   address base = __ start_a_stub(size_exception_handler());
 2248   if (base == NULL) {
 2249     ciEnv::current()->record_failure("CodeCache is full");
 2250     return 0;  // CodeBuffer::expand failed
 2251   }
 2252   int offset = __ offset();
 2253   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2254   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2255   __ end_a_stub();
 2256   return offset;
 2257 }
 2258 
 2259 // Emit deopt handler code.
 2260 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2261 {
 2262   // Note that the code buffer's insts_mark is always relative to insts.
 2263   // That's why we must use the macroassembler to generate a handler.
 2264   C2_MacroAssembler _masm(&cbuf);
 2265   address base = __ start_a_stub(size_deopt_handler());
 2266   if (base == NULL) {
 2267     ciEnv::current()->record_failure("CodeCache is full");
 2268     return 0;  // CodeBuffer::expand failed
 2269   }
 2270   int offset = __ offset();
 2271 
 2272   __ adr(lr, __ pc());
 2273   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2274 
 2275   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2276   __ end_a_stub();
 2277   return offset;
 2278 }
 2279 
 2280 // REQUIRED MATCHER CODE
 2281 
 2282 //=============================================================================
 2283 
 2284 const bool Matcher::match_rule_supported(int opcode) {
 2285   if (!has_match_rule(opcode))
 2286     return false;
 2287 
 2288   bool ret_value = true;
 2289   switch (opcode) {
 2290     case Op_OnSpinWait:
 2291       return VM_Version::supports_on_spin_wait();
 2292     case Op_CacheWB:
 2293     case Op_CacheWBPreSync:
 2294     case Op_CacheWBPostSync:
 2295       if (!VM_Version::supports_data_cache_line_flush()) {
 2296         ret_value = false;
 2297       }
 2298       break;
 2299   }
 2300 
 2301   return ret_value; // Per default match rules are supported.
 2302 }
 2303 
 2304 const RegMask* Matcher::predicate_reg_mask(void) {
 2305   return &_PR_REG_mask;
 2306 }
 2307 
 2308 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2309   return new TypeVectMask(elemTy, length);
 2310 }
 2311 
 2312 // Vector calling convention not yet implemented.
 2313 const bool Matcher::supports_vector_calling_convention(void) {
 2314   return false;
 2315 }
 2316 
 2317 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2318   Unimplemented();
 2319   return OptoRegPair(0, 0);
 2320 }
 2321 
 2322 // Is this branch offset short enough that a short branch can be used?
 2323 //
 2324 // NOTE: If the platform does not provide any short branch variants, then
 2325 //       this method should return false for offset 0.
 2326 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2327   // The passed offset is relative to address of the branch.
 2328 
 2329   return (-32768 <= offset && offset < 32768);
 2330 }
 2331 
 2332 // Vector width in bytes.
 2333 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2334   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2335   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2336   // Minimum 2 values in vector
 2337   if (size < 2*type2aelembytes(bt)) size = 0;
 2338   // But never < 4
 2339   if (size < 4) size = 0;
 2340   return size;
 2341 }
 2342 
 2343 // Limits on vector size (number of elements) loaded into vector.
 2344 const int Matcher::max_vector_size(const BasicType bt) {
 2345   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2346 }
 2347 
 2348 const int Matcher::min_vector_size(const BasicType bt) {
 2349   int max_size = max_vector_size(bt);
 2350   // Limit the min vector size to 8 bytes.
 2351   int size = 8 / type2aelembytes(bt);
 2352   if (bt == T_BYTE) {
 2353     // To support vector api shuffle/rearrange.
 2354     size = 4;
 2355   } else if (bt == T_BOOLEAN) {
 2356     // To support vector api load/store mask.
 2357     size = 2;
 2358   }
 2359   if (size < 2) size = 2;
 2360   return MIN2(size, max_size);
 2361 }
 2362 
 2363 // Actual max scalable vector register length.
 2364 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2365   return Matcher::max_vector_size(bt);
 2366 }
 2367 
 2368 // Vector ideal reg.
 2369 const uint Matcher::vector_ideal_reg(int len) {
 2370   if (UseSVE > 0 && 16 < len && len <= 256) {
 2371     return Op_VecA;
 2372   }
 2373   switch(len) {
 2374     // For 16-bit/32-bit mask vector, reuse VecD.
 2375     case  2:
 2376     case  4:
 2377     case  8: return Op_VecD;
 2378     case 16: return Op_VecX;
 2379   }
 2380   ShouldNotReachHere();
 2381   return 0;
 2382 }
 2383 
 2384 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
 2385   assert(Matcher::is_generic_vector(generic_opnd), "not generic");
 2386   switch (ideal_reg) {
 2387     case Op_VecA: return new vecAOper();
 2388     case Op_VecD: return new vecDOper();
 2389     case Op_VecX: return new vecXOper();
 2390   }
 2391   ShouldNotReachHere();
 2392   return NULL;
 2393 }
 2394 
 2395 bool Matcher::is_reg2reg_move(MachNode* m) {
 2396   return false;
 2397 }
 2398 
 2399 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2400   return opnd->opcode() == VREG;
 2401 }
 2402 
 2403 // Return whether or not this register is ever used as an argument.
 2404 // This function is used on startup to build the trampoline stubs in
 2405 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2406 // call in the trampoline, and arguments in those registers not be
 2407 // available to the callee.
 2408 bool Matcher::can_be_java_arg(int reg)
 2409 {
 2410   return
 2411     reg ==  R0_num || reg == R0_H_num ||
 2412     reg ==  R1_num || reg == R1_H_num ||
 2413     reg ==  R2_num || reg == R2_H_num ||
 2414     reg ==  R3_num || reg == R3_H_num ||
 2415     reg ==  R4_num || reg == R4_H_num ||
 2416     reg ==  R5_num || reg == R5_H_num ||
 2417     reg ==  R6_num || reg == R6_H_num ||
 2418     reg ==  R7_num || reg == R7_H_num ||
 2419     reg ==  V0_num || reg == V0_H_num ||
 2420     reg ==  V1_num || reg == V1_H_num ||
 2421     reg ==  V2_num || reg == V2_H_num ||
 2422     reg ==  V3_num || reg == V3_H_num ||
 2423     reg ==  V4_num || reg == V4_H_num ||
 2424     reg ==  V5_num || reg == V5_H_num ||
 2425     reg ==  V6_num || reg == V6_H_num ||
 2426     reg ==  V7_num || reg == V7_H_num;
 2427 }
 2428 
 2429 bool Matcher::is_spillable_arg(int reg)
 2430 {
 2431   return can_be_java_arg(reg);
 2432 }
 2433 
 2434 uint Matcher::int_pressure_limit()
 2435 {
 2436   // JDK-8183543: When taking the number of available registers as int
 2437   // register pressure threshold, the jtreg test:
 2438   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2439   // failed due to C2 compilation failure with
 2440   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2441   //
 2442   // A derived pointer is live at CallNode and then is flagged by RA
 2443   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2444   // derived pointers and lastly fail to spill after reaching maximum
 2445   // number of iterations. Lowering the default pressure threshold to
 2446   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2447   // a high register pressure area of the code so that split_DEF can
 2448   // generate DefinitionSpillCopy for the derived pointer.
 2449   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2450   if (!PreserveFramePointer) {
 2451     // When PreserveFramePointer is off, frame pointer is allocatable,
 2452     // but different from other SOC registers, it is excluded from
 2453     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2454     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2455     // See check_pressure_at_fatproj().
 2456     default_int_pressure_threshold--;
 2457   }
 2458   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2459 }
 2460 
 2461 uint Matcher::float_pressure_limit()
 2462 {
 2463   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2464   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2465 }
 2466 
 2467 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2468   return false;
 2469 }
 2470 
 2471 RegMask Matcher::divI_proj_mask() {
 2472   ShouldNotReachHere();
 2473   return RegMask();
 2474 }
 2475 
 2476 // Register for MODI projection of divmodI.
 2477 RegMask Matcher::modI_proj_mask() {
 2478   ShouldNotReachHere();
 2479   return RegMask();
 2480 }
 2481 
 2482 // Register for DIVL projection of divmodL.
 2483 RegMask Matcher::divL_proj_mask() {
 2484   ShouldNotReachHere();
 2485   return RegMask();
 2486 }
 2487 
 2488 // Register for MODL projection of divmodL.
 2489 RegMask Matcher::modL_proj_mask() {
 2490   ShouldNotReachHere();
 2491   return RegMask();
 2492 }
 2493 
 2494 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2495   return FP_REG_mask();
 2496 }
 2497 
 2498 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2499   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2500     Node* u = addp->fast_out(i);
 2501     if (u->is_LoadStore()) {
 2502       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2503       // instructions) only take register indirect as an operand, so
 2504       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2505       // must fail.
 2506       return false;
 2507     }
 2508     if (u->is_Mem()) {
 2509       int opsize = u->as_Mem()->memory_size();
 2510       assert(opsize > 0, "unexpected memory operand size");
 2511       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2512         return false;
 2513       }
 2514     }
 2515   }
 2516   return true;
 2517 }
 2518 
 2519 // Binary src (Replicate con)
 2520 bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
 2521   if (n == NULL || m == NULL) {
 2522     return false;
 2523   }
 2524 
 2525   if (UseSVE == 0 || !VectorNode::is_invariant_vector(m)) {
 2526     return false;
 2527   }
 2528 
 2529   Node* imm_node = m->in(1);
 2530   if (!imm_node->is_Con()) {
 2531     return false;
 2532   }
 2533 
 2534   const Type* t = imm_node->bottom_type();
 2535   if (!(t->isa_int() || t->isa_long())) {
 2536     return false;
 2537   }
 2538 
 2539   switch (n->Opcode()) {
 2540   case Op_AndV:
 2541   case Op_OrV:
 2542   case Op_XorV: {
 2543     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
 2544     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2545     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2546   }
 2547   case Op_AddVB:
 2548     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2549   case Op_AddVS:
 2550   case Op_AddVI:
 2551     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2552   case Op_AddVL:
 2553     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2554   default:
 2555     return false;
 2556   }
 2557 }
 2558 
 2559 // (XorV src (Replicate m1))
 2560 // (XorVMask src (MaskAll m1))
 2561 bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
 2562   if (n != NULL && m != NULL) {
 2563     return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
 2564            VectorNode::is_all_ones_vector(m);
 2565   }
 2566   return false;
 2567 }
 2568 
 2569 // Should the matcher clone input 'm' of node 'n'?
 2570 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2571   if (is_vshift_con_pattern(n, m) ||
 2572       is_vector_bitwise_not_pattern(n, m) ||
 2573       is_valid_sve_arith_imm_pattern(n, m)) {
 2574     mstack.push(m, Visit);
 2575     return true;
 2576   }
 2577   return false;
 2578 }
 2579 
 2580 // Should the Matcher clone shifts on addressing modes, expecting them
 2581 // to be subsumed into complex addressing expressions or compute them
 2582 // into registers?
 2583 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2584   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2585     return true;
 2586   }
 2587 
 2588   Node *off = m->in(AddPNode::Offset);
 2589   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2590       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2591       // Are there other uses besides address expressions?
 2592       !is_visited(off)) {
 2593     address_visited.set(off->_idx); // Flag as address_visited
 2594     mstack.push(off->in(2), Visit);
 2595     Node *conv = off->in(1);
 2596     if (conv->Opcode() == Op_ConvI2L &&
 2597         // Are there other uses besides address expressions?
 2598         !is_visited(conv)) {
 2599       address_visited.set(conv->_idx); // Flag as address_visited
 2600       mstack.push(conv->in(1), Pre_Visit);
 2601     } else {
 2602       mstack.push(conv, Pre_Visit);
 2603     }
 2604     address_visited.test_set(m->_idx); // Flag as address_visited
 2605     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2606     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2607     return true;
 2608   } else if (off->Opcode() == Op_ConvI2L &&
 2609              // Are there other uses besides address expressions?
 2610              !is_visited(off)) {
 2611     address_visited.test_set(m->_idx); // Flag as address_visited
 2612     address_visited.set(off->_idx); // Flag as address_visited
 2613     mstack.push(off->in(1), Pre_Visit);
 2614     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2615     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2616     return true;
 2617   }
 2618   return false;
 2619 }
 2620 
 2621 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2622   C2_MacroAssembler _masm(&cbuf);                                       \
 2623   {                                                                     \
 2624     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2625     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2626     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2627     __ INSN(REG, as_Register(BASE));                                    \
 2628   }
 2629 
 2630 
 2631 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2632   {
 2633     Address::extend scale;
 2634 
 2635     // Hooboy, this is fugly.  We need a way to communicate to the
 2636     // encoder that the index needs to be sign extended, so we have to
 2637     // enumerate all the cases.
 2638     switch (opcode) {
 2639     case INDINDEXSCALEDI2L:
 2640     case INDINDEXSCALEDI2LN:
 2641     case INDINDEXI2L:
 2642     case INDINDEXI2LN:
 2643       scale = Address::sxtw(size);
 2644       break;
 2645     default:
 2646       scale = Address::lsl(size);
 2647     }
 2648 
 2649     if (index == -1) {
 2650       return Address(base, disp);
 2651     } else {
 2652       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2653       return Address(base, as_Register(index), scale);
 2654     }
 2655   }
 2656 
 2657 
 2658 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2659 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2660 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2661 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2662                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2663 
 2664   // Used for all non-volatile memory accesses.  The use of
 2665   // $mem->opcode() to discover whether this pattern uses sign-extended
 2666   // offsets is something of a kludge.
 2667   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2668                         Register reg, int opcode,
 2669                         Register base, int index, int scale, int disp,
 2670                         int size_in_memory)
 2671   {
 2672     Address addr = mem2address(opcode, base, index, scale, disp);
 2673     if (addr.getMode() == Address::base_plus_offset) {
 2674       /* If we get an out-of-range offset it is a bug in the compiler,
 2675          so we assert here. */
 2676       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2677              "c2 compiler bug");
 2678       /* Fix up any out-of-range offsets. */
 2679       assert_different_registers(rscratch1, base);
 2680       assert_different_registers(rscratch1, reg);
 2681       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2682     }
 2683     (masm.*insn)(reg, addr);
 2684   }
 2685 
 2686   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2687                         FloatRegister reg, int opcode,
 2688                         Register base, int index, int size, int disp,
 2689                         int size_in_memory)
 2690   {
 2691     Address::extend scale;
 2692 
 2693     switch (opcode) {
 2694     case INDINDEXSCALEDI2L:
 2695     case INDINDEXSCALEDI2LN:
 2696       scale = Address::sxtw(size);
 2697       break;
 2698     default:
 2699       scale = Address::lsl(size);
 2700     }
 2701 
 2702     if (index == -1) {
 2703       /* If we get an out-of-range offset it is a bug in the compiler,
 2704          so we assert here. */
 2705       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2706       /* Fix up any out-of-range offsets. */
 2707       assert_different_registers(rscratch1, base);
 2708       Address addr = Address(base, disp);
 2709       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2710       (masm.*insn)(reg, addr);
 2711     } else {
 2712       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2713       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2714     }
 2715   }
 2716 
 2717   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2718                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2719                         int opcode, Register base, int index, int size, int disp)
 2720   {
 2721     if (index == -1) {
 2722       (masm.*insn)(reg, T, Address(base, disp));
 2723     } else {
 2724       assert(disp == 0, "unsupported address mode");
 2725       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2726     }
 2727   }
 2728 
 2729 %}
 2730 
 2731 
 2732 
 2733 //----------ENCODING BLOCK-----------------------------------------------------
 2734 // This block specifies the encoding classes used by the compiler to
 2735 // output byte streams.  Encoding classes are parameterized macros
 2736 // used by Machine Instruction Nodes in order to generate the bit
 2737 // encoding of the instruction.  Operands specify their base encoding
 2738 // interface with the interface keyword.  There are currently
 2739 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2740 // COND_INTER.  REG_INTER causes an operand to generate a function
 2741 // which returns its register number when queried.  CONST_INTER causes
 2742 // an operand to generate a function which returns the value of the
 2743 // constant when queried.  MEMORY_INTER causes an operand to generate
 2744 // four functions which return the Base Register, the Index Register,
 2745 // the Scale Value, and the Offset Value of the operand when queried.
 2746 // COND_INTER causes an operand to generate six functions which return
 2747 // the encoding code (ie - encoding bits for the instruction)
 2748 // associated with each basic boolean condition for a conditional
 2749 // instruction.
 2750 //
 2751 // Instructions specify two basic values for encoding.  Again, a
 2752 // function is available to check if the constant displacement is an
 2753 // oop. They use the ins_encode keyword to specify their encoding
 2754 // classes (which must be a sequence of enc_class names, and their
 2755 // parameters, specified in the encoding block), and they use the
 2756 // opcode keyword to specify, in order, their primary, secondary, and
 2757 // tertiary opcode.  Only the opcode sections which a particular
 2758 // instruction needs for encoding need to be specified.
 2759 encode %{
 2760   // Build emit functions for each basic byte or larger field in the
 2761   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2762   // from C++ code in the enc_class source block.  Emit functions will
 2763   // live in the main source block for now.  In future, we can
 2764   // generalize this by adding a syntax that specifies the sizes of
 2765   // fields in an order, so that the adlc can build the emit functions
 2766   // automagically
 2767 
 2768   // catch all for unimplemented encodings
 2769   enc_class enc_unimplemented %{
 2770     C2_MacroAssembler _masm(&cbuf);
 2771     __ unimplemented("C2 catch all");
 2772   %}
 2773 
 2774   // BEGIN Non-volatile memory access
 2775 
 2776   // This encoding class is generated automatically from ad_encode.m4.
 2777   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2778   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2779     Register dst_reg = as_Register($dst$$reg);
 2780     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2781                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2782   %}
 2783 
 2784   // This encoding class is generated automatically from ad_encode.m4.
 2785   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2786   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2787     Register dst_reg = as_Register($dst$$reg);
 2788     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2789                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2790   %}
 2791 
 2792   // This encoding class is generated automatically from ad_encode.m4.
 2793   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2794   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2795     Register dst_reg = as_Register($dst$$reg);
 2796     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2797                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2798   %}
 2799 
 2800   // This encoding class is generated automatically from ad_encode.m4.
 2801   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2802   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2803     Register dst_reg = as_Register($dst$$reg);
 2804     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2805                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2806   %}
 2807 
 2808   // This encoding class is generated automatically from ad_encode.m4.
 2809   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2810   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2811     Register dst_reg = as_Register($dst$$reg);
 2812     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2813                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2814   %}
 2815 
 2816   // This encoding class is generated automatically from ad_encode.m4.
 2817   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2818   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2819     Register dst_reg = as_Register($dst$$reg);
 2820     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2821                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2822   %}
 2823 
 2824   // This encoding class is generated automatically from ad_encode.m4.
 2825   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2826   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2827     Register dst_reg = as_Register($dst$$reg);
 2828     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2829                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2830   %}
 2831 
 2832   // This encoding class is generated automatically from ad_encode.m4.
 2833   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2834   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2835     Register dst_reg = as_Register($dst$$reg);
 2836     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2837                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2838   %}
 2839 
 2840   // This encoding class is generated automatically from ad_encode.m4.
 2841   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2842   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2843     Register dst_reg = as_Register($dst$$reg);
 2844     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2845                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2846   %}
 2847 
 2848   // This encoding class is generated automatically from ad_encode.m4.
 2849   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2850   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2851     Register dst_reg = as_Register($dst$$reg);
 2852     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2853                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2854   %}
 2855 
 2856   // This encoding class is generated automatically from ad_encode.m4.
 2857   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2858   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2859     Register dst_reg = as_Register($dst$$reg);
 2860     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2861                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2862   %}
 2863 
 2864   // This encoding class is generated automatically from ad_encode.m4.
 2865   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2866   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2867     Register dst_reg = as_Register($dst$$reg);
 2868     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2869                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2870   %}
 2871 
 2872   // This encoding class is generated automatically from ad_encode.m4.
 2873   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2874   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2875     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2876     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2877                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2878   %}
 2879 
 2880   // This encoding class is generated automatically from ad_encode.m4.
 2881   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2882   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 2883     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2884     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2885                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2886   %}
 2887 
 2888   // This encoding class is generated automatically from ad_encode.m4.
 2889   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2890   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 2891     Register src_reg = as_Register($src$$reg);
 2892     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 2893                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2894   %}
 2895 
 2896   // This encoding class is generated automatically from ad_encode.m4.
 2897   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2898   enc_class aarch64_enc_strb0(memory1 mem) %{
 2899     C2_MacroAssembler _masm(&cbuf);
 2900     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2901                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2902   %}
 2903 
 2904   // This encoding class is generated automatically from ad_encode.m4.
 2905   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2906   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 2907     Register src_reg = as_Register($src$$reg);
 2908     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 2909                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2910   %}
 2911 
 2912   // This encoding class is generated automatically from ad_encode.m4.
 2913   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2914   enc_class aarch64_enc_strh0(memory2 mem) %{
 2915     C2_MacroAssembler _masm(&cbuf);
 2916     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 2917                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2918   %}
 2919 
 2920   // This encoding class is generated automatically from ad_encode.m4.
 2921   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2922   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 2923     Register src_reg = as_Register($src$$reg);
 2924     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 2925                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2926   %}
 2927 
 2928   // This encoding class is generated automatically from ad_encode.m4.
 2929   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2930   enc_class aarch64_enc_strw0(memory4 mem) %{
 2931     C2_MacroAssembler _masm(&cbuf);
 2932     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 2933                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2934   %}
 2935 
 2936   // This encoding class is generated automatically from ad_encode.m4.
 2937   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2938   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 2939     Register src_reg = as_Register($src$$reg);
 2940     // we sometimes get asked to store the stack pointer into the
 2941     // current thread -- we cannot do that directly on AArch64
 2942     if (src_reg == r31_sp) {
 2943       C2_MacroAssembler _masm(&cbuf);
 2944       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 2945       __ mov(rscratch2, sp);
 2946       src_reg = rscratch2;
 2947     }
 2948     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 2949                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2950   %}
 2951 
 2952   // This encoding class is generated automatically from ad_encode.m4.
 2953   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2954   enc_class aarch64_enc_str0(memory8 mem) %{
 2955     C2_MacroAssembler _masm(&cbuf);
 2956     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 2957                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2958   %}
 2959 
 2960   // This encoding class is generated automatically from ad_encode.m4.
 2961   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2962   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 2963     FloatRegister src_reg = as_FloatRegister($src$$reg);
 2964     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 2965                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2966   %}
 2967 
 2968   // This encoding class is generated automatically from ad_encode.m4.
 2969   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2970   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 2971     FloatRegister src_reg = as_FloatRegister($src$$reg);
 2972     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 2973                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2974   %}
 2975 
 2976   // This encoding class is generated automatically from ad_encode.m4.
 2977   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2978   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 2979       C2_MacroAssembler _masm(&cbuf);
 2980       __ membar(Assembler::StoreStore);
 2981       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2982                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2983   %}
 2984 
 2985   // END Non-volatile memory access
 2986 
 2987   // Vector loads and stores
 2988   enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
 2989     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2990     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 2991        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 2992   %}
 2993 
 2994   enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
 2995     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2996     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 2997        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 2998   %}
 2999 
 3000   enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
 3001     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3002     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3003        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3004   %}
 3005 
 3006   enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
 3007     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3008     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3009        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3010   %}
 3011 
 3012   enc_class aarch64_enc_strvH(vReg src, memory mem) %{
 3013     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3014     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3015        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3016   %}
 3017 
 3018   enc_class aarch64_enc_strvS(vReg src, memory mem) %{
 3019     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3020     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3021        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3022   %}
 3023 
 3024   enc_class aarch64_enc_strvD(vReg src, memory mem) %{
 3025     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3026     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3027        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3028   %}
 3029 
 3030   enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
 3031     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3032     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3033        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3034   %}
 3035 
 3036   // volatile loads and stores
 3037 
 3038   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3039     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3040                  rscratch1, stlrb);
 3041   %}
 3042 
 3043   enc_class aarch64_enc_stlrb0(memory mem) %{
 3044     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3045                  rscratch1, stlrb);
 3046   %}
 3047 
 3048   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3049     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3050                  rscratch1, stlrh);
 3051   %}
 3052 
 3053   enc_class aarch64_enc_stlrh0(memory mem) %{
 3054     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3055                  rscratch1, stlrh);
 3056   %}
 3057 
 3058   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3059     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3060                  rscratch1, stlrw);
 3061   %}
 3062 
 3063   enc_class aarch64_enc_stlrw0(memory mem) %{
 3064     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3065                  rscratch1, stlrw);
 3066   %}
 3067 
 3068   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3069     Register dst_reg = as_Register($dst$$reg);
 3070     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3071              rscratch1, ldarb);
 3072     __ sxtbw(dst_reg, dst_reg);
 3073   %}
 3074 
 3075   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3076     Register dst_reg = as_Register($dst$$reg);
 3077     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3078              rscratch1, ldarb);
 3079     __ sxtb(dst_reg, dst_reg);
 3080   %}
 3081 
 3082   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3083     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3084              rscratch1, ldarb);
 3085   %}
 3086 
 3087   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3088     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3089              rscratch1, ldarb);
 3090   %}
 3091 
 3092   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3093     Register dst_reg = as_Register($dst$$reg);
 3094     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3095              rscratch1, ldarh);
 3096     __ sxthw(dst_reg, dst_reg);
 3097   %}
 3098 
 3099   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3100     Register dst_reg = as_Register($dst$$reg);
 3101     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3102              rscratch1, ldarh);
 3103     __ sxth(dst_reg, dst_reg);
 3104   %}
 3105 
 3106   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3107     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3108              rscratch1, ldarh);
 3109   %}
 3110 
 3111   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3112     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3113              rscratch1, ldarh);
 3114   %}
 3115 
 3116   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3117     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3118              rscratch1, ldarw);
 3119   %}
 3120 
 3121   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3122     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3123              rscratch1, ldarw);
 3124   %}
 3125 
 3126   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3127     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3128              rscratch1, ldar);
 3129   %}
 3130 
 3131   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3132     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3133              rscratch1, ldarw);
 3134     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3135   %}
 3136 
 3137   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3138     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3139              rscratch1, ldar);
 3140     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3141   %}
 3142 
 3143   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3144     Register src_reg = as_Register($src$$reg);
 3145     // we sometimes get asked to store the stack pointer into the
 3146     // current thread -- we cannot do that directly on AArch64
 3147     if (src_reg == r31_sp) {
 3148       C2_MacroAssembler _masm(&cbuf);
 3149       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3150       __ mov(rscratch2, sp);
 3151       src_reg = rscratch2;
 3152     }
 3153     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3154                  rscratch1, stlr);
 3155   %}
 3156 
 3157   enc_class aarch64_enc_stlr0(memory mem) %{
 3158     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3159                  rscratch1, stlr);
 3160   %}
 3161 
 3162   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3163     {
 3164       C2_MacroAssembler _masm(&cbuf);
 3165       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3166       __ fmovs(rscratch2, src_reg);
 3167     }
 3168     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3169                  rscratch1, stlrw);
 3170   %}
 3171 
 3172   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3173     {
 3174       C2_MacroAssembler _masm(&cbuf);
 3175       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3176       __ fmovd(rscratch2, src_reg);
 3177     }
 3178     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3179                  rscratch1, stlr);
 3180   %}
 3181 
 3182   // synchronized read/update encodings
 3183 
 3184   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3185     C2_MacroAssembler _masm(&cbuf);
 3186     Register dst_reg = as_Register($dst$$reg);
 3187     Register base = as_Register($mem$$base);
 3188     int index = $mem$$index;
 3189     int scale = $mem$$scale;
 3190     int disp = $mem$$disp;
 3191     if (index == -1) {
 3192        if (disp != 0) {
 3193         __ lea(rscratch1, Address(base, disp));
 3194         __ ldaxr(dst_reg, rscratch1);
 3195       } else {
 3196         // TODO
 3197         // should we ever get anything other than this case?
 3198         __ ldaxr(dst_reg, base);
 3199       }
 3200     } else {
 3201       Register index_reg = as_Register(index);
 3202       if (disp == 0) {
 3203         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3204         __ ldaxr(dst_reg, rscratch1);
 3205       } else {
 3206         __ lea(rscratch1, Address(base, disp));
 3207         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3208         __ ldaxr(dst_reg, rscratch1);
 3209       }
 3210     }
 3211   %}
 3212 
 3213   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3214     C2_MacroAssembler _masm(&cbuf);
 3215     Register src_reg = as_Register($src$$reg);
 3216     Register base = as_Register($mem$$base);
 3217     int index = $mem$$index;
 3218     int scale = $mem$$scale;
 3219     int disp = $mem$$disp;
 3220     if (index == -1) {
 3221        if (disp != 0) {
 3222         __ lea(rscratch2, Address(base, disp));
 3223         __ stlxr(rscratch1, src_reg, rscratch2);
 3224       } else {
 3225         // TODO
 3226         // should we ever get anything other than this case?
 3227         __ stlxr(rscratch1, src_reg, base);
 3228       }
 3229     } else {
 3230       Register index_reg = as_Register(index);
 3231       if (disp == 0) {
 3232         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3233         __ stlxr(rscratch1, src_reg, rscratch2);
 3234       } else {
 3235         __ lea(rscratch2, Address(base, disp));
 3236         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3237         __ stlxr(rscratch1, src_reg, rscratch2);
 3238       }
 3239     }
 3240     __ cmpw(rscratch1, zr);
 3241   %}
 3242 
 3243   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3244     C2_MacroAssembler _masm(&cbuf);
 3245     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3246     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3247                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3248                /*weak*/ false, noreg);
 3249   %}
 3250 
 3251   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3252     C2_MacroAssembler _masm(&cbuf);
 3253     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3254     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3255                Assembler::word, /*acquire*/ false, /*release*/ true,
 3256                /*weak*/ false, noreg);
 3257   %}
 3258 
 3259   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3260     C2_MacroAssembler _masm(&cbuf);
 3261     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3262     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3263                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3264                /*weak*/ false, noreg);
 3265   %}
 3266 
 3267   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3268     C2_MacroAssembler _masm(&cbuf);
 3269     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3270     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3271                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3272                /*weak*/ false, noreg);
 3273   %}
 3274 
 3275 
 3276   // The only difference between aarch64_enc_cmpxchg and
 3277   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3278   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3279   // lock.
 3280   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3281     C2_MacroAssembler _masm(&cbuf);
 3282     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3283     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3284                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3285                /*weak*/ false, noreg);
 3286   %}
 3287 
 3288   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3289     C2_MacroAssembler _masm(&cbuf);
 3290     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3291     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3292                Assembler::word, /*acquire*/ true, /*release*/ true,
 3293                /*weak*/ false, noreg);
 3294   %}
 3295 
 3296   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3297     C2_MacroAssembler _masm(&cbuf);
 3298     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3299     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3300                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3301                /*weak*/ false, noreg);
 3302   %}
 3303 
 3304   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3305     C2_MacroAssembler _masm(&cbuf);
 3306     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3307     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3308                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3309                /*weak*/ false, noreg);
 3310   %}
 3311 
 3312   // auxiliary used for CompareAndSwapX to set result register
 3313   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3314     C2_MacroAssembler _masm(&cbuf);
 3315     Register res_reg = as_Register($res$$reg);
 3316     __ cset(res_reg, Assembler::EQ);
 3317   %}
 3318 
 3319   // prefetch encodings
 3320 
 3321   enc_class aarch64_enc_prefetchw(memory mem) %{
 3322     C2_MacroAssembler _masm(&cbuf);
 3323     Register base = as_Register($mem$$base);
 3324     int index = $mem$$index;
 3325     int scale = $mem$$scale;
 3326     int disp = $mem$$disp;
 3327     if (index == -1) {
 3328       __ prfm(Address(base, disp), PSTL1KEEP);
 3329     } else {
 3330       Register index_reg = as_Register(index);
 3331       if (disp == 0) {
 3332         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3333       } else {
 3334         __ lea(rscratch1, Address(base, disp));
 3335 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3336       }
 3337     }
 3338   %}
 3339 
 3340   /// mov envcodings
 3341 
 3342   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3343     C2_MacroAssembler _masm(&cbuf);
 3344     uint32_t con = (uint32_t)$src$$constant;
 3345     Register dst_reg = as_Register($dst$$reg);
 3346     if (con == 0) {
 3347       __ movw(dst_reg, zr);
 3348     } else {
 3349       __ movw(dst_reg, con);
 3350     }
 3351   %}
 3352 
 3353   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3354     C2_MacroAssembler _masm(&cbuf);
 3355     Register dst_reg = as_Register($dst$$reg);
 3356     uint64_t con = (uint64_t)$src$$constant;
 3357     if (con == 0) {
 3358       __ mov(dst_reg, zr);
 3359     } else {
 3360       __ mov(dst_reg, con);
 3361     }
 3362   %}
 3363 
 3364   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3365     C2_MacroAssembler _masm(&cbuf);
 3366     Register dst_reg = as_Register($dst$$reg);
 3367     address con = (address)$src$$constant;
 3368     if (con == NULL || con == (address)1) {
 3369       ShouldNotReachHere();
 3370     } else {
 3371       relocInfo::relocType rtype = $src->constant_reloc();
 3372       if (rtype == relocInfo::oop_type) {
 3373         __ movoop(dst_reg, (jobject)con);
 3374       } else if (rtype == relocInfo::metadata_type) {
 3375         __ mov_metadata(dst_reg, (Metadata*)con);
 3376       } else {
 3377         assert(rtype == relocInfo::none, "unexpected reloc type");
 3378         if (con < (address)(uintptr_t)os::vm_page_size()) {
 3379           __ mov(dst_reg, con);
 3380         } else {
 3381           uint64_t offset;
 3382           __ adrp(dst_reg, con, offset);
 3383           __ add(dst_reg, dst_reg, offset);
 3384         }
 3385       }
 3386     }
 3387   %}
 3388 
 3389   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3390     C2_MacroAssembler _masm(&cbuf);
 3391     Register dst_reg = as_Register($dst$$reg);
 3392     __ mov(dst_reg, zr);
 3393   %}
 3394 
 3395   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3396     C2_MacroAssembler _masm(&cbuf);
 3397     Register dst_reg = as_Register($dst$$reg);
 3398     __ mov(dst_reg, (uint64_t)1);
 3399   %}
 3400 
 3401   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3402     C2_MacroAssembler _masm(&cbuf);
 3403     __ load_byte_map_base($dst$$Register);
 3404   %}
 3405 
 3406   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3407     C2_MacroAssembler _masm(&cbuf);
 3408     Register dst_reg = as_Register($dst$$reg);
 3409     address con = (address)$src$$constant;
 3410     if (con == NULL) {
 3411       ShouldNotReachHere();
 3412     } else {
 3413       relocInfo::relocType rtype = $src->constant_reloc();
 3414       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3415       __ set_narrow_oop(dst_reg, (jobject)con);
 3416     }
 3417   %}
 3418 
 3419   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3420     C2_MacroAssembler _masm(&cbuf);
 3421     Register dst_reg = as_Register($dst$$reg);
 3422     __ mov(dst_reg, zr);
 3423   %}
 3424 
 3425   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3426     C2_MacroAssembler _masm(&cbuf);
 3427     Register dst_reg = as_Register($dst$$reg);
 3428     address con = (address)$src$$constant;
 3429     if (con == NULL) {
 3430       ShouldNotReachHere();
 3431     } else {
 3432       relocInfo::relocType rtype = $src->constant_reloc();
 3433       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3434       __ set_narrow_klass(dst_reg, (Klass *)con);
 3435     }
 3436   %}
 3437 
 3438   // arithmetic encodings
 3439 
 3440   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3441     C2_MacroAssembler _masm(&cbuf);
 3442     Register dst_reg = as_Register($dst$$reg);
 3443     Register src_reg = as_Register($src1$$reg);
 3444     int32_t con = (int32_t)$src2$$constant;
 3445     // add has primary == 0, subtract has primary == 1
 3446     if ($primary) { con = -con; }
 3447     if (con < 0) {
 3448       __ subw(dst_reg, src_reg, -con);
 3449     } else {
 3450       __ addw(dst_reg, src_reg, con);
 3451     }
 3452   %}
 3453 
 3454   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3455     C2_MacroAssembler _masm(&cbuf);
 3456     Register dst_reg = as_Register($dst$$reg);
 3457     Register src_reg = as_Register($src1$$reg);
 3458     int32_t con = (int32_t)$src2$$constant;
 3459     // add has primary == 0, subtract has primary == 1
 3460     if ($primary) { con = -con; }
 3461     if (con < 0) {
 3462       __ sub(dst_reg, src_reg, -con);
 3463     } else {
 3464       __ add(dst_reg, src_reg, con);
 3465     }
 3466   %}
 3467 
 3468   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3469     C2_MacroAssembler _masm(&cbuf);
 3470    Register dst_reg = as_Register($dst$$reg);
 3471    Register src1_reg = as_Register($src1$$reg);
 3472    Register src2_reg = as_Register($src2$$reg);
 3473     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3474   %}
 3475 
 3476   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3477     C2_MacroAssembler _masm(&cbuf);
 3478    Register dst_reg = as_Register($dst$$reg);
 3479    Register src1_reg = as_Register($src1$$reg);
 3480    Register src2_reg = as_Register($src2$$reg);
 3481     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3482   %}
 3483 
 3484   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3485     C2_MacroAssembler _masm(&cbuf);
 3486    Register dst_reg = as_Register($dst$$reg);
 3487    Register src1_reg = as_Register($src1$$reg);
 3488    Register src2_reg = as_Register($src2$$reg);
 3489     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3490   %}
 3491 
 3492   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3493     C2_MacroAssembler _masm(&cbuf);
 3494    Register dst_reg = as_Register($dst$$reg);
 3495    Register src1_reg = as_Register($src1$$reg);
 3496    Register src2_reg = as_Register($src2$$reg);
 3497     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3498   %}
 3499 
 3500   // compare instruction encodings
 3501 
 3502   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3503     C2_MacroAssembler _masm(&cbuf);
 3504     Register reg1 = as_Register($src1$$reg);
 3505     Register reg2 = as_Register($src2$$reg);
 3506     __ cmpw(reg1, reg2);
 3507   %}
 3508 
 3509   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3510     C2_MacroAssembler _masm(&cbuf);
 3511     Register reg = as_Register($src1$$reg);
 3512     int32_t val = $src2$$constant;
 3513     if (val >= 0) {
 3514       __ subsw(zr, reg, val);
 3515     } else {
 3516       __ addsw(zr, reg, -val);
 3517     }
 3518   %}
 3519 
 3520   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3521     C2_MacroAssembler _masm(&cbuf);
 3522     Register reg1 = as_Register($src1$$reg);
 3523     uint32_t val = (uint32_t)$src2$$constant;
 3524     __ movw(rscratch1, val);
 3525     __ cmpw(reg1, rscratch1);
 3526   %}
 3527 
 3528   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3529     C2_MacroAssembler _masm(&cbuf);
 3530     Register reg1 = as_Register($src1$$reg);
 3531     Register reg2 = as_Register($src2$$reg);
 3532     __ cmp(reg1, reg2);
 3533   %}
 3534 
 3535   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3536     C2_MacroAssembler _masm(&cbuf);
 3537     Register reg = as_Register($src1$$reg);
 3538     int64_t val = $src2$$constant;
 3539     if (val >= 0) {
 3540       __ subs(zr, reg, val);
 3541     } else if (val != -val) {
 3542       __ adds(zr, reg, -val);
 3543     } else {
 3544     // aargh, Long.MIN_VALUE is a special case
 3545       __ orr(rscratch1, zr, (uint64_t)val);
 3546       __ subs(zr, reg, rscratch1);
 3547     }
 3548   %}
 3549 
 3550   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3551     C2_MacroAssembler _masm(&cbuf);
 3552     Register reg1 = as_Register($src1$$reg);
 3553     uint64_t val = (uint64_t)$src2$$constant;
 3554     __ mov(rscratch1, val);
 3555     __ cmp(reg1, rscratch1);
 3556   %}
 3557 
 3558   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3559     C2_MacroAssembler _masm(&cbuf);
 3560     Register reg1 = as_Register($src1$$reg);
 3561     Register reg2 = as_Register($src2$$reg);
 3562     __ cmp(reg1, reg2);
 3563   %}
 3564 
 3565   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3566     C2_MacroAssembler _masm(&cbuf);
 3567     Register reg1 = as_Register($src1$$reg);
 3568     Register reg2 = as_Register($src2$$reg);
 3569     __ cmpw(reg1, reg2);
 3570   %}
 3571 
 3572   enc_class aarch64_enc_testp(iRegP src) %{
 3573     C2_MacroAssembler _masm(&cbuf);
 3574     Register reg = as_Register($src$$reg);
 3575     __ cmp(reg, zr);
 3576   %}
 3577 
 3578   enc_class aarch64_enc_testn(iRegN src) %{
 3579     C2_MacroAssembler _masm(&cbuf);
 3580     Register reg = as_Register($src$$reg);
 3581     __ cmpw(reg, zr);
 3582   %}
 3583 
 3584   enc_class aarch64_enc_b(label lbl) %{
 3585     C2_MacroAssembler _masm(&cbuf);
 3586     Label *L = $lbl$$label;
 3587     __ b(*L);
 3588   %}
 3589 
 3590   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3591     C2_MacroAssembler _masm(&cbuf);
 3592     Label *L = $lbl$$label;
 3593     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3594   %}
 3595 
 3596   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3597     C2_MacroAssembler _masm(&cbuf);
 3598     Label *L = $lbl$$label;
 3599     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3600   %}
 3601 
 3602   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3603   %{
 3604      Register sub_reg = as_Register($sub$$reg);
 3605      Register super_reg = as_Register($super$$reg);
 3606      Register temp_reg = as_Register($temp$$reg);
 3607      Register result_reg = as_Register($result$$reg);
 3608 
 3609      Label miss;
 3610      C2_MacroAssembler _masm(&cbuf);
 3611      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3612                                      NULL, &miss,
 3613                                      /*set_cond_codes:*/ true);
 3614      if ($primary) {
 3615        __ mov(result_reg, zr);
 3616      }
 3617      __ bind(miss);
 3618   %}
 3619 
 3620   enc_class aarch64_enc_java_static_call(method meth) %{
 3621     C2_MacroAssembler _masm(&cbuf);
 3622 
 3623     address addr = (address)$meth$$method;
 3624     address call;
 3625     if (!_method) {
 3626       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3627       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
 3628       if (call == NULL) {
 3629         ciEnv::current()->record_failure("CodeCache is full");
 3630         return;
 3631       }
 3632     } else {
 3633       int method_index = resolved_method_index(cbuf);
 3634       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3635                                                   : static_call_Relocation::spec(method_index);
 3636       call = __ trampoline_call(Address(addr, rspec));
 3637       if (call == NULL) {
 3638         ciEnv::current()->record_failure("CodeCache is full");
 3639         return;
 3640       }
 3641       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 3642         // Calls of the same statically bound method can share
 3643         // a stub to the interpreter.
 3644         cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
 3645       } else {
 3646         // Emit stub for static call
 3647         address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
 3648         if (stub == NULL) {
 3649           ciEnv::current()->record_failure("CodeCache is full");
 3650           return;
 3651         }
 3652       }
 3653     }
 3654 
 3655     __ post_call_nop();
 3656 
 3657     // Only non uncommon_trap calls need to reinitialize ptrue.
 3658     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3659       __ reinitialize_ptrue();
 3660     }
 3661   %}
 3662 
 3663   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3664     C2_MacroAssembler _masm(&cbuf);
 3665     int method_index = resolved_method_index(cbuf);
 3666     address call = __ ic_call((address)$meth$$method, method_index);
 3667     if (call == NULL) {
 3668       ciEnv::current()->record_failure("CodeCache is full");
 3669       return;
 3670     }
 3671     _masm.clear_inst_mark();
 3672     __ post_call_nop();
 3673     if (Compile::current()->max_vector_size() > 0) {
 3674       __ reinitialize_ptrue();
 3675     }
 3676   %}
 3677 
 3678   enc_class aarch64_enc_call_epilog() %{
 3679     C2_MacroAssembler _masm(&cbuf);
 3680     if (VerifyStackAtCalls) {
 3681       // Check that stack depth is unchanged: find majik cookie on stack
 3682       __ call_Unimplemented();
 3683     }
 3684     if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic()) {
 3685       if (!_method->signature()->returns_null_free_inline_type()) {
 3686         // The last return value is not set by the callee but used to pass IsInit information to compiled code.
 3687         // Search for the corresponding projection, get the register and emit code that initialized it.
 3688         uint con = (tf()->range_cc()->cnt() - 1);
 3689         for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
 3690           ProjNode* proj = fast_out(i)->as_Proj();
 3691           if (proj->_con == con) {
 3692             // Set IsInit if r0 is non-null (a non-null value is returned buffered or scalarized)
 3693             OptoReg::Name optoReg = ra_->get_reg_first(proj);
 3694             VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
 3695             Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
 3696             __ cmp(r0, zr);
 3697             __ cset(toReg, Assembler::NE);
 3698             if (reg->is_stack()) {
 3699               int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
 3700               __ str(toReg, Address(sp, st_off));
 3701             }
 3702             break;
 3703           }
 3704         }
 3705       }
 3706       if (return_value_is_used()) {
 3707         // An inline type is returned as fields in multiple registers.
 3708         // R0 either contains an oop if the inline type is buffered or a pointer
 3709         // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
 3710         // if the lowest bit is set to allow C2 to use the oop after null checking.
 3711         // r0 &= (r0 & 1) - 1
 3712         __ andr(rscratch1, r0, 0x1);
 3713         __ sub(rscratch1, rscratch1, 0x1);
 3714         __ andr(r0, r0, rscratch1);
 3715       }
 3716     }
 3717   %}
 3718 
 3719   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3720     C2_MacroAssembler _masm(&cbuf);
 3721 
 3722     // some calls to generated routines (arraycopy code) are scheduled
 3723     // by C2 as runtime calls. if so we can call them using a br (they
 3724     // will be in a reachable segment) otherwise we have to use a blr
 3725     // which loads the absolute address into a register.
 3726     address entry = (address)$meth$$method;
 3727     CodeBlob *cb = CodeCache::find_blob(entry);
 3728     if (cb) {
 3729       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3730       if (call == NULL) {
 3731         ciEnv::current()->record_failure("CodeCache is full");
 3732         return;
 3733       }
 3734       __ post_call_nop();
 3735     } else {
 3736       Label retaddr;
 3737       __ adr(rscratch2, retaddr);
 3738       __ lea(rscratch1, RuntimeAddress(entry));
 3739       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3740       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3741       __ blr(rscratch1);
 3742       __ bind(retaddr);
 3743       __ post_call_nop();
 3744       __ add(sp, sp, 2 * wordSize);
 3745     }
 3746     if (Compile::current()->max_vector_size() > 0) {
 3747       __ reinitialize_ptrue();
 3748     }
 3749   %}
 3750 
 3751   enc_class aarch64_enc_rethrow() %{
 3752     C2_MacroAssembler _masm(&cbuf);
 3753     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3754   %}
 3755 
 3756   enc_class aarch64_enc_ret() %{
 3757     C2_MacroAssembler _masm(&cbuf);
 3758 #ifdef ASSERT
 3759     if (Compile::current()->max_vector_size() > 0) {
 3760       __ verify_ptrue();
 3761     }
 3762 #endif
 3763     __ ret(lr);
 3764   %}
 3765 
 3766   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3767     C2_MacroAssembler _masm(&cbuf);
 3768     Register target_reg = as_Register($jump_target$$reg);
 3769     __ br(target_reg);
 3770   %}
 3771 
 3772   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3773     C2_MacroAssembler _masm(&cbuf);
 3774     Register target_reg = as_Register($jump_target$$reg);
 3775     // exception oop should be in r0
 3776     // ret addr has been popped into lr
 3777     // callee expects it in r3
 3778     __ mov(r3, lr);
 3779     __ br(target_reg);
 3780   %}
 3781 
 3782   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3783     C2_MacroAssembler _masm(&cbuf);
 3784     Register oop = as_Register($object$$reg);
 3785     Register box = as_Register($box$$reg);
 3786     Register disp_hdr = as_Register($tmp$$reg);
 3787     Register tmp = as_Register($tmp2$$reg);
 3788     Label cont;
 3789     Label object_has_monitor;
 3790     Label no_count;
 3791 
 3792     assert_different_registers(oop, box, tmp, disp_hdr);
 3793 
 3794     // Load markWord from object into displaced_header.
 3795     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3796 
 3797     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3798       __ load_klass(tmp, oop);
 3799       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3800       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3801       __ br(Assembler::NE, cont);
 3802     }
 3803 
 3804     // Check for existing monitor
 3805     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3806 
 3807     if (!UseHeavyMonitors) {
 3808       // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3809       __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3810 
 3811       if (EnableValhalla) {
 3812         // Mask inline_type bit such that we go to the slow path if object is an inline type
 3813         __ andr(tmp, tmp, ~((int) markWord::inline_type_bit_in_place));
 3814       }
 3815 
 3816       // Initialize the box. (Must happen before we update the object mark!)
 3817       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3818 
 3819       // Compare object markWord with an unlocked value (tmp) and if
 3820       // equal exchange the stack address of our box with object markWord.
 3821       // On failure disp_hdr contains the possibly locked markWord.
 3822       __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3823                  /*release*/ true, /*weak*/ false, disp_hdr);
 3824       __ br(Assembler::EQ, cont);
 3825 
 3826       assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3827 
 3828       // If the compare-and-exchange succeeded, then we found an unlocked
 3829       // object, will have now locked it will continue at label cont
 3830 
 3831       // Check if the owner is self by comparing the value in the
 3832       // markWord of object (disp_hdr) with the stack pointer.
 3833       __ mov(rscratch1, sp);
 3834       __ sub(disp_hdr, disp_hdr, rscratch1);
 3835       __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3836       // If condition is true we are cont and hence we can store 0 as the
 3837       // displaced header in the box, which indicates that it is a recursive lock.
 3838       __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3839       __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3840     } else {
 3841       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3842     }
 3843     __ b(cont);
 3844 
 3845     // Handle existing monitor.
 3846     __ bind(object_has_monitor);
 3847 
 3848     // The object's monitor m is unlocked iff m->owner == NULL,
 3849     // otherwise m->owner may contain a thread or a stack address.
 3850     //
 3851     // Try to CAS m->owner from NULL to current thread.
 3852     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3853     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3854                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 3855 
 3856     // Store a non-null value into the box to avoid looking like a re-entrant
 3857     // lock. The fast-path monitor unlock code checks for
 3858     // markWord::monitor_value so use markWord::unused_mark which has the
 3859     // relevant bit set, and also matches ObjectSynchronizer::enter.
 3860     __ mov(tmp, (address)markWord::unused_mark().value());
 3861     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3862 
 3863     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 3864 
 3865     __ cmp(rscratch1, rthread);
 3866     __ br(Assembler::NE, cont); // Check for recursive locking
 3867 
 3868     // Recursive lock case
 3869     __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
 3870     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 3871 
 3872     __ bind(cont);
 3873     // flag == EQ indicates success
 3874     // flag == NE indicates failure
 3875     __ br(Assembler::NE, no_count);
 3876 
 3877     __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
 3878 
 3879     __ bind(no_count);
 3880   %}
 3881 
 3882   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3883     C2_MacroAssembler _masm(&cbuf);
 3884     Register oop = as_Register($object$$reg);
 3885     Register box = as_Register($box$$reg);
 3886     Register disp_hdr = as_Register($tmp$$reg);
 3887     Register tmp = as_Register($tmp2$$reg);
 3888     Label cont;
 3889     Label object_has_monitor;
 3890     Label no_count;
 3891 
 3892     assert_different_registers(oop, box, tmp, disp_hdr);
 3893 
 3894     if (!UseHeavyMonitors) {
 3895       // Find the lock address and load the displaced header from the stack.
 3896       __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3897 
 3898       // If the displaced header is 0, we have a recursive unlock.
 3899       __ cmp(disp_hdr, zr);
 3900       __ br(Assembler::EQ, cont);
 3901     }
 3902 
 3903     // Handle existing monitor.
 3904     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 3905     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3906 
 3907     if (!UseHeavyMonitors) {
 3908       // Check if it is still a light weight lock, this is is true if we
 3909       // see the stack address of the basicLock in the markWord of the
 3910       // object.
 3911 
 3912       __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 3913                  /*release*/ true, /*weak*/ false, tmp);
 3914     } else {
 3915       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3916     }
 3917     __ b(cont);
 3918 
 3919     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3920 
 3921     // Handle existing monitor.
 3922     __ bind(object_has_monitor);
 3923     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 3924     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 3925     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3926 
 3927     Label notRecursive;
 3928     __ cbz(disp_hdr, notRecursive);
 3929 
 3930     // Recursive lock
 3931     __ sub(disp_hdr, disp_hdr, 1u);
 3932     __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3933     __ cmp(disp_hdr, disp_hdr); // Sets flags for result
 3934     __ b(cont);
 3935 
 3936     __ bind(notRecursive);
 3937     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 3938     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 3939     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 3940     __ cmp(rscratch1, zr); // Sets flags for result
 3941     __ cbnz(rscratch1, cont);
 3942     // need a release store here
 3943     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3944     __ stlr(zr, tmp); // set unowned
 3945 
 3946     __ bind(cont);
 3947     // flag == EQ indicates success
 3948     // flag == NE indicates failure
 3949     __ br(Assembler::NE, no_count);
 3950 
 3951     __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
 3952 
 3953     __ bind(no_count);
 3954   %}
 3955 
 3956 %}
 3957 
 3958 //----------FRAME--------------------------------------------------------------
 3959 // Definition of frame structure and management information.
 3960 //
 3961 //  S T A C K   L A Y O U T    Allocators stack-slot number
 3962 //                             |   (to get allocators register number
 3963 //  G  Owned by    |        |  v    add OptoReg::stack0())
 3964 //  r   CALLER     |        |
 3965 //  o     |        +--------+      pad to even-align allocators stack-slot
 3966 //  w     V        |  pad0  |        numbers; owned by CALLER
 3967 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 3968 //  h     ^        |   in   |  5
 3969 //        |        |  args  |  4   Holes in incoming args owned by SELF
 3970 //  |     |        |        |  3
 3971 //  |     |        +--------+
 3972 //  V     |        | old out|      Empty on Intel, window on Sparc
 3973 //        |    old |preserve|      Must be even aligned.
 3974 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 3975 //        |        |   in   |  3   area for Intel ret address
 3976 //     Owned by    |preserve|      Empty on Sparc.
 3977 //       SELF      +--------+
 3978 //        |        |  pad2  |  2   pad to align old SP
 3979 //        |        +--------+  1
 3980 //        |        | locks  |  0
 3981 //        |        +--------+----> OptoReg::stack0(), even aligned
 3982 //        |        |  pad1  | 11   pad to align new SP
 3983 //        |        +--------+
 3984 //        |        |        | 10
 3985 //        |        | spills |  9   spills
 3986 //        V        |        |  8   (pad0 slot for callee)
 3987 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 3988 //        ^        |  out   |  7
 3989 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 3990 //     Owned by    +--------+
 3991 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 3992 //        |    new |preserve|      Must be even-aligned.
 3993 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 3994 //        |        |        |
 3995 //
 3996 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 3997 //         known from SELF's arguments and the Java calling convention.
 3998 //         Region 6-7 is determined per call site.
 3999 // Note 2: If the calling convention leaves holes in the incoming argument
 4000 //         area, those holes are owned by SELF.  Holes in the outgoing area
 4001 //         are owned by the CALLEE.  Holes should not be necessary in the
 4002 //         incoming area, as the Java calling convention is completely under
 4003 //         the control of the AD file.  Doubles can be sorted and packed to
 4004 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 4005 //         varargs C calling conventions.
 4006 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 4007 //         even aligned with pad0 as needed.
 4008 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 4009 //           (the latter is true on Intel but is it false on AArch64?)
 4010 //         region 6-11 is even aligned; it may be padded out more so that
 4011 //         the region from SP to FP meets the minimum stack alignment.
 4012 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 4013 //         alignment.  Region 11, pad1, may be dynamically extended so that
 4014 //         SP meets the minimum alignment.
 4015 
 4016 frame %{
 4017   // These three registers define part of the calling convention
 4018   // between compiled code and the interpreter.
 4019 
 4020   // Inline Cache Register or Method for I2C.
 4021   inline_cache_reg(R12);
 4022 
 4023   // Number of stack slots consumed by locking an object
 4024   sync_stack_slots(2);
 4025 
 4026   // Compiled code's Frame Pointer
 4027   frame_pointer(R31);
 4028 
 4029   // Interpreter stores its frame pointer in a register which is
 4030   // stored to the stack by I2CAdaptors.
 4031   // I2CAdaptors convert from interpreted java to compiled java.
 4032   interpreter_frame_pointer(R29);
 4033 
 4034   // Stack alignment requirement
 4035   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4036 
 4037   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4038   // for calls to C.  Supports the var-args backing area for register parms.
 4039   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4040 
 4041   // The after-PROLOG location of the return address.  Location of
 4042   // return address specifies a type (REG or STACK) and a number
 4043   // representing the register number (i.e. - use a register name) or
 4044   // stack slot.
 4045   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4046   // Otherwise, it is above the locks and verification slot and alignment word
 4047   // TODO this may well be correct but need to check why that - 2 is there
 4048   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4049   // which folds in the space used for monitors
 4050   return_addr(STACK - 2 +
 4051               align_up((Compile::current()->in_preserve_stack_slots() +
 4052                         Compile::current()->fixed_slots()),
 4053                        stack_alignment_in_slots()));
 4054 
 4055   // Location of compiled Java return values.  Same as C for now.
 4056   return_value
 4057   %{
 4058     // TODO do we allow ideal_reg == Op_RegN???
 4059     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4060            "only return normal values");
 4061 
 4062     static const int lo[Op_RegL + 1] = { // enum name
 4063       0,                                 // Op_Node
 4064       0,                                 // Op_Set
 4065       R0_num,                            // Op_RegN
 4066       R0_num,                            // Op_RegI
 4067       R0_num,                            // Op_RegP
 4068       V0_num,                            // Op_RegF
 4069       V0_num,                            // Op_RegD
 4070       R0_num                             // Op_RegL
 4071     };
 4072 
 4073     static const int hi[Op_RegL + 1] = { // enum name
 4074       0,                                 // Op_Node
 4075       0,                                 // Op_Set
 4076       OptoReg::Bad,                      // Op_RegN
 4077       OptoReg::Bad,                      // Op_RegI
 4078       R0_H_num,                          // Op_RegP
 4079       OptoReg::Bad,                      // Op_RegF
 4080       V0_H_num,                          // Op_RegD
 4081       R0_H_num                           // Op_RegL
 4082     };
 4083 
 4084     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4085   %}
 4086 %}
 4087 
 4088 //----------ATTRIBUTES---------------------------------------------------------
 4089 //----------Operand Attributes-------------------------------------------------
 4090 op_attrib op_cost(1);        // Required cost attribute
 4091 
 4092 //----------Instruction Attributes---------------------------------------------
 4093 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4094 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4095 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4096                                 // a non-matching short branch variant
 4097                                 // of some long branch?
 4098 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4099                                 // be a power of 2) specifies the
 4100                                 // alignment that some part of the
 4101                                 // instruction (not necessarily the
 4102                                 // start) requires.  If > 1, a
 4103                                 // compute_padding() function must be
 4104                                 // provided for the instruction
 4105 
 4106 //----------OPERANDS-----------------------------------------------------------
 4107 // Operand definitions must precede instruction definitions for correct parsing
 4108 // in the ADLC because operands constitute user defined types which are used in
 4109 // instruction definitions.
 4110 
 4111 //----------Simple Operands----------------------------------------------------
 4112 
 4113 // Integer operands 32 bit
 4114 // 32 bit immediate
 4115 operand immI()
 4116 %{
 4117   match(ConI);
 4118 
 4119   op_cost(0);
 4120   format %{ %}
 4121   interface(CONST_INTER);
 4122 %}
 4123 
 4124 // 32 bit zero
 4125 operand immI0()
 4126 %{
 4127   predicate(n->get_int() == 0);
 4128   match(ConI);
 4129 
 4130   op_cost(0);
 4131   format %{ %}
 4132   interface(CONST_INTER);
 4133 %}
 4134 
 4135 // 32 bit unit increment
 4136 operand immI_1()
 4137 %{
 4138   predicate(n->get_int() == 1);
 4139   match(ConI);
 4140 
 4141   op_cost(0);
 4142   format %{ %}
 4143   interface(CONST_INTER);
 4144 %}
 4145 
 4146 // 32 bit unit decrement
 4147 operand immI_M1()
 4148 %{
 4149   predicate(n->get_int() == -1);
 4150   match(ConI);
 4151 
 4152   op_cost(0);
 4153   format %{ %}
 4154   interface(CONST_INTER);
 4155 %}
 4156 
 4157 // Shift values for add/sub extension shift
 4158 operand immIExt()
 4159 %{
 4160   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4161   match(ConI);
 4162 
 4163   op_cost(0);
 4164   format %{ %}
 4165   interface(CONST_INTER);
 4166 %}
 4167 
 4168 operand immI_gt_1()
 4169 %{
 4170   predicate(n->get_int() > 1);
 4171   match(ConI);
 4172 
 4173   op_cost(0);
 4174   format %{ %}
 4175   interface(CONST_INTER);
 4176 %}
 4177 
 4178 operand immI_le_4()
 4179 %{
 4180   predicate(n->get_int() <= 4);
 4181   match(ConI);
 4182 
 4183   op_cost(0);
 4184   format %{ %}
 4185   interface(CONST_INTER);
 4186 %}
 4187 
 4188 operand immI_16()
 4189 %{
 4190   predicate(n->get_int() == 16);
 4191   match(ConI);
 4192 
 4193   op_cost(0);
 4194   format %{ %}
 4195   interface(CONST_INTER);
 4196 %}
 4197 
 4198 operand immI_24()
 4199 %{
 4200   predicate(n->get_int() == 24);
 4201   match(ConI);
 4202 
 4203   op_cost(0);
 4204   format %{ %}
 4205   interface(CONST_INTER);
 4206 %}
 4207 
 4208 operand immI_32()
 4209 %{
 4210   predicate(n->get_int() == 32);
 4211   match(ConI);
 4212 
 4213   op_cost(0);
 4214   format %{ %}
 4215   interface(CONST_INTER);
 4216 %}
 4217 
 4218 operand immI_48()
 4219 %{
 4220   predicate(n->get_int() == 48);
 4221   match(ConI);
 4222 
 4223   op_cost(0);
 4224   format %{ %}
 4225   interface(CONST_INTER);
 4226 %}
 4227 
 4228 operand immI_56()
 4229 %{
 4230   predicate(n->get_int() == 56);
 4231   match(ConI);
 4232 
 4233   op_cost(0);
 4234   format %{ %}
 4235   interface(CONST_INTER);
 4236 %}
 4237 
 4238 operand immI_63()
 4239 %{
 4240   predicate(n->get_int() == 63);
 4241   match(ConI);
 4242 
 4243   op_cost(0);
 4244   format %{ %}
 4245   interface(CONST_INTER);
 4246 %}
 4247 
 4248 operand immI_64()
 4249 %{
 4250   predicate(n->get_int() == 64);
 4251   match(ConI);
 4252 
 4253   op_cost(0);
 4254   format %{ %}
 4255   interface(CONST_INTER);
 4256 %}
 4257 
 4258 operand immI_255()
 4259 %{
 4260   predicate(n->get_int() == 255);
 4261   match(ConI);
 4262 
 4263   op_cost(0);
 4264   format %{ %}
 4265   interface(CONST_INTER);
 4266 %}
 4267 
 4268 operand immI_65535()
 4269 %{
 4270   predicate(n->get_int() == 65535);
 4271   match(ConI);
 4272 
 4273   op_cost(0);
 4274   format %{ %}
 4275   interface(CONST_INTER);
 4276 %}
 4277 
 4278 operand immI_positive()
 4279 %{
 4280   predicate(n->get_int() > 0);
 4281   match(ConI);
 4282 
 4283   op_cost(0);
 4284   format %{ %}
 4285   interface(CONST_INTER);
 4286 %}
 4287 
 4288 operand immL_255()
 4289 %{
 4290   predicate(n->get_long() == 255L);
 4291   match(ConL);
 4292 
 4293   op_cost(0);
 4294   format %{ %}
 4295   interface(CONST_INTER);
 4296 %}
 4297 
 4298 operand immL_65535()
 4299 %{
 4300   predicate(n->get_long() == 65535L);
 4301   match(ConL);
 4302 
 4303   op_cost(0);
 4304   format %{ %}
 4305   interface(CONST_INTER);
 4306 %}
 4307 
 4308 operand immL_4294967295()
 4309 %{
 4310   predicate(n->get_long() == 4294967295L);
 4311   match(ConL);
 4312 
 4313   op_cost(0);
 4314   format %{ %}
 4315   interface(CONST_INTER);
 4316 %}
 4317 
 4318 operand immL_bitmask()
 4319 %{
 4320   predicate((n->get_long() != 0)
 4321             && ((n->get_long() & 0xc000000000000000l) == 0)
 4322             && is_power_of_2(n->get_long() + 1));
 4323   match(ConL);
 4324 
 4325   op_cost(0);
 4326   format %{ %}
 4327   interface(CONST_INTER);
 4328 %}
 4329 
 4330 operand immI_bitmask()
 4331 %{
 4332   predicate((n->get_int() != 0)
 4333             && ((n->get_int() & 0xc0000000) == 0)
 4334             && is_power_of_2(n->get_int() + 1));
 4335   match(ConI);
 4336 
 4337   op_cost(0);
 4338   format %{ %}
 4339   interface(CONST_INTER);
 4340 %}
 4341 
 4342 operand immL_positive_bitmaskI()
 4343 %{
 4344   predicate((n->get_long() != 0)
 4345             && ((julong)n->get_long() < 0x80000000ULL)
 4346             && is_power_of_2(n->get_long() + 1));
 4347   match(ConL);
 4348 
 4349   op_cost(0);
 4350   format %{ %}
 4351   interface(CONST_INTER);
 4352 %}
 4353 
 4354 // Scale values for scaled offset addressing modes (up to long but not quad)
 4355 operand immIScale()
 4356 %{
 4357   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4358   match(ConI);
 4359 
 4360   op_cost(0);
 4361   format %{ %}
 4362   interface(CONST_INTER);
 4363 %}
 4364 
 4365 // 26 bit signed offset -- for pc-relative branches
 4366 operand immI26()
 4367 %{
 4368   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4369   match(ConI);
 4370 
 4371   op_cost(0);
 4372   format %{ %}
 4373   interface(CONST_INTER);
 4374 %}
 4375 
 4376 // 19 bit signed offset -- for pc-relative loads
 4377 operand immI19()
 4378 %{
 4379   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4380   match(ConI);
 4381 
 4382   op_cost(0);
 4383   format %{ %}
 4384   interface(CONST_INTER);
 4385 %}
 4386 
 4387 // 12 bit unsigned offset -- for base plus immediate loads
 4388 operand immIU12()
 4389 %{
 4390   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4391   match(ConI);
 4392 
 4393   op_cost(0);
 4394   format %{ %}
 4395   interface(CONST_INTER);
 4396 %}
 4397 
 4398 operand immLU12()
 4399 %{
 4400   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4401   match(ConL);
 4402 
 4403   op_cost(0);
 4404   format %{ %}
 4405   interface(CONST_INTER);
 4406 %}
 4407 
 4408 // Offset for scaled or unscaled immediate loads and stores
 4409 operand immIOffset()
 4410 %{
 4411   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4412   match(ConI);
 4413 
 4414   op_cost(0);
 4415   format %{ %}
 4416   interface(CONST_INTER);
 4417 %}
 4418 
 4419 operand immIOffset1()
 4420 %{
 4421   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4422   match(ConI);
 4423 
 4424   op_cost(0);
 4425   format %{ %}
 4426   interface(CONST_INTER);
 4427 %}
 4428 
 4429 operand immIOffset2()
 4430 %{
 4431   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4432   match(ConI);
 4433 
 4434   op_cost(0);
 4435   format %{ %}
 4436   interface(CONST_INTER);
 4437 %}
 4438 
 4439 operand immIOffset4()
 4440 %{
 4441   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4442   match(ConI);
 4443 
 4444   op_cost(0);
 4445   format %{ %}
 4446   interface(CONST_INTER);
 4447 %}
 4448 
 4449 operand immIOffset8()
 4450 %{
 4451   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4452   match(ConI);
 4453 
 4454   op_cost(0);
 4455   format %{ %}
 4456   interface(CONST_INTER);
 4457 %}
 4458 
 4459 operand immIOffset16()
 4460 %{
 4461   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4462   match(ConI);
 4463 
 4464   op_cost(0);
 4465   format %{ %}
 4466   interface(CONST_INTER);
 4467 %}
 4468 
 4469 operand immLoffset()
 4470 %{
 4471   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4472   match(ConL);
 4473 
 4474   op_cost(0);
 4475   format %{ %}
 4476   interface(CONST_INTER);
 4477 %}
 4478 
 4479 operand immLoffset1()
 4480 %{
 4481   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4482   match(ConL);
 4483 
 4484   op_cost(0);
 4485   format %{ %}
 4486   interface(CONST_INTER);
 4487 %}
 4488 
 4489 operand immLoffset2()
 4490 %{
 4491   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4492   match(ConL);
 4493 
 4494   op_cost(0);
 4495   format %{ %}
 4496   interface(CONST_INTER);
 4497 %}
 4498 
 4499 operand immLoffset4()
 4500 %{
 4501   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4502   match(ConL);
 4503 
 4504   op_cost(0);
 4505   format %{ %}
 4506   interface(CONST_INTER);
 4507 %}
 4508 
 4509 operand immLoffset8()
 4510 %{
 4511   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4512   match(ConL);
 4513 
 4514   op_cost(0);
 4515   format %{ %}
 4516   interface(CONST_INTER);
 4517 %}
 4518 
 4519 operand immLoffset16()
 4520 %{
 4521   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4522   match(ConL);
 4523 
 4524   op_cost(0);
 4525   format %{ %}
 4526   interface(CONST_INTER);
 4527 %}
 4528 
 4529 // 8 bit signed value.
 4530 operand immI8()
 4531 %{
 4532   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4533   match(ConI);
 4534 
 4535   op_cost(0);
 4536   format %{ %}
 4537   interface(CONST_INTER);
 4538 %}
 4539 
 4540 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4541 operand immI8_shift8()
 4542 %{
 4543   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4544             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4545   match(ConI);
 4546 
 4547   op_cost(0);
 4548   format %{ %}
 4549   interface(CONST_INTER);
 4550 %}
 4551 
 4552 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4553 operand immL8_shift8()
 4554 %{
 4555   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4556             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4557   match(ConL);
 4558 
 4559   op_cost(0);
 4560   format %{ %}
 4561   interface(CONST_INTER);
 4562 %}
 4563 
 4564 // 8 bit integer valid for vector add sub immediate
 4565 operand immBAddSubV()
 4566 %{
 4567   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4568   match(ConI);
 4569 
 4570   op_cost(0);
 4571   format %{ %}
 4572   interface(CONST_INTER);
 4573 %}
 4574 
 4575 // 32 bit integer valid for add sub immediate
 4576 operand immIAddSub()
 4577 %{
 4578   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4579   match(ConI);
 4580   op_cost(0);
 4581   format %{ %}
 4582   interface(CONST_INTER);
 4583 %}
 4584 
 4585 // 32 bit integer valid for vector add sub immediate
 4586 operand immIAddSubV()
 4587 %{
 4588   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4589   match(ConI);
 4590 
 4591   op_cost(0);
 4592   format %{ %}
 4593   interface(CONST_INTER);
 4594 %}
 4595 
 4596 // 32 bit unsigned integer valid for logical immediate
 4597 
 4598 operand immBLog()
 4599 %{
 4600   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4601   match(ConI);
 4602 
 4603   op_cost(0);
 4604   format %{ %}
 4605   interface(CONST_INTER);
 4606 %}
 4607 
 4608 operand immSLog()
 4609 %{
 4610   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4611   match(ConI);
 4612 
 4613   op_cost(0);
 4614   format %{ %}
 4615   interface(CONST_INTER);
 4616 %}
 4617 
 4618 operand immILog()
 4619 %{
 4620   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4621   match(ConI);
 4622 
 4623   op_cost(0);
 4624   format %{ %}
 4625   interface(CONST_INTER);
 4626 %}
 4627 
 4628 // Integer operands 64 bit
 4629 // 64 bit immediate
 4630 operand immL()
 4631 %{
 4632   match(ConL);
 4633 
 4634   op_cost(0);
 4635   format %{ %}
 4636   interface(CONST_INTER);
 4637 %}
 4638 
 4639 // 64 bit zero
 4640 operand immL0()
 4641 %{
 4642   predicate(n->get_long() == 0);
 4643   match(ConL);
 4644 
 4645   op_cost(0);
 4646   format %{ %}
 4647   interface(CONST_INTER);
 4648 %}
 4649 
 4650 // 64 bit unit increment
 4651 operand immL_1()
 4652 %{
 4653   predicate(n->get_long() == 1);
 4654   match(ConL);
 4655 
 4656   op_cost(0);
 4657   format %{ %}
 4658   interface(CONST_INTER);
 4659 %}
 4660 
 4661 // 64 bit unit decrement
 4662 operand immL_M1()
 4663 %{
 4664   predicate(n->get_long() == -1);
 4665   match(ConL);
 4666 
 4667   op_cost(0);
 4668   format %{ %}
 4669   interface(CONST_INTER);
 4670 %}
 4671 
 4672 // 32 bit offset of pc in thread anchor
 4673 
 4674 operand immL_pc_off()
 4675 %{
 4676   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4677                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4678   match(ConL);
 4679 
 4680   op_cost(0);
 4681   format %{ %}
 4682   interface(CONST_INTER);
 4683 %}
 4684 
 4685 // 64 bit integer valid for add sub immediate
 4686 operand immLAddSub()
 4687 %{
 4688   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4689   match(ConL);
 4690   op_cost(0);
 4691   format %{ %}
 4692   interface(CONST_INTER);
 4693 %}
 4694 
 4695 // 64 bit integer valid for addv subv immediate
 4696 operand immLAddSubV()
 4697 %{
 4698   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4699   match(ConL);
 4700 
 4701   op_cost(0);
 4702   format %{ %}
 4703   interface(CONST_INTER);
 4704 %}
 4705 
 4706 // 64 bit integer valid for logical immediate
 4707 operand immLLog()
 4708 %{
 4709   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4710   match(ConL);
 4711   op_cost(0);
 4712   format %{ %}
 4713   interface(CONST_INTER);
 4714 %}
 4715 
 4716 // Long Immediate: low 32-bit mask
 4717 operand immL_32bits()
 4718 %{
 4719   predicate(n->get_long() == 0xFFFFFFFFL);
 4720   match(ConL);
 4721   op_cost(0);
 4722   format %{ %}
 4723   interface(CONST_INTER);
 4724 %}
 4725 
 4726 // Pointer operands
 4727 // Pointer Immediate
 4728 operand immP()
 4729 %{
 4730   match(ConP);
 4731 
 4732   op_cost(0);
 4733   format %{ %}
 4734   interface(CONST_INTER);
 4735 %}
 4736 
 4737 // NULL Pointer Immediate
 4738 operand immP0()
 4739 %{
 4740   predicate(n->get_ptr() == 0);
 4741   match(ConP);
 4742 
 4743   op_cost(0);
 4744   format %{ %}
 4745   interface(CONST_INTER);
 4746 %}
 4747 
 4748 // Pointer Immediate One
 4749 // this is used in object initialization (initial object header)
 4750 operand immP_1()
 4751 %{
 4752   predicate(n->get_ptr() == 1);
 4753   match(ConP);
 4754 
 4755   op_cost(0);
 4756   format %{ %}
 4757   interface(CONST_INTER);
 4758 %}
 4759 
 4760 // Card Table Byte Map Base
 4761 operand immByteMapBase()
 4762 %{
 4763   // Get base of card map
 4764   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4765             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4766   match(ConP);
 4767 
 4768   op_cost(0);
 4769   format %{ %}
 4770   interface(CONST_INTER);
 4771 %}
 4772 
 4773 // Pointer Immediate Minus One
 4774 // this is used when we want to write the current PC to the thread anchor
 4775 operand immP_M1()
 4776 %{
 4777   predicate(n->get_ptr() == -1);
 4778   match(ConP);
 4779 
 4780   op_cost(0);
 4781   format %{ %}
 4782   interface(CONST_INTER);
 4783 %}
 4784 
 4785 // Pointer Immediate Minus Two
 4786 // this is used when we want to write the current PC to the thread anchor
 4787 operand immP_M2()
 4788 %{
 4789   predicate(n->get_ptr() == -2);
 4790   match(ConP);
 4791 
 4792   op_cost(0);
 4793   format %{ %}
 4794   interface(CONST_INTER);
 4795 %}
 4796 
 4797 // Float and Double operands
 4798 // Double Immediate
 4799 operand immD()
 4800 %{
 4801   match(ConD);
 4802   op_cost(0);
 4803   format %{ %}
 4804   interface(CONST_INTER);
 4805 %}
 4806 
 4807 // Double Immediate: +0.0d
 4808 operand immD0()
 4809 %{
 4810   predicate(jlong_cast(n->getd()) == 0);
 4811   match(ConD);
 4812 
 4813   op_cost(0);
 4814   format %{ %}
 4815   interface(CONST_INTER);
 4816 %}
 4817 
 4818 // constant 'double +0.0'.
 4819 operand immDPacked()
 4820 %{
 4821   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4822   match(ConD);
 4823   op_cost(0);
 4824   format %{ %}
 4825   interface(CONST_INTER);
 4826 %}
 4827 
 4828 // Float Immediate
 4829 operand immF()
 4830 %{
 4831   match(ConF);
 4832   op_cost(0);
 4833   format %{ %}
 4834   interface(CONST_INTER);
 4835 %}
 4836 
 4837 // Float Immediate: +0.0f.
 4838 operand immF0()
 4839 %{
 4840   predicate(jint_cast(n->getf()) == 0);
 4841   match(ConF);
 4842 
 4843   op_cost(0);
 4844   format %{ %}
 4845   interface(CONST_INTER);
 4846 %}
 4847 
 4848 //
 4849 operand immFPacked()
 4850 %{
 4851   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4852   match(ConF);
 4853   op_cost(0);
 4854   format %{ %}
 4855   interface(CONST_INTER);
 4856 %}
 4857 
 4858 // Narrow pointer operands
 4859 // Narrow Pointer Immediate
 4860 operand immN()
 4861 %{
 4862   match(ConN);
 4863 
 4864   op_cost(0);
 4865   format %{ %}
 4866   interface(CONST_INTER);
 4867 %}
 4868 
 4869 // Narrow NULL Pointer Immediate
 4870 operand immN0()
 4871 %{
 4872   predicate(n->get_narrowcon() == 0);
 4873   match(ConN);
 4874 
 4875   op_cost(0);
 4876   format %{ %}
 4877   interface(CONST_INTER);
 4878 %}
 4879 
 4880 operand immNKlass()
 4881 %{
 4882   match(ConNKlass);
 4883 
 4884   op_cost(0);
 4885   format %{ %}
 4886   interface(CONST_INTER);
 4887 %}
 4888 
 4889 // Integer 32 bit Register Operands
 4890 // Integer 32 bitRegister (excludes SP)
 4891 operand iRegI()
 4892 %{
 4893   constraint(ALLOC_IN_RC(any_reg32));
 4894   match(RegI);
 4895   match(iRegINoSp);
 4896   op_cost(0);
 4897   format %{ %}
 4898   interface(REG_INTER);
 4899 %}
 4900 
 4901 // Integer 32 bit Register not Special
 4902 operand iRegINoSp()
 4903 %{
 4904   constraint(ALLOC_IN_RC(no_special_reg32));
 4905   match(RegI);
 4906   op_cost(0);
 4907   format %{ %}
 4908   interface(REG_INTER);
 4909 %}
 4910 
 4911 // Integer 64 bit Register Operands
 4912 // Integer 64 bit Register (includes SP)
 4913 operand iRegL()
 4914 %{
 4915   constraint(ALLOC_IN_RC(any_reg));
 4916   match(RegL);
 4917   match(iRegLNoSp);
 4918   op_cost(0);
 4919   format %{ %}
 4920   interface(REG_INTER);
 4921 %}
 4922 
 4923 // Integer 64 bit Register not Special
 4924 operand iRegLNoSp()
 4925 %{
 4926   constraint(ALLOC_IN_RC(no_special_reg));
 4927   match(RegL);
 4928   match(iRegL_R0);
 4929   format %{ %}
 4930   interface(REG_INTER);
 4931 %}
 4932 
 4933 // Pointer Register Operands
 4934 // Pointer Register
 4935 operand iRegP()
 4936 %{
 4937   constraint(ALLOC_IN_RC(ptr_reg));
 4938   match(RegP);
 4939   match(iRegPNoSp);
 4940   match(iRegP_R0);
 4941   //match(iRegP_R2);
 4942   //match(iRegP_R4);
 4943   //match(iRegP_R5);
 4944   match(thread_RegP);
 4945   op_cost(0);
 4946   format %{ %}
 4947   interface(REG_INTER);
 4948 %}
 4949 
 4950 // Pointer 64 bit Register not Special
 4951 operand iRegPNoSp()
 4952 %{
 4953   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4954   match(RegP);
 4955   // match(iRegP);
 4956   // match(iRegP_R0);
 4957   // match(iRegP_R2);
 4958   // match(iRegP_R4);
 4959   // match(iRegP_R5);
 4960   // match(thread_RegP);
 4961   op_cost(0);
 4962   format %{ %}
 4963   interface(REG_INTER);
 4964 %}
 4965 
 4966 // Pointer 64 bit Register R0 only
 4967 operand iRegP_R0()
 4968 %{
 4969   constraint(ALLOC_IN_RC(r0_reg));
 4970   match(RegP);
 4971   // match(iRegP);
 4972   match(iRegPNoSp);
 4973   op_cost(0);
 4974   format %{ %}
 4975   interface(REG_INTER);
 4976 %}
 4977 
 4978 // Pointer 64 bit Register R1 only
 4979 operand iRegP_R1()
 4980 %{
 4981   constraint(ALLOC_IN_RC(r1_reg));
 4982   match(RegP);
 4983   // match(iRegP);
 4984   match(iRegPNoSp);
 4985   op_cost(0);
 4986   format %{ %}
 4987   interface(REG_INTER);
 4988 %}
 4989 
 4990 // Pointer 64 bit Register R2 only
 4991 operand iRegP_R2()
 4992 %{
 4993   constraint(ALLOC_IN_RC(r2_reg));
 4994   match(RegP);
 4995   // match(iRegP);
 4996   match(iRegPNoSp);
 4997   op_cost(0);
 4998   format %{ %}
 4999   interface(REG_INTER);
 5000 %}
 5001 
 5002 // Pointer 64 bit Register R3 only
 5003 operand iRegP_R3()
 5004 %{
 5005   constraint(ALLOC_IN_RC(r3_reg));
 5006   match(RegP);
 5007   // match(iRegP);
 5008   match(iRegPNoSp);
 5009   op_cost(0);
 5010   format %{ %}
 5011   interface(REG_INTER);
 5012 %}
 5013 
 5014 // Pointer 64 bit Register R4 only
 5015 operand iRegP_R4()
 5016 %{
 5017   constraint(ALLOC_IN_RC(r4_reg));
 5018   match(RegP);
 5019   // match(iRegP);
 5020   match(iRegPNoSp);
 5021   op_cost(0);
 5022   format %{ %}
 5023   interface(REG_INTER);
 5024 %}
 5025 
 5026 // Pointer 64 bit Register R5 only
 5027 operand iRegP_R5()
 5028 %{
 5029   constraint(ALLOC_IN_RC(r5_reg));
 5030   match(RegP);
 5031   // match(iRegP);
 5032   match(iRegPNoSp);
 5033   op_cost(0);
 5034   format %{ %}
 5035   interface(REG_INTER);
 5036 %}
 5037 
 5038 // Pointer 64 bit Register R10 only
 5039 operand iRegP_R10()
 5040 %{
 5041   constraint(ALLOC_IN_RC(r10_reg));
 5042   match(RegP);
 5043   // match(iRegP);
 5044   match(iRegPNoSp);
 5045   op_cost(0);
 5046   format %{ %}
 5047   interface(REG_INTER);
 5048 %}
 5049 
 5050 // Long 64 bit Register R0 only
 5051 operand iRegL_R0()
 5052 %{
 5053   constraint(ALLOC_IN_RC(r0_reg));
 5054   match(RegL);
 5055   match(iRegLNoSp);
 5056   op_cost(0);
 5057   format %{ %}
 5058   interface(REG_INTER);
 5059 %}
 5060 
 5061 // Long 64 bit Register R2 only
 5062 operand iRegL_R2()
 5063 %{
 5064   constraint(ALLOC_IN_RC(r2_reg));
 5065   match(RegL);
 5066   match(iRegLNoSp);
 5067   op_cost(0);
 5068   format %{ %}
 5069   interface(REG_INTER);
 5070 %}
 5071 
 5072 // Long 64 bit Register R3 only
 5073 operand iRegL_R3()
 5074 %{
 5075   constraint(ALLOC_IN_RC(r3_reg));
 5076   match(RegL);
 5077   match(iRegLNoSp);
 5078   op_cost(0);
 5079   format %{ %}
 5080   interface(REG_INTER);
 5081 %}
 5082 
 5083 // Long 64 bit Register R11 only
 5084 operand iRegL_R11()
 5085 %{
 5086   constraint(ALLOC_IN_RC(r11_reg));
 5087   match(RegL);
 5088   match(iRegLNoSp);
 5089   op_cost(0);
 5090   format %{ %}
 5091   interface(REG_INTER);
 5092 %}
 5093 
 5094 // Pointer 64 bit Register FP only
 5095 operand iRegP_FP()
 5096 %{
 5097   constraint(ALLOC_IN_RC(fp_reg));
 5098   match(RegP);
 5099   // match(iRegP);
 5100   op_cost(0);
 5101   format %{ %}
 5102   interface(REG_INTER);
 5103 %}
 5104 
 5105 // Register R0 only
 5106 operand iRegI_R0()
 5107 %{
 5108   constraint(ALLOC_IN_RC(int_r0_reg));
 5109   match(RegI);
 5110   match(iRegINoSp);
 5111   op_cost(0);
 5112   format %{ %}
 5113   interface(REG_INTER);
 5114 %}
 5115 
 5116 // Register R2 only
 5117 operand iRegI_R2()
 5118 %{
 5119   constraint(ALLOC_IN_RC(int_r2_reg));
 5120   match(RegI);
 5121   match(iRegINoSp);
 5122   op_cost(0);
 5123   format %{ %}
 5124   interface(REG_INTER);
 5125 %}
 5126 
 5127 // Register R3 only
 5128 operand iRegI_R3()
 5129 %{
 5130   constraint(ALLOC_IN_RC(int_r3_reg));
 5131   match(RegI);
 5132   match(iRegINoSp);
 5133   op_cost(0);
 5134   format %{ %}
 5135   interface(REG_INTER);
 5136 %}
 5137 
 5138 
 5139 // Register R4 only
 5140 operand iRegI_R4()
 5141 %{
 5142   constraint(ALLOC_IN_RC(int_r4_reg));
 5143   match(RegI);
 5144   match(iRegINoSp);
 5145   op_cost(0);
 5146   format %{ %}
 5147   interface(REG_INTER);
 5148 %}
 5149 
 5150 
 5151 // Pointer Register Operands
 5152 // Narrow Pointer Register
 5153 operand iRegN()
 5154 %{
 5155   constraint(ALLOC_IN_RC(any_reg32));
 5156   match(RegN);
 5157   match(iRegNNoSp);
 5158   op_cost(0);
 5159   format %{ %}
 5160   interface(REG_INTER);
 5161 %}
 5162 
 5163 operand iRegN_R0()
 5164 %{
 5165   constraint(ALLOC_IN_RC(r0_reg));
 5166   match(iRegN);
 5167   op_cost(0);
 5168   format %{ %}
 5169   interface(REG_INTER);
 5170 %}
 5171 
 5172 operand iRegN_R2()
 5173 %{
 5174   constraint(ALLOC_IN_RC(r2_reg));
 5175   match(iRegN);
 5176   op_cost(0);
 5177   format %{ %}
 5178   interface(REG_INTER);
 5179 %}
 5180 
 5181 operand iRegN_R3()
 5182 %{
 5183   constraint(ALLOC_IN_RC(r3_reg));
 5184   match(iRegN);
 5185   op_cost(0);
 5186   format %{ %}
 5187   interface(REG_INTER);
 5188 %}
 5189 
 5190 // Integer 64 bit Register not Special
 5191 operand iRegNNoSp()
 5192 %{
 5193   constraint(ALLOC_IN_RC(no_special_reg32));
 5194   match(RegN);
 5195   op_cost(0);
 5196   format %{ %}
 5197   interface(REG_INTER);
 5198 %}
 5199 
 5200 // heap base register -- used for encoding immN0
 5201 
 5202 operand iRegIHeapbase()
 5203 %{
 5204   constraint(ALLOC_IN_RC(heapbase_reg));
 5205   match(RegI);
 5206   op_cost(0);
 5207   format %{ %}
 5208   interface(REG_INTER);
 5209 %}
 5210 
 5211 // Float Register
 5212 // Float register operands
 5213 operand vRegF()
 5214 %{
 5215   constraint(ALLOC_IN_RC(float_reg));
 5216   match(RegF);
 5217 
 5218   op_cost(0);
 5219   format %{ %}
 5220   interface(REG_INTER);
 5221 %}
 5222 
 5223 // Double Register
 5224 // Double register operands
 5225 operand vRegD()
 5226 %{
 5227   constraint(ALLOC_IN_RC(double_reg));
 5228   match(RegD);
 5229 
 5230   op_cost(0);
 5231   format %{ %}
 5232   interface(REG_INTER);
 5233 %}
 5234 
 5235 // Generic vector class. This will be used for
 5236 // all vector operands, including NEON and SVE.
 5237 operand vReg()
 5238 %{
 5239   constraint(ALLOC_IN_RC(dynamic));
 5240   match(VecA);
 5241   match(VecD);
 5242   match(VecX);
 5243 
 5244   op_cost(0);
 5245   format %{ %}
 5246   interface(REG_INTER);
 5247 %}
 5248 
 5249 operand vecA()
 5250 %{
 5251   constraint(ALLOC_IN_RC(vectora_reg));
 5252   match(VecA);
 5253 
 5254   op_cost(0);
 5255   format %{ %}
 5256   interface(REG_INTER);
 5257 %}
 5258 
 5259 operand vecD()
 5260 %{
 5261   constraint(ALLOC_IN_RC(vectord_reg));
 5262   match(VecD);
 5263 
 5264   op_cost(0);
 5265   format %{ %}
 5266   interface(REG_INTER);
 5267 %}
 5268 
 5269 operand vecX()
 5270 %{
 5271   constraint(ALLOC_IN_RC(vectorx_reg));
 5272   match(VecX);
 5273 
 5274   op_cost(0);
 5275   format %{ %}
 5276   interface(REG_INTER);
 5277 %}
 5278 
 5279 operand vRegD_V0()
 5280 %{
 5281   constraint(ALLOC_IN_RC(v0_reg));
 5282   match(RegD);
 5283   op_cost(0);
 5284   format %{ %}
 5285   interface(REG_INTER);
 5286 %}
 5287 
 5288 operand vRegD_V1()
 5289 %{
 5290   constraint(ALLOC_IN_RC(v1_reg));
 5291   match(RegD);
 5292   op_cost(0);
 5293   format %{ %}
 5294   interface(REG_INTER);
 5295 %}
 5296 
 5297 operand vRegD_V2()
 5298 %{
 5299   constraint(ALLOC_IN_RC(v2_reg));
 5300   match(RegD);
 5301   op_cost(0);
 5302   format %{ %}
 5303   interface(REG_INTER);
 5304 %}
 5305 
 5306 operand vRegD_V3()
 5307 %{
 5308   constraint(ALLOC_IN_RC(v3_reg));
 5309   match(RegD);
 5310   op_cost(0);
 5311   format %{ %}
 5312   interface(REG_INTER);
 5313 %}
 5314 
 5315 operand vRegD_V4()
 5316 %{
 5317   constraint(ALLOC_IN_RC(v4_reg));
 5318   match(RegD);
 5319   op_cost(0);
 5320   format %{ %}
 5321   interface(REG_INTER);
 5322 %}
 5323 
 5324 operand vRegD_V5()
 5325 %{
 5326   constraint(ALLOC_IN_RC(v5_reg));
 5327   match(RegD);
 5328   op_cost(0);
 5329   format %{ %}
 5330   interface(REG_INTER);
 5331 %}
 5332 
 5333 operand vRegD_V6()
 5334 %{
 5335   constraint(ALLOC_IN_RC(v6_reg));
 5336   match(RegD);
 5337   op_cost(0);
 5338   format %{ %}
 5339   interface(REG_INTER);
 5340 %}
 5341 
 5342 operand vRegD_V7()
 5343 %{
 5344   constraint(ALLOC_IN_RC(v7_reg));
 5345   match(RegD);
 5346   op_cost(0);
 5347   format %{ %}
 5348   interface(REG_INTER);
 5349 %}
 5350 
 5351 operand vRegD_V8()
 5352 %{
 5353   constraint(ALLOC_IN_RC(v8_reg));
 5354   match(RegD);
 5355   op_cost(0);
 5356   format %{ %}
 5357   interface(REG_INTER);
 5358 %}
 5359 
 5360 operand vRegD_V9()
 5361 %{
 5362   constraint(ALLOC_IN_RC(v9_reg));
 5363   match(RegD);
 5364   op_cost(0);
 5365   format %{ %}
 5366   interface(REG_INTER);
 5367 %}
 5368 
 5369 operand vRegD_V10()
 5370 %{
 5371   constraint(ALLOC_IN_RC(v10_reg));
 5372   match(RegD);
 5373   op_cost(0);
 5374   format %{ %}
 5375   interface(REG_INTER);
 5376 %}
 5377 
 5378 operand vRegD_V11()
 5379 %{
 5380   constraint(ALLOC_IN_RC(v11_reg));
 5381   match(RegD);
 5382   op_cost(0);
 5383   format %{ %}
 5384   interface(REG_INTER);
 5385 %}
 5386 
 5387 operand vRegD_V12()
 5388 %{
 5389   constraint(ALLOC_IN_RC(v12_reg));
 5390   match(RegD);
 5391   op_cost(0);
 5392   format %{ %}
 5393   interface(REG_INTER);
 5394 %}
 5395 
 5396 operand vRegD_V13()
 5397 %{
 5398   constraint(ALLOC_IN_RC(v13_reg));
 5399   match(RegD);
 5400   op_cost(0);
 5401   format %{ %}
 5402   interface(REG_INTER);
 5403 %}
 5404 
 5405 operand vRegD_V14()
 5406 %{
 5407   constraint(ALLOC_IN_RC(v14_reg));
 5408   match(RegD);
 5409   op_cost(0);
 5410   format %{ %}
 5411   interface(REG_INTER);
 5412 %}
 5413 
 5414 operand vRegD_V15()
 5415 %{
 5416   constraint(ALLOC_IN_RC(v15_reg));
 5417   match(RegD);
 5418   op_cost(0);
 5419   format %{ %}
 5420   interface(REG_INTER);
 5421 %}
 5422 
 5423 operand vRegD_V16()
 5424 %{
 5425   constraint(ALLOC_IN_RC(v16_reg));
 5426   match(RegD);
 5427   op_cost(0);
 5428   format %{ %}
 5429   interface(REG_INTER);
 5430 %}
 5431 
 5432 operand vRegD_V17()
 5433 %{
 5434   constraint(ALLOC_IN_RC(v17_reg));
 5435   match(RegD);
 5436   op_cost(0);
 5437   format %{ %}
 5438   interface(REG_INTER);
 5439 %}
 5440 
 5441 operand vRegD_V18()
 5442 %{
 5443   constraint(ALLOC_IN_RC(v18_reg));
 5444   match(RegD);
 5445   op_cost(0);
 5446   format %{ %}
 5447   interface(REG_INTER);
 5448 %}
 5449 
 5450 operand vRegD_V19()
 5451 %{
 5452   constraint(ALLOC_IN_RC(v19_reg));
 5453   match(RegD);
 5454   op_cost(0);
 5455   format %{ %}
 5456   interface(REG_INTER);
 5457 %}
 5458 
 5459 operand vRegD_V20()
 5460 %{
 5461   constraint(ALLOC_IN_RC(v20_reg));
 5462   match(RegD);
 5463   op_cost(0);
 5464   format %{ %}
 5465   interface(REG_INTER);
 5466 %}
 5467 
 5468 operand vRegD_V21()
 5469 %{
 5470   constraint(ALLOC_IN_RC(v21_reg));
 5471   match(RegD);
 5472   op_cost(0);
 5473   format %{ %}
 5474   interface(REG_INTER);
 5475 %}
 5476 
 5477 operand vRegD_V22()
 5478 %{
 5479   constraint(ALLOC_IN_RC(v22_reg));
 5480   match(RegD);
 5481   op_cost(0);
 5482   format %{ %}
 5483   interface(REG_INTER);
 5484 %}
 5485 
 5486 operand vRegD_V23()
 5487 %{
 5488   constraint(ALLOC_IN_RC(v23_reg));
 5489   match(RegD);
 5490   op_cost(0);
 5491   format %{ %}
 5492   interface(REG_INTER);
 5493 %}
 5494 
 5495 operand vRegD_V24()
 5496 %{
 5497   constraint(ALLOC_IN_RC(v24_reg));
 5498   match(RegD);
 5499   op_cost(0);
 5500   format %{ %}
 5501   interface(REG_INTER);
 5502 %}
 5503 
 5504 operand vRegD_V25()
 5505 %{
 5506   constraint(ALLOC_IN_RC(v25_reg));
 5507   match(RegD);
 5508   op_cost(0);
 5509   format %{ %}
 5510   interface(REG_INTER);
 5511 %}
 5512 
 5513 operand vRegD_V26()
 5514 %{
 5515   constraint(ALLOC_IN_RC(v26_reg));
 5516   match(RegD);
 5517   op_cost(0);
 5518   format %{ %}
 5519   interface(REG_INTER);
 5520 %}
 5521 
 5522 operand vRegD_V27()
 5523 %{
 5524   constraint(ALLOC_IN_RC(v27_reg));
 5525   match(RegD);
 5526   op_cost(0);
 5527   format %{ %}
 5528   interface(REG_INTER);
 5529 %}
 5530 
 5531 operand vRegD_V28()
 5532 %{
 5533   constraint(ALLOC_IN_RC(v28_reg));
 5534   match(RegD);
 5535   op_cost(0);
 5536   format %{ %}
 5537   interface(REG_INTER);
 5538 %}
 5539 
 5540 operand vRegD_V29()
 5541 %{
 5542   constraint(ALLOC_IN_RC(v29_reg));
 5543   match(RegD);
 5544   op_cost(0);
 5545   format %{ %}
 5546   interface(REG_INTER);
 5547 %}
 5548 
 5549 operand vRegD_V30()
 5550 %{
 5551   constraint(ALLOC_IN_RC(v30_reg));
 5552   match(RegD);
 5553   op_cost(0);
 5554   format %{ %}
 5555   interface(REG_INTER);
 5556 %}
 5557 
 5558 operand vRegD_V31()
 5559 %{
 5560   constraint(ALLOC_IN_RC(v31_reg));
 5561   match(RegD);
 5562   op_cost(0);
 5563   format %{ %}
 5564   interface(REG_INTER);
 5565 %}
 5566 
 5567 operand pReg()
 5568 %{
 5569   constraint(ALLOC_IN_RC(pr_reg));
 5570   match(RegVectMask);
 5571   match(pRegGov);
 5572   op_cost(0);
 5573   format %{ %}
 5574   interface(REG_INTER);
 5575 %}
 5576 
 5577 operand pRegGov()
 5578 %{
 5579   constraint(ALLOC_IN_RC(gov_pr));
 5580   match(RegVectMask);
 5581   op_cost(0);
 5582   format %{ %}
 5583   interface(REG_INTER);
 5584 %}
 5585 
 5586 operand pRegGov_P0()
 5587 %{
 5588   constraint(ALLOC_IN_RC(p0_reg));
 5589   match(RegVectMask);
 5590   op_cost(0);
 5591   format %{ %}
 5592   interface(REG_INTER);
 5593 %}
 5594 
 5595 operand pRegGov_P1()
 5596 %{
 5597   constraint(ALLOC_IN_RC(p1_reg));
 5598   match(RegVectMask);
 5599   op_cost(0);
 5600   format %{ %}
 5601   interface(REG_INTER);
 5602 %}
 5603 
 5604 // Flags register, used as output of signed compare instructions
 5605 
 5606 // note that on AArch64 we also use this register as the output for
 5607 // for floating point compare instructions (CmpF CmpD). this ensures
 5608 // that ordered inequality tests use GT, GE, LT or LE none of which
 5609 // pass through cases where the result is unordered i.e. one or both
 5610 // inputs to the compare is a NaN. this means that the ideal code can
 5611 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5612 // (where the comparison should always fail). EQ and NE tests are
 5613 // always generated in ideal code so that unordered folds into the NE
 5614 // case, matching the behaviour of AArch64 NE.
 5615 //
 5616 // This differs from x86 where the outputs of FP compares use a
 5617 // special FP flags registers and where compares based on this
 5618 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5619 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5620 // to explicitly handle the unordered case in branches. x86 also has
 5621 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5622 
 5623 operand rFlagsReg()
 5624 %{
 5625   constraint(ALLOC_IN_RC(int_flags));
 5626   match(RegFlags);
 5627 
 5628   op_cost(0);
 5629   format %{ "RFLAGS" %}
 5630   interface(REG_INTER);
 5631 %}
 5632 
 5633 // Flags register, used as output of unsigned compare instructions
 5634 operand rFlagsRegU()
 5635 %{
 5636   constraint(ALLOC_IN_RC(int_flags));
 5637   match(RegFlags);
 5638 
 5639   op_cost(0);
 5640   format %{ "RFLAGSU" %}
 5641   interface(REG_INTER);
 5642 %}
 5643 
 5644 // Special Registers
 5645 
 5646 // Method Register
 5647 operand inline_cache_RegP(iRegP reg)
 5648 %{
 5649   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5650   match(reg);
 5651   match(iRegPNoSp);
 5652   op_cost(0);
 5653   format %{ %}
 5654   interface(REG_INTER);
 5655 %}
 5656 
 5657 // Thread Register
 5658 operand thread_RegP(iRegP reg)
 5659 %{
 5660   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5661   match(reg);
 5662   op_cost(0);
 5663   format %{ %}
 5664   interface(REG_INTER);
 5665 %}
 5666 
 5667 operand lr_RegP(iRegP reg)
 5668 %{
 5669   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5670   match(reg);
 5671   op_cost(0);
 5672   format %{ %}
 5673   interface(REG_INTER);
 5674 %}
 5675 
 5676 //----------Memory Operands----------------------------------------------------
 5677 
 5678 operand indirect(iRegP reg)
 5679 %{
 5680   constraint(ALLOC_IN_RC(ptr_reg));
 5681   match(reg);
 5682   op_cost(0);
 5683   format %{ "[$reg]" %}
 5684   interface(MEMORY_INTER) %{
 5685     base($reg);
 5686     index(0xffffffff);
 5687     scale(0x0);
 5688     disp(0x0);
 5689   %}
 5690 %}
 5691 
 5692 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5693 %{
 5694   constraint(ALLOC_IN_RC(ptr_reg));
 5695   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5696   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5697   op_cost(0);
 5698   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5699   interface(MEMORY_INTER) %{
 5700     base($reg);
 5701     index($ireg);
 5702     scale($scale);
 5703     disp(0x0);
 5704   %}
 5705 %}
 5706 
 5707 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5708 %{
 5709   constraint(ALLOC_IN_RC(ptr_reg));
 5710   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5711   match(AddP reg (LShiftL lreg scale));
 5712   op_cost(0);
 5713   format %{ "$reg, $lreg lsl($scale)" %}
 5714   interface(MEMORY_INTER) %{
 5715     base($reg);
 5716     index($lreg);
 5717     scale($scale);
 5718     disp(0x0);
 5719   %}
 5720 %}
 5721 
 5722 operand indIndexI2L(iRegP reg, iRegI ireg)
 5723 %{
 5724   constraint(ALLOC_IN_RC(ptr_reg));
 5725   match(AddP reg (ConvI2L ireg));
 5726   op_cost(0);
 5727   format %{ "$reg, $ireg, 0, I2L" %}
 5728   interface(MEMORY_INTER) %{
 5729     base($reg);
 5730     index($ireg);
 5731     scale(0x0);
 5732     disp(0x0);
 5733   %}
 5734 %}
 5735 
 5736 operand indIndex(iRegP reg, iRegL lreg)
 5737 %{
 5738   constraint(ALLOC_IN_RC(ptr_reg));
 5739   match(AddP reg lreg);
 5740   op_cost(0);
 5741   format %{ "$reg, $lreg" %}
 5742   interface(MEMORY_INTER) %{
 5743     base($reg);
 5744     index($lreg);
 5745     scale(0x0);
 5746     disp(0x0);
 5747   %}
 5748 %}
 5749 
 5750 operand indOffI(iRegP reg, immIOffset off)
 5751 %{
 5752   constraint(ALLOC_IN_RC(ptr_reg));
 5753   match(AddP reg off);
 5754   op_cost(0);
 5755   format %{ "[$reg, $off]" %}
 5756   interface(MEMORY_INTER) %{
 5757     base($reg);
 5758     index(0xffffffff);
 5759     scale(0x0);
 5760     disp($off);
 5761   %}
 5762 %}
 5763 
 5764 operand indOffI1(iRegP reg, immIOffset1 off)
 5765 %{
 5766   constraint(ALLOC_IN_RC(ptr_reg));
 5767   match(AddP reg off);
 5768   op_cost(0);
 5769   format %{ "[$reg, $off]" %}
 5770   interface(MEMORY_INTER) %{
 5771     base($reg);
 5772     index(0xffffffff);
 5773     scale(0x0);
 5774     disp($off);
 5775   %}
 5776 %}
 5777 
 5778 operand indOffI2(iRegP reg, immIOffset2 off)
 5779 %{
 5780   constraint(ALLOC_IN_RC(ptr_reg));
 5781   match(AddP reg off);
 5782   op_cost(0);
 5783   format %{ "[$reg, $off]" %}
 5784   interface(MEMORY_INTER) %{
 5785     base($reg);
 5786     index(0xffffffff);
 5787     scale(0x0);
 5788     disp($off);
 5789   %}
 5790 %}
 5791 
 5792 operand indOffI4(iRegP reg, immIOffset4 off)
 5793 %{
 5794   constraint(ALLOC_IN_RC(ptr_reg));
 5795   match(AddP reg off);
 5796   op_cost(0);
 5797   format %{ "[$reg, $off]" %}
 5798   interface(MEMORY_INTER) %{
 5799     base($reg);
 5800     index(0xffffffff);
 5801     scale(0x0);
 5802     disp($off);
 5803   %}
 5804 %}
 5805 
 5806 operand indOffI8(iRegP reg, immIOffset8 off)
 5807 %{
 5808   constraint(ALLOC_IN_RC(ptr_reg));
 5809   match(AddP reg off);
 5810   op_cost(0);
 5811   format %{ "[$reg, $off]" %}
 5812   interface(MEMORY_INTER) %{
 5813     base($reg);
 5814     index(0xffffffff);
 5815     scale(0x0);
 5816     disp($off);
 5817   %}
 5818 %}
 5819 
 5820 operand indOffI16(iRegP reg, immIOffset16 off)
 5821 %{
 5822   constraint(ALLOC_IN_RC(ptr_reg));
 5823   match(AddP reg off);
 5824   op_cost(0);
 5825   format %{ "[$reg, $off]" %}
 5826   interface(MEMORY_INTER) %{
 5827     base($reg);
 5828     index(0xffffffff);
 5829     scale(0x0);
 5830     disp($off);
 5831   %}
 5832 %}
 5833 
 5834 operand indOffL(iRegP reg, immLoffset off)
 5835 %{
 5836   constraint(ALLOC_IN_RC(ptr_reg));
 5837   match(AddP reg off);
 5838   op_cost(0);
 5839   format %{ "[$reg, $off]" %}
 5840   interface(MEMORY_INTER) %{
 5841     base($reg);
 5842     index(0xffffffff);
 5843     scale(0x0);
 5844     disp($off);
 5845   %}
 5846 %}
 5847 
 5848 operand indOffL1(iRegP reg, immLoffset1 off)
 5849 %{
 5850   constraint(ALLOC_IN_RC(ptr_reg));
 5851   match(AddP reg off);
 5852   op_cost(0);
 5853   format %{ "[$reg, $off]" %}
 5854   interface(MEMORY_INTER) %{
 5855     base($reg);
 5856     index(0xffffffff);
 5857     scale(0x0);
 5858     disp($off);
 5859   %}
 5860 %}
 5861 
 5862 operand indOffL2(iRegP reg, immLoffset2 off)
 5863 %{
 5864   constraint(ALLOC_IN_RC(ptr_reg));
 5865   match(AddP reg off);
 5866   op_cost(0);
 5867   format %{ "[$reg, $off]" %}
 5868   interface(MEMORY_INTER) %{
 5869     base($reg);
 5870     index(0xffffffff);
 5871     scale(0x0);
 5872     disp($off);
 5873   %}
 5874 %}
 5875 
 5876 operand indOffL4(iRegP reg, immLoffset4 off)
 5877 %{
 5878   constraint(ALLOC_IN_RC(ptr_reg));
 5879   match(AddP reg off);
 5880   op_cost(0);
 5881   format %{ "[$reg, $off]" %}
 5882   interface(MEMORY_INTER) %{
 5883     base($reg);
 5884     index(0xffffffff);
 5885     scale(0x0);
 5886     disp($off);
 5887   %}
 5888 %}
 5889 
 5890 operand indOffL8(iRegP reg, immLoffset8 off)
 5891 %{
 5892   constraint(ALLOC_IN_RC(ptr_reg));
 5893   match(AddP reg off);
 5894   op_cost(0);
 5895   format %{ "[$reg, $off]" %}
 5896   interface(MEMORY_INTER) %{
 5897     base($reg);
 5898     index(0xffffffff);
 5899     scale(0x0);
 5900     disp($off);
 5901   %}
 5902 %}
 5903 
 5904 operand indOffL16(iRegP reg, immLoffset16 off)
 5905 %{
 5906   constraint(ALLOC_IN_RC(ptr_reg));
 5907   match(AddP reg off);
 5908   op_cost(0);
 5909   format %{ "[$reg, $off]" %}
 5910   interface(MEMORY_INTER) %{
 5911     base($reg);
 5912     index(0xffffffff);
 5913     scale(0x0);
 5914     disp($off);
 5915   %}
 5916 %}
 5917 
 5918 operand indirectN(iRegN reg)
 5919 %{
 5920   predicate(CompressedOops::shift() == 0);
 5921   constraint(ALLOC_IN_RC(ptr_reg));
 5922   match(DecodeN reg);
 5923   op_cost(0);
 5924   format %{ "[$reg]\t# narrow" %}
 5925   interface(MEMORY_INTER) %{
 5926     base($reg);
 5927     index(0xffffffff);
 5928     scale(0x0);
 5929     disp(0x0);
 5930   %}
 5931 %}
 5932 
 5933 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5934 %{
 5935   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5936   constraint(ALLOC_IN_RC(ptr_reg));
 5937   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5938   op_cost(0);
 5939   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5940   interface(MEMORY_INTER) %{
 5941     base($reg);
 5942     index($ireg);
 5943     scale($scale);
 5944     disp(0x0);
 5945   %}
 5946 %}
 5947 
 5948 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5949 %{
 5950   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5951   constraint(ALLOC_IN_RC(ptr_reg));
 5952   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5953   op_cost(0);
 5954   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5955   interface(MEMORY_INTER) %{
 5956     base($reg);
 5957     index($lreg);
 5958     scale($scale);
 5959     disp(0x0);
 5960   %}
 5961 %}
 5962 
 5963 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5964 %{
 5965   predicate(CompressedOops::shift() == 0);
 5966   constraint(ALLOC_IN_RC(ptr_reg));
 5967   match(AddP (DecodeN reg) (ConvI2L ireg));
 5968   op_cost(0);
 5969   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5970   interface(MEMORY_INTER) %{
 5971     base($reg);
 5972     index($ireg);
 5973     scale(0x0);
 5974     disp(0x0);
 5975   %}
 5976 %}
 5977 
 5978 operand indIndexN(iRegN reg, iRegL lreg)
 5979 %{
 5980   predicate(CompressedOops::shift() == 0);
 5981   constraint(ALLOC_IN_RC(ptr_reg));
 5982   match(AddP (DecodeN reg) lreg);
 5983   op_cost(0);
 5984   format %{ "$reg, $lreg\t# narrow" %}
 5985   interface(MEMORY_INTER) %{
 5986     base($reg);
 5987     index($lreg);
 5988     scale(0x0);
 5989     disp(0x0);
 5990   %}
 5991 %}
 5992 
 5993 operand indOffIN(iRegN reg, immIOffset off)
 5994 %{
 5995   predicate(CompressedOops::shift() == 0);
 5996   constraint(ALLOC_IN_RC(ptr_reg));
 5997   match(AddP (DecodeN reg) off);
 5998   op_cost(0);
 5999   format %{ "[$reg, $off]\t# narrow" %}
 6000   interface(MEMORY_INTER) %{
 6001     base($reg);
 6002     index(0xffffffff);
 6003     scale(0x0);
 6004     disp($off);
 6005   %}
 6006 %}
 6007 
 6008 operand indOffLN(iRegN reg, immLoffset off)
 6009 %{
 6010   predicate(CompressedOops::shift() == 0);
 6011   constraint(ALLOC_IN_RC(ptr_reg));
 6012   match(AddP (DecodeN reg) off);
 6013   op_cost(0);
 6014   format %{ "[$reg, $off]\t# narrow" %}
 6015   interface(MEMORY_INTER) %{
 6016     base($reg);
 6017     index(0xffffffff);
 6018     scale(0x0);
 6019     disp($off);
 6020   %}
 6021 %}
 6022 
 6023 
 6024 
 6025 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 6026 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 6027 %{
 6028   constraint(ALLOC_IN_RC(ptr_reg));
 6029   match(AddP reg off);
 6030   op_cost(0);
 6031   format %{ "[$reg, $off]" %}
 6032   interface(MEMORY_INTER) %{
 6033     base($reg);
 6034     index(0xffffffff);
 6035     scale(0x0);
 6036     disp($off);
 6037   %}
 6038 %}
 6039 
 6040 //----------Special Memory Operands--------------------------------------------
 6041 // Stack Slot Operand - This operand is used for loading and storing temporary
 6042 //                      values on the stack where a match requires a value to
 6043 //                      flow through memory.
 6044 operand stackSlotP(sRegP reg)
 6045 %{
 6046   constraint(ALLOC_IN_RC(stack_slots));
 6047   op_cost(100);
 6048   // No match rule because this operand is only generated in matching
 6049   // match(RegP);
 6050   format %{ "[$reg]" %}
 6051   interface(MEMORY_INTER) %{
 6052     base(0x1e);  // RSP
 6053     index(0x0);  // No Index
 6054     scale(0x0);  // No Scale
 6055     disp($reg);  // Stack Offset
 6056   %}
 6057 %}
 6058 
 6059 operand stackSlotI(sRegI reg)
 6060 %{
 6061   constraint(ALLOC_IN_RC(stack_slots));
 6062   // No match rule because this operand is only generated in matching
 6063   // match(RegI);
 6064   format %{ "[$reg]" %}
 6065   interface(MEMORY_INTER) %{
 6066     base(0x1e);  // RSP
 6067     index(0x0);  // No Index
 6068     scale(0x0);  // No Scale
 6069     disp($reg);  // Stack Offset
 6070   %}
 6071 %}
 6072 
 6073 operand stackSlotF(sRegF reg)
 6074 %{
 6075   constraint(ALLOC_IN_RC(stack_slots));
 6076   // No match rule because this operand is only generated in matching
 6077   // match(RegF);
 6078   format %{ "[$reg]" %}
 6079   interface(MEMORY_INTER) %{
 6080     base(0x1e);  // RSP
 6081     index(0x0);  // No Index
 6082     scale(0x0);  // No Scale
 6083     disp($reg);  // Stack Offset
 6084   %}
 6085 %}
 6086 
 6087 operand stackSlotD(sRegD reg)
 6088 %{
 6089   constraint(ALLOC_IN_RC(stack_slots));
 6090   // No match rule because this operand is only generated in matching
 6091   // match(RegD);
 6092   format %{ "[$reg]" %}
 6093   interface(MEMORY_INTER) %{
 6094     base(0x1e);  // RSP
 6095     index(0x0);  // No Index
 6096     scale(0x0);  // No Scale
 6097     disp($reg);  // Stack Offset
 6098   %}
 6099 %}
 6100 
 6101 operand stackSlotL(sRegL reg)
 6102 %{
 6103   constraint(ALLOC_IN_RC(stack_slots));
 6104   // No match rule because this operand is only generated in matching
 6105   // match(RegL);
 6106   format %{ "[$reg]" %}
 6107   interface(MEMORY_INTER) %{
 6108     base(0x1e);  // RSP
 6109     index(0x0);  // No Index
 6110     scale(0x0);  // No Scale
 6111     disp($reg);  // Stack Offset
 6112   %}
 6113 %}
 6114 
 6115 // Operands for expressing Control Flow
 6116 // NOTE: Label is a predefined operand which should not be redefined in
 6117 //       the AD file. It is generically handled within the ADLC.
 6118 
 6119 //----------Conditional Branch Operands----------------------------------------
 6120 // Comparison Op  - This is the operation of the comparison, and is limited to
 6121 //                  the following set of codes:
 6122 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6123 //
 6124 // Other attributes of the comparison, such as unsignedness, are specified
 6125 // by the comparison instruction that sets a condition code flags register.
 6126 // That result is represented by a flags operand whose subtype is appropriate
 6127 // to the unsignedness (etc.) of the comparison.
 6128 //
 6129 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6130 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6131 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6132 
 6133 // used for signed integral comparisons and fp comparisons
 6134 
 6135 operand cmpOp()
 6136 %{
 6137   match(Bool);
 6138 
 6139   format %{ "" %}
 6140   interface(COND_INTER) %{
 6141     equal(0x0, "eq");
 6142     not_equal(0x1, "ne");
 6143     less(0xb, "lt");
 6144     greater_equal(0xa, "ge");
 6145     less_equal(0xd, "le");
 6146     greater(0xc, "gt");
 6147     overflow(0x6, "vs");
 6148     no_overflow(0x7, "vc");
 6149   %}
 6150 %}
 6151 
 6152 // used for unsigned integral comparisons
 6153 
 6154 operand cmpOpU()
 6155 %{
 6156   match(Bool);
 6157 
 6158   format %{ "" %}
 6159   interface(COND_INTER) %{
 6160     equal(0x0, "eq");
 6161     not_equal(0x1, "ne");
 6162     less(0x3, "lo");
 6163     greater_equal(0x2, "hs");
 6164     less_equal(0x9, "ls");
 6165     greater(0x8, "hi");
 6166     overflow(0x6, "vs");
 6167     no_overflow(0x7, "vc");
 6168   %}
 6169 %}
 6170 
 6171 // used for certain integral comparisons which can be
 6172 // converted to cbxx or tbxx instructions
 6173 
 6174 operand cmpOpEqNe()
 6175 %{
 6176   match(Bool);
 6177   op_cost(0);
 6178   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6179             || n->as_Bool()->_test._test == BoolTest::eq);
 6180 
 6181   format %{ "" %}
 6182   interface(COND_INTER) %{
 6183     equal(0x0, "eq");
 6184     not_equal(0x1, "ne");
 6185     less(0xb, "lt");
 6186     greater_equal(0xa, "ge");
 6187     less_equal(0xd, "le");
 6188     greater(0xc, "gt");
 6189     overflow(0x6, "vs");
 6190     no_overflow(0x7, "vc");
 6191   %}
 6192 %}
 6193 
 6194 // used for certain integral comparisons which can be
 6195 // converted to cbxx or tbxx instructions
 6196 
 6197 operand cmpOpLtGe()
 6198 %{
 6199   match(Bool);
 6200   op_cost(0);
 6201 
 6202   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6203             || n->as_Bool()->_test._test == BoolTest::ge);
 6204 
 6205   format %{ "" %}
 6206   interface(COND_INTER) %{
 6207     equal(0x0, "eq");
 6208     not_equal(0x1, "ne");
 6209     less(0xb, "lt");
 6210     greater_equal(0xa, "ge");
 6211     less_equal(0xd, "le");
 6212     greater(0xc, "gt");
 6213     overflow(0x6, "vs");
 6214     no_overflow(0x7, "vc");
 6215   %}
 6216 %}
 6217 
 6218 // used for certain unsigned integral comparisons which can be
 6219 // converted to cbxx or tbxx instructions
 6220 
 6221 operand cmpOpUEqNeLtGe()
 6222 %{
 6223   match(Bool);
 6224   op_cost(0);
 6225 
 6226   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6227             || n->as_Bool()->_test._test == BoolTest::ne
 6228             || n->as_Bool()->_test._test == BoolTest::lt
 6229             || n->as_Bool()->_test._test == BoolTest::ge);
 6230 
 6231   format %{ "" %}
 6232   interface(COND_INTER) %{
 6233     equal(0x0, "eq");
 6234     not_equal(0x1, "ne");
 6235     less(0xb, "lt");
 6236     greater_equal(0xa, "ge");
 6237     less_equal(0xd, "le");
 6238     greater(0xc, "gt");
 6239     overflow(0x6, "vs");
 6240     no_overflow(0x7, "vc");
 6241   %}
 6242 %}
 6243 
 6244 // Special operand allowing long args to int ops to be truncated for free
 6245 
 6246 operand iRegL2I(iRegL reg) %{
 6247 
 6248   op_cost(0);
 6249 
 6250   match(ConvL2I reg);
 6251 
 6252   format %{ "l2i($reg)" %}
 6253 
 6254   interface(REG_INTER)
 6255 %}
 6256 
 6257 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6258 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6259 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6260 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6261 
 6262 //----------OPERAND CLASSES----------------------------------------------------
 6263 // Operand Classes are groups of operands that are used as to simplify
 6264 // instruction definitions by not requiring the AD writer to specify
 6265 // separate instructions for every form of operand when the
 6266 // instruction accepts multiple operand types with the same basic
 6267 // encoding and format. The classic case of this is memory operands.
 6268 
 6269 // memory is used to define read/write location for load/store
 6270 // instruction defs. we can turn a memory op into an Address
 6271 
 6272 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6273                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6274 
 6275 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6276                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6277 
 6278 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6279                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6280 
 6281 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6282                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6283 
 6284 // All of the memory operands. For the pipeline description.
 6285 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6286                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6287                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6288 
 6289 
 6290 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6291 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6292 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6293 // can be elided because the 32-bit instruction will just employ the
 6294 // lower 32 bits anyway.
 6295 //
 6296 // n.b. this does not elide all L2I conversions. if the truncated
 6297 // value is consumed by more than one operation then the ConvL2I
 6298 // cannot be bundled into the consuming nodes so an l2i gets planted
 6299 // (actually a movw $dst $src) and the downstream instructions consume
 6300 // the result of the l2i as an iRegI input. That's a shame since the
 6301 // movw is actually redundant but its not too costly.
 6302 
 6303 opclass iRegIorL2I(iRegI, iRegL2I);
 6304 
 6305 //----------PIPELINE-----------------------------------------------------------
 6306 // Rules which define the behavior of the target architectures pipeline.
 6307 
 6308 // For specific pipelines, eg A53, define the stages of that pipeline
 6309 //pipe_desc(ISS, EX1, EX2, WR);
 6310 #define ISS S0
 6311 #define EX1 S1
 6312 #define EX2 S2
 6313 #define WR  S3
 6314 
 6315 // Integer ALU reg operation
 6316 pipeline %{
 6317 
 6318 attributes %{
 6319   // ARM instructions are of fixed length
 6320   fixed_size_instructions;        // Fixed size instructions TODO does
 6321   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6322   // ARM instructions come in 32-bit word units
 6323   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6324   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6325   instruction_fetch_units = 1;       // of 64 bytes
 6326 
 6327   // List of nop instructions
 6328   nops( MachNop );
 6329 %}
 6330 
 6331 // We don't use an actual pipeline model so don't care about resources
 6332 // or description. we do use pipeline classes to introduce fixed
 6333 // latencies
 6334 
 6335 //----------RESOURCES----------------------------------------------------------
 6336 // Resources are the functional units available to the machine
 6337 
 6338 resources( INS0, INS1, INS01 = INS0 | INS1,
 6339            ALU0, ALU1, ALU = ALU0 | ALU1,
 6340            MAC,
 6341            DIV,
 6342            BRANCH,
 6343            LDST,
 6344            NEON_FP);
 6345 
 6346 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6347 // Pipeline Description specifies the stages in the machine's pipeline
 6348 
 6349 // Define the pipeline as a generic 6 stage pipeline
 6350 pipe_desc(S0, S1, S2, S3, S4, S5);
 6351 
 6352 //----------PIPELINE CLASSES---------------------------------------------------
 6353 // Pipeline Classes describe the stages in which input and output are
 6354 // referenced by the hardware pipeline.
 6355 
 6356 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6357 %{
 6358   single_instruction;
 6359   src1   : S1(read);
 6360   src2   : S2(read);
 6361   dst    : S5(write);
 6362   INS01  : ISS;
 6363   NEON_FP : S5;
 6364 %}
 6365 
 6366 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6367 %{
 6368   single_instruction;
 6369   src1   : S1(read);
 6370   src2   : S2(read);
 6371   dst    : S5(write);
 6372   INS01  : ISS;
 6373   NEON_FP : S5;
 6374 %}
 6375 
 6376 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6377 %{
 6378   single_instruction;
 6379   src    : S1(read);
 6380   dst    : S5(write);
 6381   INS01  : ISS;
 6382   NEON_FP : S5;
 6383 %}
 6384 
 6385 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6386 %{
 6387   single_instruction;
 6388   src    : S1(read);
 6389   dst    : S5(write);
 6390   INS01  : ISS;
 6391   NEON_FP : S5;
 6392 %}
 6393 
 6394 pipe_class fp_d2f(vRegF dst, vRegD src)
 6395 %{
 6396   single_instruction;
 6397   src    : S1(read);
 6398   dst    : S5(write);
 6399   INS01  : ISS;
 6400   NEON_FP : S5;
 6401 %}
 6402 
 6403 pipe_class fp_f2d(vRegD dst, vRegF src)
 6404 %{
 6405   single_instruction;
 6406   src    : S1(read);
 6407   dst    : S5(write);
 6408   INS01  : ISS;
 6409   NEON_FP : S5;
 6410 %}
 6411 
 6412 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6413 %{
 6414   single_instruction;
 6415   src    : S1(read);
 6416   dst    : S5(write);
 6417   INS01  : ISS;
 6418   NEON_FP : S5;
 6419 %}
 6420 
 6421 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6422 %{
 6423   single_instruction;
 6424   src    : S1(read);
 6425   dst    : S5(write);
 6426   INS01  : ISS;
 6427   NEON_FP : S5;
 6428 %}
 6429 
 6430 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6431 %{
 6432   single_instruction;
 6433   src    : S1(read);
 6434   dst    : S5(write);
 6435   INS01  : ISS;
 6436   NEON_FP : S5;
 6437 %}
 6438 
 6439 pipe_class fp_l2f(vRegF dst, iRegL src)
 6440 %{
 6441   single_instruction;
 6442   src    : S1(read);
 6443   dst    : S5(write);
 6444   INS01  : ISS;
 6445   NEON_FP : S5;
 6446 %}
 6447 
 6448 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6449 %{
 6450   single_instruction;
 6451   src    : S1(read);
 6452   dst    : S5(write);
 6453   INS01  : ISS;
 6454   NEON_FP : S5;
 6455 %}
 6456 
 6457 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6458 %{
 6459   single_instruction;
 6460   src    : S1(read);
 6461   dst    : S5(write);
 6462   INS01  : ISS;
 6463   NEON_FP : S5;
 6464 %}
 6465 
 6466 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6467 %{
 6468   single_instruction;
 6469   src    : S1(read);
 6470   dst    : S5(write);
 6471   INS01  : ISS;
 6472   NEON_FP : S5;
 6473 %}
 6474 
 6475 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6476 %{
 6477   single_instruction;
 6478   src    : S1(read);
 6479   dst    : S5(write);
 6480   INS01  : ISS;
 6481   NEON_FP : S5;
 6482 %}
 6483 
 6484 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6485 %{
 6486   single_instruction;
 6487   src1   : S1(read);
 6488   src2   : S2(read);
 6489   dst    : S5(write);
 6490   INS0   : ISS;
 6491   NEON_FP : S5;
 6492 %}
 6493 
 6494 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6495 %{
 6496   single_instruction;
 6497   src1   : S1(read);
 6498   src2   : S2(read);
 6499   dst    : S5(write);
 6500   INS0   : ISS;
 6501   NEON_FP : S5;
 6502 %}
 6503 
 6504 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6505 %{
 6506   single_instruction;
 6507   cr     : S1(read);
 6508   src1   : S1(read);
 6509   src2   : S1(read);
 6510   dst    : S3(write);
 6511   INS01  : ISS;
 6512   NEON_FP : S3;
 6513 %}
 6514 
 6515 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6516 %{
 6517   single_instruction;
 6518   cr     : S1(read);
 6519   src1   : S1(read);
 6520   src2   : S1(read);
 6521   dst    : S3(write);
 6522   INS01  : ISS;
 6523   NEON_FP : S3;
 6524 %}
 6525 
 6526 pipe_class fp_imm_s(vRegF dst)
 6527 %{
 6528   single_instruction;
 6529   dst    : S3(write);
 6530   INS01  : ISS;
 6531   NEON_FP : S3;
 6532 %}
 6533 
 6534 pipe_class fp_imm_d(vRegD dst)
 6535 %{
 6536   single_instruction;
 6537   dst    : S3(write);
 6538   INS01  : ISS;
 6539   NEON_FP : S3;
 6540 %}
 6541 
 6542 pipe_class fp_load_constant_s(vRegF dst)
 6543 %{
 6544   single_instruction;
 6545   dst    : S4(write);
 6546   INS01  : ISS;
 6547   NEON_FP : S4;
 6548 %}
 6549 
 6550 pipe_class fp_load_constant_d(vRegD dst)
 6551 %{
 6552   single_instruction;
 6553   dst    : S4(write);
 6554   INS01  : ISS;
 6555   NEON_FP : S4;
 6556 %}
 6557 
 6558 //------- Integer ALU operations --------------------------
 6559 
 6560 // Integer ALU reg-reg operation
 6561 // Operands needed in EX1, result generated in EX2
 6562 // Eg.  ADD     x0, x1, x2
 6563 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6564 %{
 6565   single_instruction;
 6566   dst    : EX2(write);
 6567   src1   : EX1(read);
 6568   src2   : EX1(read);
 6569   INS01  : ISS; // Dual issue as instruction 0 or 1
 6570   ALU    : EX2;
 6571 %}
 6572 
 6573 // Integer ALU reg-reg operation with constant shift
 6574 // Shifted register must be available in LATE_ISS instead of EX1
 6575 // Eg.  ADD     x0, x1, x2, LSL #2
 6576 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6577 %{
 6578   single_instruction;
 6579   dst    : EX2(write);
 6580   src1   : EX1(read);
 6581   src2   : ISS(read);
 6582   INS01  : ISS;
 6583   ALU    : EX2;
 6584 %}
 6585 
 6586 // Integer ALU reg operation with constant shift
 6587 // Eg.  LSL     x0, x1, #shift
 6588 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6589 %{
 6590   single_instruction;
 6591   dst    : EX2(write);
 6592   src1   : ISS(read);
 6593   INS01  : ISS;
 6594   ALU    : EX2;
 6595 %}
 6596 
 6597 // Integer ALU reg-reg operation with variable shift
 6598 // Both operands must be available in LATE_ISS instead of EX1
 6599 // Result is available in EX1 instead of EX2
 6600 // Eg.  LSLV    x0, x1, x2
 6601 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6602 %{
 6603   single_instruction;
 6604   dst    : EX1(write);
 6605   src1   : ISS(read);
 6606   src2   : ISS(read);
 6607   INS01  : ISS;
 6608   ALU    : EX1;
 6609 %}
 6610 
 6611 // Integer ALU reg-reg operation with extract
 6612 // As for _vshift above, but result generated in EX2
 6613 // Eg.  EXTR    x0, x1, x2, #N
 6614 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6615 %{
 6616   single_instruction;
 6617   dst    : EX2(write);
 6618   src1   : ISS(read);
 6619   src2   : ISS(read);
 6620   INS1   : ISS; // Can only dual issue as Instruction 1
 6621   ALU    : EX1;
 6622 %}
 6623 
 6624 // Integer ALU reg operation
 6625 // Eg.  NEG     x0, x1
 6626 pipe_class ialu_reg(iRegI dst, iRegI src)
 6627 %{
 6628   single_instruction;
 6629   dst    : EX2(write);
 6630   src    : EX1(read);
 6631   INS01  : ISS;
 6632   ALU    : EX2;
 6633 %}
 6634 
 6635 // Integer ALU reg mmediate operation
 6636 // Eg.  ADD     x0, x1, #N
 6637 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6638 %{
 6639   single_instruction;
 6640   dst    : EX2(write);
 6641   src1   : EX1(read);
 6642   INS01  : ISS;
 6643   ALU    : EX2;
 6644 %}
 6645 
 6646 // Integer ALU immediate operation (no source operands)
 6647 // Eg.  MOV     x0, #N
 6648 pipe_class ialu_imm(iRegI dst)
 6649 %{
 6650   single_instruction;
 6651   dst    : EX1(write);
 6652   INS01  : ISS;
 6653   ALU    : EX1;
 6654 %}
 6655 
 6656 //------- Compare operation -------------------------------
 6657 
 6658 // Compare reg-reg
 6659 // Eg.  CMP     x0, x1
 6660 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6661 %{
 6662   single_instruction;
 6663 //  fixed_latency(16);
 6664   cr     : EX2(write);
 6665   op1    : EX1(read);
 6666   op2    : EX1(read);
 6667   INS01  : ISS;
 6668   ALU    : EX2;
 6669 %}
 6670 
 6671 // Compare reg-reg
 6672 // Eg.  CMP     x0, #N
 6673 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6674 %{
 6675   single_instruction;
 6676 //  fixed_latency(16);
 6677   cr     : EX2(write);
 6678   op1    : EX1(read);
 6679   INS01  : ISS;
 6680   ALU    : EX2;
 6681 %}
 6682 
 6683 //------- Conditional instructions ------------------------
 6684 
 6685 // Conditional no operands
 6686 // Eg.  CSINC   x0, zr, zr, <cond>
 6687 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6688 %{
 6689   single_instruction;
 6690   cr     : EX1(read);
 6691   dst    : EX2(write);
 6692   INS01  : ISS;
 6693   ALU    : EX2;
 6694 %}
 6695 
 6696 // Conditional 2 operand
 6697 // EG.  CSEL    X0, X1, X2, <cond>
 6698 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6699 %{
 6700   single_instruction;
 6701   cr     : EX1(read);
 6702   src1   : EX1(read);
 6703   src2   : EX1(read);
 6704   dst    : EX2(write);
 6705   INS01  : ISS;
 6706   ALU    : EX2;
 6707 %}
 6708 
 6709 // Conditional 2 operand
 6710 // EG.  CSEL    X0, X1, X2, <cond>
 6711 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6712 %{
 6713   single_instruction;
 6714   cr     : EX1(read);
 6715   src    : EX1(read);
 6716   dst    : EX2(write);
 6717   INS01  : ISS;
 6718   ALU    : EX2;
 6719 %}
 6720 
 6721 //------- Multiply pipeline operations --------------------
 6722 
 6723 // Multiply reg-reg
 6724 // Eg.  MUL     w0, w1, w2
 6725 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6726 %{
 6727   single_instruction;
 6728   dst    : WR(write);
 6729   src1   : ISS(read);
 6730   src2   : ISS(read);
 6731   INS01  : ISS;
 6732   MAC    : WR;
 6733 %}
 6734 
 6735 // Multiply accumulate
 6736 // Eg.  MADD    w0, w1, w2, w3
 6737 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6738 %{
 6739   single_instruction;
 6740   dst    : WR(write);
 6741   src1   : ISS(read);
 6742   src2   : ISS(read);
 6743   src3   : ISS(read);
 6744   INS01  : ISS;
 6745   MAC    : WR;
 6746 %}
 6747 
 6748 // Eg.  MUL     w0, w1, w2
 6749 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6750 %{
 6751   single_instruction;
 6752   fixed_latency(3); // Maximum latency for 64 bit mul
 6753   dst    : WR(write);
 6754   src1   : ISS(read);
 6755   src2   : ISS(read);
 6756   INS01  : ISS;
 6757   MAC    : WR;
 6758 %}
 6759 
 6760 // Multiply accumulate
 6761 // Eg.  MADD    w0, w1, w2, w3
 6762 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6763 %{
 6764   single_instruction;
 6765   fixed_latency(3); // Maximum latency for 64 bit mul
 6766   dst    : WR(write);
 6767   src1   : ISS(read);
 6768   src2   : ISS(read);
 6769   src3   : ISS(read);
 6770   INS01  : ISS;
 6771   MAC    : WR;
 6772 %}
 6773 
 6774 //------- Divide pipeline operations --------------------
 6775 
 6776 // Eg.  SDIV    w0, w1, w2
 6777 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6778 %{
 6779   single_instruction;
 6780   fixed_latency(8); // Maximum latency for 32 bit divide
 6781   dst    : WR(write);
 6782   src1   : ISS(read);
 6783   src2   : ISS(read);
 6784   INS0   : ISS; // Can only dual issue as instruction 0
 6785   DIV    : WR;
 6786 %}
 6787 
 6788 // Eg.  SDIV    x0, x1, x2
 6789 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6790 %{
 6791   single_instruction;
 6792   fixed_latency(16); // Maximum latency for 64 bit divide
 6793   dst    : WR(write);
 6794   src1   : ISS(read);
 6795   src2   : ISS(read);
 6796   INS0   : ISS; // Can only dual issue as instruction 0
 6797   DIV    : WR;
 6798 %}
 6799 
 6800 //------- Load pipeline operations ------------------------
 6801 
 6802 // Load - prefetch
 6803 // Eg.  PFRM    <mem>
 6804 pipe_class iload_prefetch(memory mem)
 6805 %{
 6806   single_instruction;
 6807   mem    : ISS(read);
 6808   INS01  : ISS;
 6809   LDST   : WR;
 6810 %}
 6811 
 6812 // Load - reg, mem
 6813 // Eg.  LDR     x0, <mem>
 6814 pipe_class iload_reg_mem(iRegI dst, memory mem)
 6815 %{
 6816   single_instruction;
 6817   dst    : WR(write);
 6818   mem    : ISS(read);
 6819   INS01  : ISS;
 6820   LDST   : WR;
 6821 %}
 6822 
 6823 // Load - reg, reg
 6824 // Eg.  LDR     x0, [sp, x1]
 6825 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 6826 %{
 6827   single_instruction;
 6828   dst    : WR(write);
 6829   src    : ISS(read);
 6830   INS01  : ISS;
 6831   LDST   : WR;
 6832 %}
 6833 
 6834 //------- Store pipeline operations -----------------------
 6835 
 6836 // Store - zr, mem
 6837 // Eg.  STR     zr, <mem>
 6838 pipe_class istore_mem(memory mem)
 6839 %{
 6840   single_instruction;
 6841   mem    : ISS(read);
 6842   INS01  : ISS;
 6843   LDST   : WR;
 6844 %}
 6845 
 6846 // Store - reg, mem
 6847 // Eg.  STR     x0, <mem>
 6848 pipe_class istore_reg_mem(iRegI src, memory mem)
 6849 %{
 6850   single_instruction;
 6851   mem    : ISS(read);
 6852   src    : EX2(read);
 6853   INS01  : ISS;
 6854   LDST   : WR;
 6855 %}
 6856 
 6857 // Store - reg, reg
 6858 // Eg. STR      x0, [sp, x1]
 6859 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 6860 %{
 6861   single_instruction;
 6862   dst    : ISS(read);
 6863   src    : EX2(read);
 6864   INS01  : ISS;
 6865   LDST   : WR;
 6866 %}
 6867 
 6868 //------- Store pipeline operations -----------------------
 6869 
 6870 // Branch
 6871 pipe_class pipe_branch()
 6872 %{
 6873   single_instruction;
 6874   INS01  : ISS;
 6875   BRANCH : EX1;
 6876 %}
 6877 
 6878 // Conditional branch
 6879 pipe_class pipe_branch_cond(rFlagsReg cr)
 6880 %{
 6881   single_instruction;
 6882   cr     : EX1(read);
 6883   INS01  : ISS;
 6884   BRANCH : EX1;
 6885 %}
 6886 
 6887 // Compare & Branch
 6888 // EG.  CBZ/CBNZ
 6889 pipe_class pipe_cmp_branch(iRegI op1)
 6890 %{
 6891   single_instruction;
 6892   op1    : EX1(read);
 6893   INS01  : ISS;
 6894   BRANCH : EX1;
 6895 %}
 6896 
 6897 //------- Synchronisation operations ----------------------
 6898 
 6899 // Any operation requiring serialization.
 6900 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 6901 pipe_class pipe_serial()
 6902 %{
 6903   single_instruction;
 6904   force_serialization;
 6905   fixed_latency(16);
 6906   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6907   LDST   : WR;
 6908 %}
 6909 
 6910 // Generic big/slow expanded idiom - also serialized
 6911 pipe_class pipe_slow()
 6912 %{
 6913   instruction_count(10);
 6914   multiple_bundles;
 6915   force_serialization;
 6916   fixed_latency(16);
 6917   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6918   LDST   : WR;
 6919 %}
 6920 
 6921 // Empty pipeline class
 6922 pipe_class pipe_class_empty()
 6923 %{
 6924   single_instruction;
 6925   fixed_latency(0);
 6926 %}
 6927 
 6928 // Default pipeline class.
 6929 pipe_class pipe_class_default()
 6930 %{
 6931   single_instruction;
 6932   fixed_latency(2);
 6933 %}
 6934 
 6935 // Pipeline class for compares.
 6936 pipe_class pipe_class_compare()
 6937 %{
 6938   single_instruction;
 6939   fixed_latency(16);
 6940 %}
 6941 
 6942 // Pipeline class for memory operations.
 6943 pipe_class pipe_class_memory()
 6944 %{
 6945   single_instruction;
 6946   fixed_latency(16);
 6947 %}
 6948 
 6949 // Pipeline class for call.
 6950 pipe_class pipe_class_call()
 6951 %{
 6952   single_instruction;
 6953   fixed_latency(100);
 6954 %}
 6955 
 6956 // Define the class for the Nop node.
 6957 define %{
 6958    MachNop = pipe_class_empty;
 6959 %}
 6960 
 6961 %}
 6962 //----------INSTRUCTIONS-------------------------------------------------------
 6963 //
 6964 // match      -- States which machine-independent subtree may be replaced
 6965 //               by this instruction.
 6966 // ins_cost   -- The estimated cost of this instruction is used by instruction
 6967 //               selection to identify a minimum cost tree of machine
 6968 //               instructions that matches a tree of machine-independent
 6969 //               instructions.
 6970 // format     -- A string providing the disassembly for this instruction.
 6971 //               The value of an instruction's operand may be inserted
 6972 //               by referring to it with a '$' prefix.
 6973 // opcode     -- Three instruction opcodes may be provided.  These are referred
 6974 //               to within an encode class as $primary, $secondary, and $tertiary
 6975 //               rrspectively.  The primary opcode is commonly used to
 6976 //               indicate the type of machine instruction, while secondary
 6977 //               and tertiary are often used for prefix options or addressing
 6978 //               modes.
 6979 // ins_encode -- A list of encode classes with parameters. The encode class
 6980 //               name must have been defined in an 'enc_class' specification
 6981 //               in the encode section of the architecture description.
 6982 
 6983 // ============================================================================
 6984 // Memory (Load/Store) Instructions
 6985 
 6986 // Load Instructions
 6987 
 6988 // Load Byte (8 bit signed)
 6989 instruct loadB(iRegINoSp dst, memory1 mem)
 6990 %{
 6991   match(Set dst (LoadB mem));
 6992   predicate(!needs_acquiring_load(n));
 6993 
 6994   ins_cost(4 * INSN_COST);
 6995   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 6996 
 6997   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 6998 
 6999   ins_pipe(iload_reg_mem);
 7000 %}
 7001 
 7002 // Load Byte (8 bit signed) into long
 7003 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 7004 %{
 7005   match(Set dst (ConvI2L (LoadB mem)));
 7006   predicate(!needs_acquiring_load(n->in(1)));
 7007 
 7008   ins_cost(4 * INSN_COST);
 7009   format %{ "ldrsb  $dst, $mem\t# byte" %}
 7010 
 7011   ins_encode(aarch64_enc_ldrsb(dst, mem));
 7012 
 7013   ins_pipe(iload_reg_mem);
 7014 %}
 7015 
 7016 // Load Byte (8 bit unsigned)
 7017 instruct loadUB(iRegINoSp dst, memory1 mem)
 7018 %{
 7019   match(Set dst (LoadUB mem));
 7020   predicate(!needs_acquiring_load(n));
 7021 
 7022   ins_cost(4 * INSN_COST);
 7023   format %{ "ldrbw  $dst, $mem\t# byte" %}
 7024 
 7025   ins_encode(aarch64_enc_ldrb(dst, mem));
 7026 
 7027   ins_pipe(iload_reg_mem);
 7028 %}
 7029 
 7030 // Load Byte (8 bit unsigned) into long
 7031 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7032 %{
 7033   match(Set dst (ConvI2L (LoadUB mem)));
 7034   predicate(!needs_acquiring_load(n->in(1)));
 7035 
 7036   ins_cost(4 * INSN_COST);
 7037   format %{ "ldrb  $dst, $mem\t# byte" %}
 7038 
 7039   ins_encode(aarch64_enc_ldrb(dst, mem));
 7040 
 7041   ins_pipe(iload_reg_mem);
 7042 %}
 7043 
 7044 // Load Short (16 bit signed)
 7045 instruct loadS(iRegINoSp dst, memory2 mem)
 7046 %{
 7047   match(Set dst (LoadS mem));
 7048   predicate(!needs_acquiring_load(n));
 7049 
 7050   ins_cost(4 * INSN_COST);
 7051   format %{ "ldrshw  $dst, $mem\t# short" %}
 7052 
 7053   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7054 
 7055   ins_pipe(iload_reg_mem);
 7056 %}
 7057 
 7058 // Load Short (16 bit signed) into long
 7059 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7060 %{
 7061   match(Set dst (ConvI2L (LoadS mem)));
 7062   predicate(!needs_acquiring_load(n->in(1)));
 7063 
 7064   ins_cost(4 * INSN_COST);
 7065   format %{ "ldrsh  $dst, $mem\t# short" %}
 7066 
 7067   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7068 
 7069   ins_pipe(iload_reg_mem);
 7070 %}
 7071 
 7072 // Load Char (16 bit unsigned)
 7073 instruct loadUS(iRegINoSp dst, memory2 mem)
 7074 %{
 7075   match(Set dst (LoadUS mem));
 7076   predicate(!needs_acquiring_load(n));
 7077 
 7078   ins_cost(4 * INSN_COST);
 7079   format %{ "ldrh  $dst, $mem\t# short" %}
 7080 
 7081   ins_encode(aarch64_enc_ldrh(dst, mem));
 7082 
 7083   ins_pipe(iload_reg_mem);
 7084 %}
 7085 
 7086 // Load Short/Char (16 bit unsigned) into long
 7087 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7088 %{
 7089   match(Set dst (ConvI2L (LoadUS mem)));
 7090   predicate(!needs_acquiring_load(n->in(1)));
 7091 
 7092   ins_cost(4 * INSN_COST);
 7093   format %{ "ldrh  $dst, $mem\t# short" %}
 7094 
 7095   ins_encode(aarch64_enc_ldrh(dst, mem));
 7096 
 7097   ins_pipe(iload_reg_mem);
 7098 %}
 7099 
 7100 // Load Integer (32 bit signed)
 7101 instruct loadI(iRegINoSp dst, memory4 mem)
 7102 %{
 7103   match(Set dst (LoadI mem));
 7104   predicate(!needs_acquiring_load(n));
 7105 
 7106   ins_cost(4 * INSN_COST);
 7107   format %{ "ldrw  $dst, $mem\t# int" %}
 7108 
 7109   ins_encode(aarch64_enc_ldrw(dst, mem));
 7110 
 7111   ins_pipe(iload_reg_mem);
 7112 %}
 7113 
 7114 // Load Integer (32 bit signed) into long
 7115 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7116 %{
 7117   match(Set dst (ConvI2L (LoadI mem)));
 7118   predicate(!needs_acquiring_load(n->in(1)));
 7119 
 7120   ins_cost(4 * INSN_COST);
 7121   format %{ "ldrsw  $dst, $mem\t# int" %}
 7122 
 7123   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7124 
 7125   ins_pipe(iload_reg_mem);
 7126 %}
 7127 
 7128 // Load Integer (32 bit unsigned) into long
 7129 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7130 %{
 7131   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7132   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7133 
 7134   ins_cost(4 * INSN_COST);
 7135   format %{ "ldrw  $dst, $mem\t# int" %}
 7136 
 7137   ins_encode(aarch64_enc_ldrw(dst, mem));
 7138 
 7139   ins_pipe(iload_reg_mem);
 7140 %}
 7141 
 7142 // Load Long (64 bit signed)
 7143 instruct loadL(iRegLNoSp dst, memory8 mem)
 7144 %{
 7145   match(Set dst (LoadL mem));
 7146   predicate(!needs_acquiring_load(n));
 7147 
 7148   ins_cost(4 * INSN_COST);
 7149   format %{ "ldr  $dst, $mem\t# int" %}
 7150 
 7151   ins_encode(aarch64_enc_ldr(dst, mem));
 7152 
 7153   ins_pipe(iload_reg_mem);
 7154 %}
 7155 
 7156 // Load Range
 7157 instruct loadRange(iRegINoSp dst, memory4 mem)
 7158 %{
 7159   match(Set dst (LoadRange mem));
 7160 
 7161   ins_cost(4 * INSN_COST);
 7162   format %{ "ldrw  $dst, $mem\t# range" %}
 7163 
 7164   ins_encode(aarch64_enc_ldrw(dst, mem));
 7165 
 7166   ins_pipe(iload_reg_mem);
 7167 %}
 7168 
 7169 // Load Pointer
 7170 instruct loadP(iRegPNoSp dst, memory8 mem)
 7171 %{
 7172   match(Set dst (LoadP mem));
 7173   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7174 
 7175   ins_cost(4 * INSN_COST);
 7176   format %{ "ldr  $dst, $mem\t# ptr" %}
 7177 
 7178   ins_encode(aarch64_enc_ldr(dst, mem));
 7179 
 7180   ins_pipe(iload_reg_mem);
 7181 %}
 7182 
 7183 // Load Compressed Pointer
 7184 instruct loadN(iRegNNoSp dst, memory4 mem)
 7185 %{
 7186   match(Set dst (LoadN mem));
 7187   predicate(!needs_acquiring_load(n));
 7188 
 7189   ins_cost(4 * INSN_COST);
 7190   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7191 
 7192   ins_encode(aarch64_enc_ldrw(dst, mem));
 7193 
 7194   ins_pipe(iload_reg_mem);
 7195 %}
 7196 
 7197 // Load Klass Pointer
 7198 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7199 %{
 7200   match(Set dst (LoadKlass mem));
 7201   predicate(!needs_acquiring_load(n));
 7202 
 7203   ins_cost(4 * INSN_COST);
 7204   format %{ "ldr  $dst, $mem\t# class" %}
 7205 
 7206   ins_encode(aarch64_enc_ldr(dst, mem));
 7207 
 7208   ins_pipe(iload_reg_mem);
 7209 %}
 7210 
 7211 // Load Narrow Klass Pointer
 7212 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7213 %{
 7214   match(Set dst (LoadNKlass mem));
 7215   predicate(!needs_acquiring_load(n));
 7216 
 7217   ins_cost(4 * INSN_COST);
 7218   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7219 
 7220   ins_encode(aarch64_enc_ldrw(dst, mem));
 7221 
 7222   ins_pipe(iload_reg_mem);
 7223 %}
 7224 
 7225 // Load Float
 7226 instruct loadF(vRegF dst, memory4 mem)
 7227 %{
 7228   match(Set dst (LoadF mem));
 7229   predicate(!needs_acquiring_load(n));
 7230 
 7231   ins_cost(4 * INSN_COST);
 7232   format %{ "ldrs  $dst, $mem\t# float" %}
 7233 
 7234   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7235 
 7236   ins_pipe(pipe_class_memory);
 7237 %}
 7238 
 7239 // Load Double
 7240 instruct loadD(vRegD dst, memory8 mem)
 7241 %{
 7242   match(Set dst (LoadD mem));
 7243   predicate(!needs_acquiring_load(n));
 7244 
 7245   ins_cost(4 * INSN_COST);
 7246   format %{ "ldrd  $dst, $mem\t# double" %}
 7247 
 7248   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7249 
 7250   ins_pipe(pipe_class_memory);
 7251 %}
 7252 
 7253 
 7254 // Load Int Constant
 7255 instruct loadConI(iRegINoSp dst, immI src)
 7256 %{
 7257   match(Set dst src);
 7258 
 7259   ins_cost(INSN_COST);
 7260   format %{ "mov $dst, $src\t# int" %}
 7261 
 7262   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7263 
 7264   ins_pipe(ialu_imm);
 7265 %}
 7266 
 7267 // Load Long Constant
 7268 instruct loadConL(iRegLNoSp dst, immL src)
 7269 %{
 7270   match(Set dst src);
 7271 
 7272   ins_cost(INSN_COST);
 7273   format %{ "mov $dst, $src\t# long" %}
 7274 
 7275   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7276 
 7277   ins_pipe(ialu_imm);
 7278 %}
 7279 
 7280 // Load Pointer Constant
 7281 
 7282 instruct loadConP(iRegPNoSp dst, immP con)
 7283 %{
 7284   match(Set dst con);
 7285 
 7286   ins_cost(INSN_COST * 4);
 7287   format %{
 7288     "mov  $dst, $con\t# ptr"
 7289   %}
 7290 
 7291   ins_encode(aarch64_enc_mov_p(dst, con));
 7292 
 7293   ins_pipe(ialu_imm);
 7294 %}
 7295 
 7296 // Load Null Pointer Constant
 7297 
 7298 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7299 %{
 7300   match(Set dst con);
 7301 
 7302   ins_cost(INSN_COST);
 7303   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7304 
 7305   ins_encode(aarch64_enc_mov_p0(dst, con));
 7306 
 7307   ins_pipe(ialu_imm);
 7308 %}
 7309 
 7310 // Load Pointer Constant One
 7311 
 7312 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7313 %{
 7314   match(Set dst con);
 7315 
 7316   ins_cost(INSN_COST);
 7317   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7318 
 7319   ins_encode(aarch64_enc_mov_p1(dst, con));
 7320 
 7321   ins_pipe(ialu_imm);
 7322 %}
 7323 
 7324 // Load Byte Map Base Constant
 7325 
 7326 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7327 %{
 7328   match(Set dst con);
 7329 
 7330   ins_cost(INSN_COST);
 7331   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7332 
 7333   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7334 
 7335   ins_pipe(ialu_imm);
 7336 %}
 7337 
 7338 // Load Narrow Pointer Constant
 7339 
 7340 instruct loadConN(iRegNNoSp dst, immN con)
 7341 %{
 7342   match(Set dst con);
 7343 
 7344   ins_cost(INSN_COST * 4);
 7345   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7346 
 7347   ins_encode(aarch64_enc_mov_n(dst, con));
 7348 
 7349   ins_pipe(ialu_imm);
 7350 %}
 7351 
 7352 // Load Narrow Null Pointer Constant
 7353 
 7354 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7355 %{
 7356   match(Set dst con);
 7357 
 7358   ins_cost(INSN_COST);
 7359   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7360 
 7361   ins_encode(aarch64_enc_mov_n0(dst, con));
 7362 
 7363   ins_pipe(ialu_imm);
 7364 %}
 7365 
 7366 // Load Narrow Klass Constant
 7367 
 7368 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7369 %{
 7370   match(Set dst con);
 7371 
 7372   ins_cost(INSN_COST);
 7373   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7374 
 7375   ins_encode(aarch64_enc_mov_nk(dst, con));
 7376 
 7377   ins_pipe(ialu_imm);
 7378 %}
 7379 
 7380 // Load Packed Float Constant
 7381 
 7382 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7383   match(Set dst con);
 7384   ins_cost(INSN_COST * 4);
 7385   format %{ "fmovs  $dst, $con"%}
 7386   ins_encode %{
 7387     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7388   %}
 7389 
 7390   ins_pipe(fp_imm_s);
 7391 %}
 7392 
 7393 // Load Float Constant
 7394 
 7395 instruct loadConF(vRegF dst, immF con) %{
 7396   match(Set dst con);
 7397 
 7398   ins_cost(INSN_COST * 4);
 7399 
 7400   format %{
 7401     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7402   %}
 7403 
 7404   ins_encode %{
 7405     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7406   %}
 7407 
 7408   ins_pipe(fp_load_constant_s);
 7409 %}
 7410 
 7411 // Load Packed Double Constant
 7412 
 7413 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7414   match(Set dst con);
 7415   ins_cost(INSN_COST);
 7416   format %{ "fmovd  $dst, $con"%}
 7417   ins_encode %{
 7418     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7419   %}
 7420 
 7421   ins_pipe(fp_imm_d);
 7422 %}
 7423 
 7424 // Load Double Constant
 7425 
 7426 instruct loadConD(vRegD dst, immD con) %{
 7427   match(Set dst con);
 7428 
 7429   ins_cost(INSN_COST * 5);
 7430   format %{
 7431     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7432   %}
 7433 
 7434   ins_encode %{
 7435     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7436   %}
 7437 
 7438   ins_pipe(fp_load_constant_d);
 7439 %}
 7440 
 7441 // Store Instructions
 7442 
 7443 // Store CMS card-mark Immediate
 7444 instruct storeimmCM0(immI0 zero, memory1 mem)
 7445 %{
 7446   match(Set mem (StoreCM mem zero));
 7447 
 7448   ins_cost(INSN_COST);
 7449   format %{ "storestore (elided)\n\t"
 7450             "strb zr, $mem\t# byte" %}
 7451 
 7452   ins_encode(aarch64_enc_strb0(mem));
 7453 
 7454   ins_pipe(istore_mem);
 7455 %}
 7456 
 7457 // Store CMS card-mark Immediate with intervening StoreStore
 7458 // needed when using CMS with no conditional card marking
 7459 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7460 %{
 7461   match(Set mem (StoreCM mem zero));
 7462 
 7463   ins_cost(INSN_COST * 2);
 7464   format %{ "storestore\n\t"
 7465             "dmb ishst"
 7466             "\n\tstrb zr, $mem\t# byte" %}
 7467 
 7468   ins_encode(aarch64_enc_strb0_ordered(mem));
 7469 
 7470   ins_pipe(istore_mem);
 7471 %}
 7472 
 7473 // Store Byte
 7474 instruct storeB(iRegIorL2I src, memory1 mem)
 7475 %{
 7476   match(Set mem (StoreB mem src));
 7477   predicate(!needs_releasing_store(n));
 7478 
 7479   ins_cost(INSN_COST);
 7480   format %{ "strb  $src, $mem\t# byte" %}
 7481 
 7482   ins_encode(aarch64_enc_strb(src, mem));
 7483 
 7484   ins_pipe(istore_reg_mem);
 7485 %}
 7486 
 7487 
 7488 instruct storeimmB0(immI0 zero, memory1 mem)
 7489 %{
 7490   match(Set mem (StoreB mem zero));
 7491   predicate(!needs_releasing_store(n));
 7492 
 7493   ins_cost(INSN_COST);
 7494   format %{ "strb rscractch2, $mem\t# byte" %}
 7495 
 7496   ins_encode(aarch64_enc_strb0(mem));
 7497 
 7498   ins_pipe(istore_mem);
 7499 %}
 7500 
 7501 // Store Char/Short
 7502 instruct storeC(iRegIorL2I src, memory2 mem)
 7503 %{
 7504   match(Set mem (StoreC mem src));
 7505   predicate(!needs_releasing_store(n));
 7506 
 7507   ins_cost(INSN_COST);
 7508   format %{ "strh  $src, $mem\t# short" %}
 7509 
 7510   ins_encode(aarch64_enc_strh(src, mem));
 7511 
 7512   ins_pipe(istore_reg_mem);
 7513 %}
 7514 
 7515 instruct storeimmC0(immI0 zero, memory2 mem)
 7516 %{
 7517   match(Set mem (StoreC mem zero));
 7518   predicate(!needs_releasing_store(n));
 7519 
 7520   ins_cost(INSN_COST);
 7521   format %{ "strh  zr, $mem\t# short" %}
 7522 
 7523   ins_encode(aarch64_enc_strh0(mem));
 7524 
 7525   ins_pipe(istore_mem);
 7526 %}
 7527 
 7528 // Store Integer
 7529 
 7530 instruct storeI(iRegIorL2I src, memory4 mem)
 7531 %{
 7532   match(Set mem(StoreI mem src));
 7533   predicate(!needs_releasing_store(n));
 7534 
 7535   ins_cost(INSN_COST);
 7536   format %{ "strw  $src, $mem\t# int" %}
 7537 
 7538   ins_encode(aarch64_enc_strw(src, mem));
 7539 
 7540   ins_pipe(istore_reg_mem);
 7541 %}
 7542 
 7543 instruct storeimmI0(immI0 zero, memory4 mem)
 7544 %{
 7545   match(Set mem(StoreI mem zero));
 7546   predicate(!needs_releasing_store(n));
 7547 
 7548   ins_cost(INSN_COST);
 7549   format %{ "strw  zr, $mem\t# int" %}
 7550 
 7551   ins_encode(aarch64_enc_strw0(mem));
 7552 
 7553   ins_pipe(istore_mem);
 7554 %}
 7555 
 7556 // Store Long (64 bit signed)
 7557 instruct storeL(iRegL src, memory8 mem)
 7558 %{
 7559   match(Set mem (StoreL mem src));
 7560   predicate(!needs_releasing_store(n));
 7561 
 7562   ins_cost(INSN_COST);
 7563   format %{ "str  $src, $mem\t# int" %}
 7564 
 7565   ins_encode(aarch64_enc_str(src, mem));
 7566 
 7567   ins_pipe(istore_reg_mem);
 7568 %}
 7569 
 7570 // Store Long (64 bit signed)
 7571 instruct storeimmL0(immL0 zero, memory8 mem)
 7572 %{
 7573   match(Set mem (StoreL mem zero));
 7574   predicate(!needs_releasing_store(n));
 7575 
 7576   ins_cost(INSN_COST);
 7577   format %{ "str  zr, $mem\t# int" %}
 7578 
 7579   ins_encode(aarch64_enc_str0(mem));
 7580 
 7581   ins_pipe(istore_mem);
 7582 %}
 7583 
 7584 // Store Pointer
 7585 instruct storeP(iRegP src, memory8 mem)
 7586 %{
 7587   match(Set mem (StoreP mem src));
 7588   predicate(!needs_releasing_store(n));
 7589 
 7590   ins_cost(INSN_COST);
 7591   format %{ "str  $src, $mem\t# ptr" %}
 7592 
 7593   ins_encode(aarch64_enc_str(src, mem));
 7594 
 7595   ins_pipe(istore_reg_mem);
 7596 %}
 7597 
 7598 // Store Pointer
 7599 instruct storeimmP0(immP0 zero, memory8 mem)
 7600 %{
 7601   match(Set mem (StoreP mem zero));
 7602   predicate(!needs_releasing_store(n));
 7603 
 7604   ins_cost(INSN_COST);
 7605   format %{ "str zr, $mem\t# ptr" %}
 7606 
 7607   ins_encode(aarch64_enc_str0(mem));
 7608 
 7609   ins_pipe(istore_mem);
 7610 %}
 7611 
 7612 // Store Compressed Pointer
 7613 instruct storeN(iRegN src, memory4 mem)
 7614 %{
 7615   match(Set mem (StoreN mem src));
 7616   predicate(!needs_releasing_store(n));
 7617 
 7618   ins_cost(INSN_COST);
 7619   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7620 
 7621   ins_encode(aarch64_enc_strw(src, mem));
 7622 
 7623   ins_pipe(istore_reg_mem);
 7624 %}
 7625 
 7626 instruct storeImmN0(immN0 zero, memory4 mem)
 7627 %{
 7628   match(Set mem (StoreN mem zero));
 7629   predicate(!needs_releasing_store(n));
 7630 
 7631   ins_cost(INSN_COST);
 7632   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7633 
 7634   ins_encode(aarch64_enc_strw0(mem));
 7635 
 7636   ins_pipe(istore_mem);
 7637 %}
 7638 
 7639 // Store Float
 7640 instruct storeF(vRegF src, memory4 mem)
 7641 %{
 7642   match(Set mem (StoreF mem src));
 7643   predicate(!needs_releasing_store(n));
 7644 
 7645   ins_cost(INSN_COST);
 7646   format %{ "strs  $src, $mem\t# float" %}
 7647 
 7648   ins_encode( aarch64_enc_strs(src, mem) );
 7649 
 7650   ins_pipe(pipe_class_memory);
 7651 %}
 7652 
 7653 // TODO
 7654 // implement storeImmF0 and storeFImmPacked
 7655 
 7656 // Store Double
 7657 instruct storeD(vRegD src, memory8 mem)
 7658 %{
 7659   match(Set mem (StoreD mem src));
 7660   predicate(!needs_releasing_store(n));
 7661 
 7662   ins_cost(INSN_COST);
 7663   format %{ "strd  $src, $mem\t# double" %}
 7664 
 7665   ins_encode( aarch64_enc_strd(src, mem) );
 7666 
 7667   ins_pipe(pipe_class_memory);
 7668 %}
 7669 
 7670 // Store Compressed Klass Pointer
 7671 instruct storeNKlass(iRegN src, memory4 mem)
 7672 %{
 7673   predicate(!needs_releasing_store(n));
 7674   match(Set mem (StoreNKlass mem src));
 7675 
 7676   ins_cost(INSN_COST);
 7677   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7678 
 7679   ins_encode(aarch64_enc_strw(src, mem));
 7680 
 7681   ins_pipe(istore_reg_mem);
 7682 %}
 7683 
 7684 // TODO
 7685 // implement storeImmD0 and storeDImmPacked
 7686 
 7687 // prefetch instructions
 7688 // Must be safe to execute with invalid address (cannot fault).
 7689 
 7690 instruct prefetchalloc( memory8 mem ) %{
 7691   match(PrefetchAllocation mem);
 7692 
 7693   ins_cost(INSN_COST);
 7694   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7695 
 7696   ins_encode( aarch64_enc_prefetchw(mem) );
 7697 
 7698   ins_pipe(iload_prefetch);
 7699 %}
 7700 
 7701 //  ---------------- volatile loads and stores ----------------
 7702 
 7703 // Load Byte (8 bit signed)
 7704 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7705 %{
 7706   match(Set dst (LoadB mem));
 7707 
 7708   ins_cost(VOLATILE_REF_COST);
 7709   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7710 
 7711   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7712 
 7713   ins_pipe(pipe_serial);
 7714 %}
 7715 
 7716 // Load Byte (8 bit signed) into long
 7717 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7718 %{
 7719   match(Set dst (ConvI2L (LoadB mem)));
 7720 
 7721   ins_cost(VOLATILE_REF_COST);
 7722   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7723 
 7724   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7725 
 7726   ins_pipe(pipe_serial);
 7727 %}
 7728 
 7729 // Load Byte (8 bit unsigned)
 7730 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7731 %{
 7732   match(Set dst (LoadUB mem));
 7733 
 7734   ins_cost(VOLATILE_REF_COST);
 7735   format %{ "ldarb  $dst, $mem\t# byte" %}
 7736 
 7737   ins_encode(aarch64_enc_ldarb(dst, mem));
 7738 
 7739   ins_pipe(pipe_serial);
 7740 %}
 7741 
 7742 // Load Byte (8 bit unsigned) into long
 7743 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7744 %{
 7745   match(Set dst (ConvI2L (LoadUB mem)));
 7746 
 7747   ins_cost(VOLATILE_REF_COST);
 7748   format %{ "ldarb  $dst, $mem\t# byte" %}
 7749 
 7750   ins_encode(aarch64_enc_ldarb(dst, mem));
 7751 
 7752   ins_pipe(pipe_serial);
 7753 %}
 7754 
 7755 // Load Short (16 bit signed)
 7756 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7757 %{
 7758   match(Set dst (LoadS mem));
 7759 
 7760   ins_cost(VOLATILE_REF_COST);
 7761   format %{ "ldarshw  $dst, $mem\t# short" %}
 7762 
 7763   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7764 
 7765   ins_pipe(pipe_serial);
 7766 %}
 7767 
 7768 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7769 %{
 7770   match(Set dst (LoadUS mem));
 7771 
 7772   ins_cost(VOLATILE_REF_COST);
 7773   format %{ "ldarhw  $dst, $mem\t# short" %}
 7774 
 7775   ins_encode(aarch64_enc_ldarhw(dst, mem));
 7776 
 7777   ins_pipe(pipe_serial);
 7778 %}
 7779 
 7780 // Load Short/Char (16 bit unsigned) into long
 7781 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7782 %{
 7783   match(Set dst (ConvI2L (LoadUS mem)));
 7784 
 7785   ins_cost(VOLATILE_REF_COST);
 7786   format %{ "ldarh  $dst, $mem\t# short" %}
 7787 
 7788   ins_encode(aarch64_enc_ldarh(dst, mem));
 7789 
 7790   ins_pipe(pipe_serial);
 7791 %}
 7792 
 7793 // Load Short/Char (16 bit signed) into long
 7794 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7795 %{
 7796   match(Set dst (ConvI2L (LoadS mem)));
 7797 
 7798   ins_cost(VOLATILE_REF_COST);
 7799   format %{ "ldarh  $dst, $mem\t# short" %}
 7800 
 7801   ins_encode(aarch64_enc_ldarsh(dst, mem));
 7802 
 7803   ins_pipe(pipe_serial);
 7804 %}
 7805 
 7806 // Load Integer (32 bit signed)
 7807 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7808 %{
 7809   match(Set dst (LoadI mem));
 7810 
 7811   ins_cost(VOLATILE_REF_COST);
 7812   format %{ "ldarw  $dst, $mem\t# int" %}
 7813 
 7814   ins_encode(aarch64_enc_ldarw(dst, mem));
 7815 
 7816   ins_pipe(pipe_serial);
 7817 %}
 7818 
 7819 // Load Integer (32 bit unsigned) into long
 7820 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 7821 %{
 7822   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7823 
 7824   ins_cost(VOLATILE_REF_COST);
 7825   format %{ "ldarw  $dst, $mem\t# int" %}
 7826 
 7827   ins_encode(aarch64_enc_ldarw(dst, mem));
 7828 
 7829   ins_pipe(pipe_serial);
 7830 %}
 7831 
 7832 // Load Long (64 bit signed)
 7833 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7834 %{
 7835   match(Set dst (LoadL mem));
 7836 
 7837   ins_cost(VOLATILE_REF_COST);
 7838   format %{ "ldar  $dst, $mem\t# int" %}
 7839 
 7840   ins_encode(aarch64_enc_ldar(dst, mem));
 7841 
 7842   ins_pipe(pipe_serial);
 7843 %}
 7844 
 7845 // Load Pointer
 7846 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 7847 %{
 7848   match(Set dst (LoadP mem));
 7849   predicate(n->as_Load()->barrier_data() == 0);
 7850 
 7851   ins_cost(VOLATILE_REF_COST);
 7852   format %{ "ldar  $dst, $mem\t# ptr" %}
 7853 
 7854   ins_encode(aarch64_enc_ldar(dst, mem));
 7855 
 7856   ins_pipe(pipe_serial);
 7857 %}
 7858 
 7859 // Load Compressed Pointer
 7860 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 7861 %{
 7862   match(Set dst (LoadN mem));
 7863 
 7864   ins_cost(VOLATILE_REF_COST);
 7865   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 7866 
 7867   ins_encode(aarch64_enc_ldarw(dst, mem));
 7868 
 7869   ins_pipe(pipe_serial);
 7870 %}
 7871 
 7872 // Load Float
 7873 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 7874 %{
 7875   match(Set dst (LoadF mem));
 7876 
 7877   ins_cost(VOLATILE_REF_COST);
 7878   format %{ "ldars  $dst, $mem\t# float" %}
 7879 
 7880   ins_encode( aarch64_enc_fldars(dst, mem) );
 7881 
 7882   ins_pipe(pipe_serial);
 7883 %}
 7884 
 7885 // Load Double
 7886 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 7887 %{
 7888   match(Set dst (LoadD mem));
 7889 
 7890   ins_cost(VOLATILE_REF_COST);
 7891   format %{ "ldard  $dst, $mem\t# double" %}
 7892 
 7893   ins_encode( aarch64_enc_fldard(dst, mem) );
 7894 
 7895   ins_pipe(pipe_serial);
 7896 %}
 7897 
 7898 // Store Byte
 7899 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7900 %{
 7901   match(Set mem (StoreB mem src));
 7902 
 7903   ins_cost(VOLATILE_REF_COST);
 7904   format %{ "stlrb  $src, $mem\t# byte" %}
 7905 
 7906   ins_encode(aarch64_enc_stlrb(src, mem));
 7907 
 7908   ins_pipe(pipe_class_memory);
 7909 %}
 7910 
 7911 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7912 %{
 7913   match(Set mem (StoreB mem zero));
 7914 
 7915   ins_cost(VOLATILE_REF_COST);
 7916   format %{ "stlrb  zr, $mem\t# byte" %}
 7917 
 7918   ins_encode(aarch64_enc_stlrb0(mem));
 7919 
 7920   ins_pipe(pipe_class_memory);
 7921 %}
 7922 
 7923 // Store Char/Short
 7924 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7925 %{
 7926   match(Set mem (StoreC mem src));
 7927 
 7928   ins_cost(VOLATILE_REF_COST);
 7929   format %{ "stlrh  $src, $mem\t# short" %}
 7930 
 7931   ins_encode(aarch64_enc_stlrh(src, mem));
 7932 
 7933   ins_pipe(pipe_class_memory);
 7934 %}
 7935 
 7936 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7937 %{
 7938   match(Set mem (StoreC mem zero));
 7939 
 7940   ins_cost(VOLATILE_REF_COST);
 7941   format %{ "stlrh  zr, $mem\t# short" %}
 7942 
 7943   ins_encode(aarch64_enc_stlrh0(mem));
 7944 
 7945   ins_pipe(pipe_class_memory);
 7946 %}
 7947 
 7948 // Store Integer
 7949 
 7950 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7951 %{
 7952   match(Set mem(StoreI mem src));
 7953 
 7954   ins_cost(VOLATILE_REF_COST);
 7955   format %{ "stlrw  $src, $mem\t# int" %}
 7956 
 7957   ins_encode(aarch64_enc_stlrw(src, mem));
 7958 
 7959   ins_pipe(pipe_class_memory);
 7960 %}
 7961 
 7962 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7963 %{
 7964   match(Set mem(StoreI mem zero));
 7965 
 7966   ins_cost(VOLATILE_REF_COST);
 7967   format %{ "stlrw  zr, $mem\t# int" %}
 7968 
 7969   ins_encode(aarch64_enc_stlrw0(mem));
 7970 
 7971   ins_pipe(pipe_class_memory);
 7972 %}
 7973 
 7974 // Store Long (64 bit signed)
 7975 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 7976 %{
 7977   match(Set mem (StoreL mem src));
 7978 
 7979   ins_cost(VOLATILE_REF_COST);
 7980   format %{ "stlr  $src, $mem\t# int" %}
 7981 
 7982   ins_encode(aarch64_enc_stlr(src, mem));
 7983 
 7984   ins_pipe(pipe_class_memory);
 7985 %}
 7986 
 7987 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 7988 %{
 7989   match(Set mem (StoreL mem zero));
 7990 
 7991   ins_cost(VOLATILE_REF_COST);
 7992   format %{ "stlr  zr, $mem\t# int" %}
 7993 
 7994   ins_encode(aarch64_enc_stlr0(mem));
 7995 
 7996   ins_pipe(pipe_class_memory);
 7997 %}
 7998 
 7999 // Store Pointer
 8000 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 8001 %{
 8002   match(Set mem (StoreP mem src));
 8003 
 8004   ins_cost(VOLATILE_REF_COST);
 8005   format %{ "stlr  $src, $mem\t# ptr" %}
 8006 
 8007   ins_encode(aarch64_enc_stlr(src, mem));
 8008 
 8009   ins_pipe(pipe_class_memory);
 8010 %}
 8011 
 8012 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 8013 %{
 8014   match(Set mem (StoreP mem zero));
 8015 
 8016   ins_cost(VOLATILE_REF_COST);
 8017   format %{ "stlr  zr, $mem\t# ptr" %}
 8018 
 8019   ins_encode(aarch64_enc_stlr0(mem));
 8020 
 8021   ins_pipe(pipe_class_memory);
 8022 %}
 8023 
 8024 // Store Compressed Pointer
 8025 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8026 %{
 8027   match(Set mem (StoreN mem src));
 8028 
 8029   ins_cost(VOLATILE_REF_COST);
 8030   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8031 
 8032   ins_encode(aarch64_enc_stlrw(src, mem));
 8033 
 8034   ins_pipe(pipe_class_memory);
 8035 %}
 8036 
 8037 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 8038 %{
 8039   match(Set mem (StoreN mem zero));
 8040 
 8041   ins_cost(VOLATILE_REF_COST);
 8042   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 8043 
 8044   ins_encode(aarch64_enc_stlrw0(mem));
 8045 
 8046   ins_pipe(pipe_class_memory);
 8047 %}
 8048 
 8049 // Store Float
 8050 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8051 %{
 8052   match(Set mem (StoreF mem src));
 8053 
 8054   ins_cost(VOLATILE_REF_COST);
 8055   format %{ "stlrs  $src, $mem\t# float" %}
 8056 
 8057   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8058 
 8059   ins_pipe(pipe_class_memory);
 8060 %}
 8061 
 8062 // TODO
 8063 // implement storeImmF0 and storeFImmPacked
 8064 
 8065 // Store Double
 8066 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8067 %{
 8068   match(Set mem (StoreD mem src));
 8069 
 8070   ins_cost(VOLATILE_REF_COST);
 8071   format %{ "stlrd  $src, $mem\t# double" %}
 8072 
 8073   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8074 
 8075   ins_pipe(pipe_class_memory);
 8076 %}
 8077 
 8078 //  ---------------- end of volatile loads and stores ----------------
 8079 
 8080 instruct cacheWB(indirect addr)
 8081 %{
 8082   predicate(VM_Version::supports_data_cache_line_flush());
 8083   match(CacheWB addr);
 8084 
 8085   ins_cost(100);
 8086   format %{"cache wb $addr" %}
 8087   ins_encode %{
 8088     assert($addr->index_position() < 0, "should be");
 8089     assert($addr$$disp == 0, "should be");
 8090     __ cache_wb(Address($addr$$base$$Register, 0));
 8091   %}
 8092   ins_pipe(pipe_slow); // XXX
 8093 %}
 8094 
 8095 instruct cacheWBPreSync()
 8096 %{
 8097   predicate(VM_Version::supports_data_cache_line_flush());
 8098   match(CacheWBPreSync);
 8099 
 8100   ins_cost(100);
 8101   format %{"cache wb presync" %}
 8102   ins_encode %{
 8103     __ cache_wbsync(true);
 8104   %}
 8105   ins_pipe(pipe_slow); // XXX
 8106 %}
 8107 
 8108 instruct cacheWBPostSync()
 8109 %{
 8110   predicate(VM_Version::supports_data_cache_line_flush());
 8111   match(CacheWBPostSync);
 8112 
 8113   ins_cost(100);
 8114   format %{"cache wb postsync" %}
 8115   ins_encode %{
 8116     __ cache_wbsync(false);
 8117   %}
 8118   ins_pipe(pipe_slow); // XXX
 8119 %}
 8120 
 8121 // ============================================================================
 8122 // BSWAP Instructions
 8123 
 8124 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8125   match(Set dst (ReverseBytesI src));
 8126 
 8127   ins_cost(INSN_COST);
 8128   format %{ "revw  $dst, $src" %}
 8129 
 8130   ins_encode %{
 8131     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8132   %}
 8133 
 8134   ins_pipe(ialu_reg);
 8135 %}
 8136 
 8137 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8138   match(Set dst (ReverseBytesL src));
 8139 
 8140   ins_cost(INSN_COST);
 8141   format %{ "rev  $dst, $src" %}
 8142 
 8143   ins_encode %{
 8144     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8145   %}
 8146 
 8147   ins_pipe(ialu_reg);
 8148 %}
 8149 
 8150 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8151   match(Set dst (ReverseBytesUS src));
 8152 
 8153   ins_cost(INSN_COST);
 8154   format %{ "rev16w  $dst, $src" %}
 8155 
 8156   ins_encode %{
 8157     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8158   %}
 8159 
 8160   ins_pipe(ialu_reg);
 8161 %}
 8162 
 8163 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8164   match(Set dst (ReverseBytesS src));
 8165 
 8166   ins_cost(INSN_COST);
 8167   format %{ "rev16w  $dst, $src\n\t"
 8168             "sbfmw $dst, $dst, #0, #15" %}
 8169 
 8170   ins_encode %{
 8171     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8172     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8173   %}
 8174 
 8175   ins_pipe(ialu_reg);
 8176 %}
 8177 
 8178 // ============================================================================
 8179 // Zero Count Instructions
 8180 
 8181 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8182   match(Set dst (CountLeadingZerosI src));
 8183 
 8184   ins_cost(INSN_COST);
 8185   format %{ "clzw  $dst, $src" %}
 8186   ins_encode %{
 8187     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8188   %}
 8189 
 8190   ins_pipe(ialu_reg);
 8191 %}
 8192 
 8193 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8194   match(Set dst (CountLeadingZerosL src));
 8195 
 8196   ins_cost(INSN_COST);
 8197   format %{ "clz   $dst, $src" %}
 8198   ins_encode %{
 8199     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8200   %}
 8201 
 8202   ins_pipe(ialu_reg);
 8203 %}
 8204 
 8205 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8206   match(Set dst (CountTrailingZerosI src));
 8207 
 8208   ins_cost(INSN_COST * 2);
 8209   format %{ "rbitw  $dst, $src\n\t"
 8210             "clzw   $dst, $dst" %}
 8211   ins_encode %{
 8212     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8213     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8214   %}
 8215 
 8216   ins_pipe(ialu_reg);
 8217 %}
 8218 
 8219 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8220   match(Set dst (CountTrailingZerosL src));
 8221 
 8222   ins_cost(INSN_COST * 2);
 8223   format %{ "rbit   $dst, $src\n\t"
 8224             "clz    $dst, $dst" %}
 8225   ins_encode %{
 8226     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8227     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8228   %}
 8229 
 8230   ins_pipe(ialu_reg);
 8231 %}
 8232 
 8233 //---------- Population Count Instructions -------------------------------------
 8234 //
 8235 
 8236 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8237   match(Set dst (PopCountI src));
 8238   effect(TEMP tmp);
 8239   ins_cost(INSN_COST * 13);
 8240 
 8241   format %{ "movw   $src, $src\n\t"
 8242             "mov    $tmp, $src\t# vector (1D)\n\t"
 8243             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8244             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8245             "mov    $dst, $tmp\t# vector (1D)" %}
 8246   ins_encode %{
 8247     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8248     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8249     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8250     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8251     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8252   %}
 8253 
 8254   ins_pipe(pipe_class_default);
 8255 %}
 8256 
 8257 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8258   match(Set dst (PopCountI (LoadI mem)));
 8259   effect(TEMP tmp);
 8260   ins_cost(INSN_COST * 13);
 8261 
 8262   format %{ "ldrs   $tmp, $mem\n\t"
 8263             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8264             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8265             "mov    $dst, $tmp\t# vector (1D)" %}
 8266   ins_encode %{
 8267     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8268     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8269               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8270     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8271     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8272     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8273   %}
 8274 
 8275   ins_pipe(pipe_class_default);
 8276 %}
 8277 
 8278 // Note: Long.bitCount(long) returns an int.
 8279 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8280   match(Set dst (PopCountL src));
 8281   effect(TEMP tmp);
 8282   ins_cost(INSN_COST * 13);
 8283 
 8284   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8285             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8286             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8287             "mov    $dst, $tmp\t# vector (1D)" %}
 8288   ins_encode %{
 8289     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8290     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8291     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8292     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8293   %}
 8294 
 8295   ins_pipe(pipe_class_default);
 8296 %}
 8297 
 8298 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8299   match(Set dst (PopCountL (LoadL mem)));
 8300   effect(TEMP tmp);
 8301   ins_cost(INSN_COST * 13);
 8302 
 8303   format %{ "ldrd   $tmp, $mem\n\t"
 8304             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8305             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8306             "mov    $dst, $tmp\t# vector (1D)" %}
 8307   ins_encode %{
 8308     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8309     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8310               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8311     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8312     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8313     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8314   %}
 8315 
 8316   ins_pipe(pipe_class_default);
 8317 %}
 8318 
 8319 // ============================================================================
 8320 // MemBar Instruction
 8321 
 8322 instruct load_fence() %{
 8323   match(LoadFence);
 8324   ins_cost(VOLATILE_REF_COST);
 8325 
 8326   format %{ "load_fence" %}
 8327 
 8328   ins_encode %{
 8329     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8330   %}
 8331   ins_pipe(pipe_serial);
 8332 %}
 8333 
 8334 instruct unnecessary_membar_acquire() %{
 8335   predicate(unnecessary_acquire(n));
 8336   match(MemBarAcquire);
 8337   ins_cost(0);
 8338 
 8339   format %{ "membar_acquire (elided)" %}
 8340 
 8341   ins_encode %{
 8342     __ block_comment("membar_acquire (elided)");
 8343   %}
 8344 
 8345   ins_pipe(pipe_class_empty);
 8346 %}
 8347 
 8348 instruct membar_acquire() %{
 8349   match(MemBarAcquire);
 8350   ins_cost(VOLATILE_REF_COST);
 8351 
 8352   format %{ "membar_acquire\n\t"
 8353             "dmb ish" %}
 8354 
 8355   ins_encode %{
 8356     __ block_comment("membar_acquire");
 8357     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8358   %}
 8359 
 8360   ins_pipe(pipe_serial);
 8361 %}
 8362 
 8363 
 8364 instruct membar_acquire_lock() %{
 8365   match(MemBarAcquireLock);
 8366   ins_cost(VOLATILE_REF_COST);
 8367 
 8368   format %{ "membar_acquire_lock (elided)" %}
 8369 
 8370   ins_encode %{
 8371     __ block_comment("membar_acquire_lock (elided)");
 8372   %}
 8373 
 8374   ins_pipe(pipe_serial);
 8375 %}
 8376 
 8377 instruct store_fence() %{
 8378   match(StoreFence);
 8379   ins_cost(VOLATILE_REF_COST);
 8380 
 8381   format %{ "store_fence" %}
 8382 
 8383   ins_encode %{
 8384     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8385   %}
 8386   ins_pipe(pipe_serial);
 8387 %}
 8388 
 8389 instruct unnecessary_membar_release() %{
 8390   predicate(unnecessary_release(n));
 8391   match(MemBarRelease);
 8392   ins_cost(0);
 8393 
 8394   format %{ "membar_release (elided)" %}
 8395 
 8396   ins_encode %{
 8397     __ block_comment("membar_release (elided)");
 8398   %}
 8399   ins_pipe(pipe_serial);
 8400 %}
 8401 
 8402 instruct membar_release() %{
 8403   match(MemBarRelease);
 8404   ins_cost(VOLATILE_REF_COST);
 8405 
 8406   format %{ "membar_release\n\t"
 8407             "dmb ish" %}
 8408 
 8409   ins_encode %{
 8410     __ block_comment("membar_release");
 8411     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8412   %}
 8413   ins_pipe(pipe_serial);
 8414 %}
 8415 
 8416 instruct membar_storestore() %{
 8417   match(MemBarStoreStore);
 8418   match(StoreStoreFence);
 8419   ins_cost(VOLATILE_REF_COST);
 8420 
 8421   format %{ "MEMBAR-store-store" %}
 8422 
 8423   ins_encode %{
 8424     __ membar(Assembler::StoreStore);
 8425   %}
 8426   ins_pipe(pipe_serial);
 8427 %}
 8428 
 8429 instruct membar_release_lock() %{
 8430   match(MemBarReleaseLock);
 8431   ins_cost(VOLATILE_REF_COST);
 8432 
 8433   format %{ "membar_release_lock (elided)" %}
 8434 
 8435   ins_encode %{
 8436     __ block_comment("membar_release_lock (elided)");
 8437   %}
 8438 
 8439   ins_pipe(pipe_serial);
 8440 %}
 8441 
 8442 instruct unnecessary_membar_volatile() %{
 8443   predicate(unnecessary_volatile(n));
 8444   match(MemBarVolatile);
 8445   ins_cost(0);
 8446 
 8447   format %{ "membar_volatile (elided)" %}
 8448 
 8449   ins_encode %{
 8450     __ block_comment("membar_volatile (elided)");
 8451   %}
 8452 
 8453   ins_pipe(pipe_serial);
 8454 %}
 8455 
 8456 instruct membar_volatile() %{
 8457   match(MemBarVolatile);
 8458   ins_cost(VOLATILE_REF_COST*100);
 8459 
 8460   format %{ "membar_volatile\n\t"
 8461              "dmb ish"%}
 8462 
 8463   ins_encode %{
 8464     __ block_comment("membar_volatile");
 8465     __ membar(Assembler::StoreLoad);
 8466   %}
 8467 
 8468   ins_pipe(pipe_serial);
 8469 %}
 8470 
 8471 // ============================================================================
 8472 // Cast/Convert Instructions
 8473 
 8474 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8475   match(Set dst (CastX2P src));
 8476 
 8477   ins_cost(INSN_COST);
 8478   format %{ "mov $dst, $src\t# long -> ptr" %}
 8479 
 8480   ins_encode %{
 8481     if ($dst$$reg != $src$$reg) {
 8482       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8483     }
 8484   %}
 8485 
 8486   ins_pipe(ialu_reg);
 8487 %}
 8488 
 8489 instruct castN2X(iRegLNoSp dst, iRegN src) %{
 8490   match(Set dst (CastP2X src));
 8491 
 8492   ins_cost(INSN_COST);
 8493   format %{ "mov $dst, $src\t# ptr -> long" %}
 8494 
 8495   ins_encode %{
 8496     if ($dst$$reg != $src$$reg) {
 8497       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8498     }
 8499   %}
 8500 
 8501   ins_pipe(ialu_reg);
 8502 %}
 8503 
 8504 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8505   match(Set dst (CastP2X src));
 8506 
 8507   ins_cost(INSN_COST);
 8508   format %{ "mov $dst, $src\t# ptr -> long" %}
 8509 
 8510   ins_encode %{
 8511     if ($dst$$reg != $src$$reg) {
 8512       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8513     }
 8514   %}
 8515 
 8516   ins_pipe(ialu_reg);
 8517 %}
 8518 
 8519 // Convert oop into int for vectors alignment masking
 8520 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8521   match(Set dst (ConvL2I (CastP2X src)));
 8522 
 8523   ins_cost(INSN_COST);
 8524   format %{ "movw $dst, $src\t# ptr -> int" %}
 8525   ins_encode %{
 8526     __ movw($dst$$Register, $src$$Register);
 8527   %}
 8528 
 8529   ins_pipe(ialu_reg);
 8530 %}
 8531 
 8532 // Convert compressed oop into int for vectors alignment masking
 8533 // in case of 32bit oops (heap < 4Gb).
 8534 instruct convN2I(iRegINoSp dst, iRegN src)
 8535 %{
 8536   predicate(CompressedOops::shift() == 0);
 8537   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8538 
 8539   ins_cost(INSN_COST);
 8540   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8541   ins_encode %{
 8542     __ movw($dst$$Register, $src$$Register);
 8543   %}
 8544 
 8545   ins_pipe(ialu_reg);
 8546 %}
 8547 
 8548 
 8549 // Convert oop pointer into compressed form
 8550 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8551   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8552   match(Set dst (EncodeP src));
 8553   effect(KILL cr);
 8554   ins_cost(INSN_COST * 3);
 8555   format %{ "encode_heap_oop $dst, $src" %}
 8556   ins_encode %{
 8557     Register s = $src$$Register;
 8558     Register d = $dst$$Register;
 8559     __ encode_heap_oop(d, s);
 8560   %}
 8561   ins_pipe(ialu_reg);
 8562 %}
 8563 
 8564 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8565   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8566   match(Set dst (EncodeP src));
 8567   ins_cost(INSN_COST * 3);
 8568   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8569   ins_encode %{
 8570     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8571   %}
 8572   ins_pipe(ialu_reg);
 8573 %}
 8574 
 8575 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8576   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8577             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8578   match(Set dst (DecodeN src));
 8579   ins_cost(INSN_COST * 3);
 8580   format %{ "decode_heap_oop $dst, $src" %}
 8581   ins_encode %{
 8582     Register s = $src$$Register;
 8583     Register d = $dst$$Register;
 8584     __ decode_heap_oop(d, s);
 8585   %}
 8586   ins_pipe(ialu_reg);
 8587 %}
 8588 
 8589 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8590   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8591             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8592   match(Set dst (DecodeN src));
 8593   ins_cost(INSN_COST * 3);
 8594   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8595   ins_encode %{
 8596     Register s = $src$$Register;
 8597     Register d = $dst$$Register;
 8598     __ decode_heap_oop_not_null(d, s);
 8599   %}
 8600   ins_pipe(ialu_reg);
 8601 %}
 8602 
 8603 // n.b. AArch64 implementations of encode_klass_not_null and
 8604 // decode_klass_not_null do not modify the flags register so, unlike
 8605 // Intel, we don't kill CR as a side effect here
 8606 
 8607 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8608   match(Set dst (EncodePKlass src));
 8609 
 8610   ins_cost(INSN_COST * 3);
 8611   format %{ "encode_klass_not_null $dst,$src" %}
 8612 
 8613   ins_encode %{
 8614     Register src_reg = as_Register($src$$reg);
 8615     Register dst_reg = as_Register($dst$$reg);
 8616     __ encode_klass_not_null(dst_reg, src_reg);
 8617   %}
 8618 
 8619    ins_pipe(ialu_reg);
 8620 %}
 8621 
 8622 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8623   match(Set dst (DecodeNKlass src));
 8624 
 8625   ins_cost(INSN_COST * 3);
 8626   format %{ "decode_klass_not_null $dst,$src" %}
 8627 
 8628   ins_encode %{
 8629     Register src_reg = as_Register($src$$reg);
 8630     Register dst_reg = as_Register($dst$$reg);
 8631     if (dst_reg != src_reg) {
 8632       __ decode_klass_not_null(dst_reg, src_reg);
 8633     } else {
 8634       __ decode_klass_not_null(dst_reg);
 8635     }
 8636   %}
 8637 
 8638    ins_pipe(ialu_reg);
 8639 %}
 8640 
 8641 instruct checkCastPP(iRegPNoSp dst)
 8642 %{
 8643   match(Set dst (CheckCastPP dst));
 8644 
 8645   size(0);
 8646   format %{ "# checkcastPP of $dst" %}
 8647   ins_encode(/* empty encoding */);
 8648   ins_pipe(pipe_class_empty);
 8649 %}
 8650 
 8651 instruct castPP(iRegPNoSp dst)
 8652 %{
 8653   match(Set dst (CastPP dst));
 8654 
 8655   size(0);
 8656   format %{ "# castPP of $dst" %}
 8657   ins_encode(/* empty encoding */);
 8658   ins_pipe(pipe_class_empty);
 8659 %}
 8660 
 8661 instruct castII(iRegI dst)
 8662 %{
 8663   match(Set dst (CastII dst));
 8664 
 8665   size(0);
 8666   format %{ "# castII of $dst" %}
 8667   ins_encode(/* empty encoding */);
 8668   ins_cost(0);
 8669   ins_pipe(pipe_class_empty);
 8670 %}
 8671 
 8672 instruct castLL(iRegL dst)
 8673 %{
 8674   match(Set dst (CastLL dst));
 8675 
 8676   size(0);
 8677   format %{ "# castLL of $dst" %}
 8678   ins_encode(/* empty encoding */);
 8679   ins_cost(0);
 8680   ins_pipe(pipe_class_empty);
 8681 %}
 8682 
 8683 instruct castFF(vRegF dst)
 8684 %{
 8685   match(Set dst (CastFF dst));
 8686 
 8687   size(0);
 8688   format %{ "# castFF of $dst" %}
 8689   ins_encode(/* empty encoding */);
 8690   ins_cost(0);
 8691   ins_pipe(pipe_class_empty);
 8692 %}
 8693 
 8694 instruct castDD(vRegD dst)
 8695 %{
 8696   match(Set dst (CastDD dst));
 8697 
 8698   size(0);
 8699   format %{ "# castDD of $dst" %}
 8700   ins_encode(/* empty encoding */);
 8701   ins_cost(0);
 8702   ins_pipe(pipe_class_empty);
 8703 %}
 8704 
 8705 instruct castVV(vReg dst)
 8706 %{
 8707   match(Set dst (CastVV dst));
 8708 
 8709   size(0);
 8710   format %{ "# castVV of $dst" %}
 8711   ins_encode(/* empty encoding */);
 8712   ins_cost(0);
 8713   ins_pipe(pipe_class_empty);
 8714 %}
 8715 
 8716 instruct castVVMask(pRegGov dst)
 8717 %{
 8718   match(Set dst (CastVV dst));
 8719 
 8720   size(0);
 8721   format %{ "# castVV of $dst" %}
 8722   ins_encode(/* empty encoding */);
 8723   ins_cost(0);
 8724   ins_pipe(pipe_class_empty);
 8725 %}
 8726 
 8727 // ============================================================================
 8728 // Atomic operation instructions
 8729 //
 8730 
 8731 // standard CompareAndSwapX when we are using barriers
 8732 // these have higher priority than the rules selected by a predicate
 8733 
 8734 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 8735 // can't match them
 8736 
 8737 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8738 
 8739   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8740   ins_cost(2 * VOLATILE_REF_COST);
 8741 
 8742   effect(KILL cr);
 8743 
 8744   format %{
 8745     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8746     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8747   %}
 8748 
 8749   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 8750             aarch64_enc_cset_eq(res));
 8751 
 8752   ins_pipe(pipe_slow);
 8753 %}
 8754 
 8755 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8756 
 8757   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8758   ins_cost(2 * VOLATILE_REF_COST);
 8759 
 8760   effect(KILL cr);
 8761 
 8762   format %{
 8763     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8764     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8765   %}
 8766 
 8767   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 8768             aarch64_enc_cset_eq(res));
 8769 
 8770   ins_pipe(pipe_slow);
 8771 %}
 8772 
 8773 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8774 
 8775   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8776   ins_cost(2 * VOLATILE_REF_COST);
 8777 
 8778   effect(KILL cr);
 8779 
 8780  format %{
 8781     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8782     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8783  %}
 8784 
 8785  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8786             aarch64_enc_cset_eq(res));
 8787 
 8788   ins_pipe(pipe_slow);
 8789 %}
 8790 
 8791 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8792 
 8793   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8794   ins_cost(2 * VOLATILE_REF_COST);
 8795 
 8796   effect(KILL cr);
 8797 
 8798  format %{
 8799     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8800     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8801  %}
 8802 
 8803  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8804             aarch64_enc_cset_eq(res));
 8805 
 8806   ins_pipe(pipe_slow);
 8807 %}
 8808 
 8809 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8810 
 8811   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8812   predicate(n->as_LoadStore()->barrier_data() == 0);
 8813   ins_cost(2 * VOLATILE_REF_COST);
 8814 
 8815   effect(KILL cr);
 8816 
 8817  format %{
 8818     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8819     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8820  %}
 8821 
 8822  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8823             aarch64_enc_cset_eq(res));
 8824 
 8825   ins_pipe(pipe_slow);
 8826 %}
 8827 
 8828 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8829 
 8830   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8831   ins_cost(2 * VOLATILE_REF_COST);
 8832 
 8833   effect(KILL cr);
 8834 
 8835  format %{
 8836     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8837     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8838  %}
 8839 
 8840  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8841             aarch64_enc_cset_eq(res));
 8842 
 8843   ins_pipe(pipe_slow);
 8844 %}
 8845 
 8846 // alternative CompareAndSwapX when we are eliding barriers
 8847 
 8848 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8849 
 8850   predicate(needs_acquiring_load_exclusive(n));
 8851   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8852   ins_cost(VOLATILE_REF_COST);
 8853 
 8854   effect(KILL cr);
 8855 
 8856   format %{
 8857     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8858     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8859   %}
 8860 
 8861   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 8862             aarch64_enc_cset_eq(res));
 8863 
 8864   ins_pipe(pipe_slow);
 8865 %}
 8866 
 8867 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8868 
 8869   predicate(needs_acquiring_load_exclusive(n));
 8870   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8871   ins_cost(VOLATILE_REF_COST);
 8872 
 8873   effect(KILL cr);
 8874 
 8875   format %{
 8876     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8877     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8878   %}
 8879 
 8880   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 8881             aarch64_enc_cset_eq(res));
 8882 
 8883   ins_pipe(pipe_slow);
 8884 %}
 8885 
 8886 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8887 
 8888   predicate(needs_acquiring_load_exclusive(n));
 8889   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8890   ins_cost(VOLATILE_REF_COST);
 8891 
 8892   effect(KILL cr);
 8893 
 8894  format %{
 8895     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8896     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8897  %}
 8898 
 8899  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8900             aarch64_enc_cset_eq(res));
 8901 
 8902   ins_pipe(pipe_slow);
 8903 %}
 8904 
 8905 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8906 
 8907   predicate(needs_acquiring_load_exclusive(n));
 8908   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8909   ins_cost(VOLATILE_REF_COST);
 8910 
 8911   effect(KILL cr);
 8912 
 8913  format %{
 8914     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8915     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8916  %}
 8917 
 8918  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8919             aarch64_enc_cset_eq(res));
 8920 
 8921   ins_pipe(pipe_slow);
 8922 %}
 8923 
 8924 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8925 
 8926   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8927   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8928   ins_cost(VOLATILE_REF_COST);
 8929 
 8930   effect(KILL cr);
 8931 
 8932  format %{
 8933     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8934     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8935  %}
 8936 
 8937  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8938             aarch64_enc_cset_eq(res));
 8939 
 8940   ins_pipe(pipe_slow);
 8941 %}
 8942 
 8943 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8944 
 8945   predicate(needs_acquiring_load_exclusive(n));
 8946   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8947   ins_cost(VOLATILE_REF_COST);
 8948 
 8949   effect(KILL cr);
 8950 
 8951  format %{
 8952     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8953     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8954  %}
 8955 
 8956  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8957             aarch64_enc_cset_eq(res));
 8958 
 8959   ins_pipe(pipe_slow);
 8960 %}
 8961 
 8962 
 8963 // ---------------------------------------------------------------------
 8964 
 8965 // BEGIN This section of the file is automatically generated. Do not edit --------------
 8966 
 8967 // Sundry CAS operations.  Note that release is always true,
 8968 // regardless of the memory ordering of the CAS.  This is because we
 8969 // need the volatile case to be sequentially consistent but there is
 8970 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 8971 // can't check the type of memory ordering here, so we always emit a
 8972 // STLXR.
 8973 
 8974 // This section is generated from aarch64_ad_cas.m4
 8975 
 8976 
 8977 
 8978 // This pattern is generated automatically from cas.m4.
 8979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8980 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8981 
 8982   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8983   ins_cost(2 * VOLATILE_REF_COST);
 8984   effect(TEMP_DEF res, KILL cr);
 8985   format %{
 8986     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8987   %}
 8988   ins_encode %{
 8989     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8990                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8991                /*weak*/ false, $res$$Register);
 8992     __ sxtbw($res$$Register, $res$$Register);
 8993   %}
 8994   ins_pipe(pipe_slow);
 8995 %}
 8996 
 8997 // This pattern is generated automatically from cas.m4.
 8998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8999 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9000 
 9001   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9002   ins_cost(2 * VOLATILE_REF_COST);
 9003   effect(TEMP_DEF res, KILL cr);
 9004   format %{
 9005     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9006   %}
 9007   ins_encode %{
 9008     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9009                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9010                /*weak*/ false, $res$$Register);
 9011     __ sxthw($res$$Register, $res$$Register);
 9012   %}
 9013   ins_pipe(pipe_slow);
 9014 %}
 9015 
 9016 // This pattern is generated automatically from cas.m4.
 9017 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9018 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9019 
 9020   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9021   ins_cost(2 * VOLATILE_REF_COST);
 9022   effect(TEMP_DEF res, KILL cr);
 9023   format %{
 9024     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9025   %}
 9026   ins_encode %{
 9027     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9028                Assembler::word, /*acquire*/ false, /*release*/ true,
 9029                /*weak*/ false, $res$$Register);
 9030   %}
 9031   ins_pipe(pipe_slow);
 9032 %}
 9033 
 9034 // This pattern is generated automatically from cas.m4.
 9035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9036 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9037 
 9038   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9039   ins_cost(2 * VOLATILE_REF_COST);
 9040   effect(TEMP_DEF res, KILL cr);
 9041   format %{
 9042     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9043   %}
 9044   ins_encode %{
 9045     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9046                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9047                /*weak*/ false, $res$$Register);
 9048   %}
 9049   ins_pipe(pipe_slow);
 9050 %}
 9051 
 9052 // This pattern is generated automatically from cas.m4.
 9053 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9054 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9055 
 9056   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9057   ins_cost(2 * VOLATILE_REF_COST);
 9058   effect(TEMP_DEF res, KILL cr);
 9059   format %{
 9060     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9061   %}
 9062   ins_encode %{
 9063     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9064                Assembler::word, /*acquire*/ false, /*release*/ true,
 9065                /*weak*/ false, $res$$Register);
 9066   %}
 9067   ins_pipe(pipe_slow);
 9068 %}
 9069 
 9070 // This pattern is generated automatically from cas.m4.
 9071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9072 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9073   predicate(n->as_LoadStore()->barrier_data() == 0);
 9074   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9075   ins_cost(2 * VOLATILE_REF_COST);
 9076   effect(TEMP_DEF res, KILL cr);
 9077   format %{
 9078     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9079   %}
 9080   ins_encode %{
 9081     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9082                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9083                /*weak*/ false, $res$$Register);
 9084   %}
 9085   ins_pipe(pipe_slow);
 9086 %}
 9087 
 9088 // This pattern is generated automatically from cas.m4.
 9089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9090 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9091   predicate(needs_acquiring_load_exclusive(n));
 9092   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9093   ins_cost(VOLATILE_REF_COST);
 9094   effect(TEMP_DEF res, KILL cr);
 9095   format %{
 9096     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9097   %}
 9098   ins_encode %{
 9099     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9100                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9101                /*weak*/ false, $res$$Register);
 9102     __ sxtbw($res$$Register, $res$$Register);
 9103   %}
 9104   ins_pipe(pipe_slow);
 9105 %}
 9106 
 9107 // This pattern is generated automatically from cas.m4.
 9108 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9109 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9110   predicate(needs_acquiring_load_exclusive(n));
 9111   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9112   ins_cost(VOLATILE_REF_COST);
 9113   effect(TEMP_DEF res, KILL cr);
 9114   format %{
 9115     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9116   %}
 9117   ins_encode %{
 9118     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9119                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9120                /*weak*/ false, $res$$Register);
 9121     __ sxthw($res$$Register, $res$$Register);
 9122   %}
 9123   ins_pipe(pipe_slow);
 9124 %}
 9125 
 9126 // This pattern is generated automatically from cas.m4.
 9127 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9128 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9129   predicate(needs_acquiring_load_exclusive(n));
 9130   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9131   ins_cost(VOLATILE_REF_COST);
 9132   effect(TEMP_DEF res, KILL cr);
 9133   format %{
 9134     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9135   %}
 9136   ins_encode %{
 9137     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9138                Assembler::word, /*acquire*/ true, /*release*/ true,
 9139                /*weak*/ false, $res$$Register);
 9140   %}
 9141   ins_pipe(pipe_slow);
 9142 %}
 9143 
 9144 // This pattern is generated automatically from cas.m4.
 9145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9146 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9147   predicate(needs_acquiring_load_exclusive(n));
 9148   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9149   ins_cost(VOLATILE_REF_COST);
 9150   effect(TEMP_DEF res, KILL cr);
 9151   format %{
 9152     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9153   %}
 9154   ins_encode %{
 9155     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9156                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9157                /*weak*/ false, $res$$Register);
 9158   %}
 9159   ins_pipe(pipe_slow);
 9160 %}
 9161 
 9162 // This pattern is generated automatically from cas.m4.
 9163 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9164 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9165   predicate(needs_acquiring_load_exclusive(n));
 9166   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9167   ins_cost(VOLATILE_REF_COST);
 9168   effect(TEMP_DEF res, KILL cr);
 9169   format %{
 9170     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9171   %}
 9172   ins_encode %{
 9173     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9174                Assembler::word, /*acquire*/ true, /*release*/ true,
 9175                /*weak*/ false, $res$$Register);
 9176   %}
 9177   ins_pipe(pipe_slow);
 9178 %}
 9179 
 9180 // This pattern is generated automatically from cas.m4.
 9181 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9182 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9183   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9184   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9185   ins_cost(VOLATILE_REF_COST);
 9186   effect(TEMP_DEF res, KILL cr);
 9187   format %{
 9188     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9189   %}
 9190   ins_encode %{
 9191     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9192                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9193                /*weak*/ false, $res$$Register);
 9194   %}
 9195   ins_pipe(pipe_slow);
 9196 %}
 9197 
 9198 // This pattern is generated automatically from cas.m4.
 9199 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9200 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9201 
 9202   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9203   ins_cost(2 * VOLATILE_REF_COST);
 9204   effect(KILL cr);
 9205   format %{
 9206     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9207     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9208   %}
 9209   ins_encode %{
 9210     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9211                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9212                /*weak*/ true, noreg);
 9213     __ csetw($res$$Register, Assembler::EQ);
 9214   %}
 9215   ins_pipe(pipe_slow);
 9216 %}
 9217 
 9218 // This pattern is generated automatically from cas.m4.
 9219 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9220 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9221 
 9222   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9223   ins_cost(2 * VOLATILE_REF_COST);
 9224   effect(KILL cr);
 9225   format %{
 9226     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9227     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9228   %}
 9229   ins_encode %{
 9230     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9231                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9232                /*weak*/ true, noreg);
 9233     __ csetw($res$$Register, Assembler::EQ);
 9234   %}
 9235   ins_pipe(pipe_slow);
 9236 %}
 9237 
 9238 // This pattern is generated automatically from cas.m4.
 9239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9240 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9241 
 9242   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9243   ins_cost(2 * VOLATILE_REF_COST);
 9244   effect(KILL cr);
 9245   format %{
 9246     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9247     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9248   %}
 9249   ins_encode %{
 9250     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9251                Assembler::word, /*acquire*/ false, /*release*/ true,
 9252                /*weak*/ true, noreg);
 9253     __ csetw($res$$Register, Assembler::EQ);
 9254   %}
 9255   ins_pipe(pipe_slow);
 9256 %}
 9257 
 9258 // This pattern is generated automatically from cas.m4.
 9259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9260 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9261 
 9262   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9263   ins_cost(2 * VOLATILE_REF_COST);
 9264   effect(KILL cr);
 9265   format %{
 9266     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9267     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9268   %}
 9269   ins_encode %{
 9270     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9271                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9272                /*weak*/ true, noreg);
 9273     __ csetw($res$$Register, Assembler::EQ);
 9274   %}
 9275   ins_pipe(pipe_slow);
 9276 %}
 9277 
 9278 // This pattern is generated automatically from cas.m4.
 9279 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9280 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9281 
 9282   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9283   ins_cost(2 * VOLATILE_REF_COST);
 9284   effect(KILL cr);
 9285   format %{
 9286     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9287     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9288   %}
 9289   ins_encode %{
 9290     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9291                Assembler::word, /*acquire*/ false, /*release*/ true,
 9292                /*weak*/ true, noreg);
 9293     __ csetw($res$$Register, Assembler::EQ);
 9294   %}
 9295   ins_pipe(pipe_slow);
 9296 %}
 9297 
 9298 // This pattern is generated automatically from cas.m4.
 9299 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9300 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9301   predicate(n->as_LoadStore()->barrier_data() == 0);
 9302   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9303   ins_cost(2 * VOLATILE_REF_COST);
 9304   effect(KILL cr);
 9305   format %{
 9306     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9307     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9308   %}
 9309   ins_encode %{
 9310     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9311                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9312                /*weak*/ true, noreg);
 9313     __ csetw($res$$Register, Assembler::EQ);
 9314   %}
 9315   ins_pipe(pipe_slow);
 9316 %}
 9317 
 9318 // This pattern is generated automatically from cas.m4.
 9319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9320 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9321   predicate(needs_acquiring_load_exclusive(n));
 9322   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9323   ins_cost(VOLATILE_REF_COST);
 9324   effect(KILL cr);
 9325   format %{
 9326     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9327     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9328   %}
 9329   ins_encode %{
 9330     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9331                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9332                /*weak*/ true, noreg);
 9333     __ csetw($res$$Register, Assembler::EQ);
 9334   %}
 9335   ins_pipe(pipe_slow);
 9336 %}
 9337 
 9338 // This pattern is generated automatically from cas.m4.
 9339 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9340 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9341   predicate(needs_acquiring_load_exclusive(n));
 9342   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9343   ins_cost(VOLATILE_REF_COST);
 9344   effect(KILL cr);
 9345   format %{
 9346     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9347     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9348   %}
 9349   ins_encode %{
 9350     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9351                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9352                /*weak*/ true, noreg);
 9353     __ csetw($res$$Register, Assembler::EQ);
 9354   %}
 9355   ins_pipe(pipe_slow);
 9356 %}
 9357 
 9358 // This pattern is generated automatically from cas.m4.
 9359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9360 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9361   predicate(needs_acquiring_load_exclusive(n));
 9362   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9363   ins_cost(VOLATILE_REF_COST);
 9364   effect(KILL cr);
 9365   format %{
 9366     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9367     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9368   %}
 9369   ins_encode %{
 9370     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9371                Assembler::word, /*acquire*/ true, /*release*/ true,
 9372                /*weak*/ true, noreg);
 9373     __ csetw($res$$Register, Assembler::EQ);
 9374   %}
 9375   ins_pipe(pipe_slow);
 9376 %}
 9377 
 9378 // This pattern is generated automatically from cas.m4.
 9379 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9380 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9381   predicate(needs_acquiring_load_exclusive(n));
 9382   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9383   ins_cost(VOLATILE_REF_COST);
 9384   effect(KILL cr);
 9385   format %{
 9386     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9387     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9388   %}
 9389   ins_encode %{
 9390     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9391                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9392                /*weak*/ true, noreg);
 9393     __ csetw($res$$Register, Assembler::EQ);
 9394   %}
 9395   ins_pipe(pipe_slow);
 9396 %}
 9397 
 9398 // This pattern is generated automatically from cas.m4.
 9399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9400 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9401   predicate(needs_acquiring_load_exclusive(n));
 9402   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9403   ins_cost(VOLATILE_REF_COST);
 9404   effect(KILL cr);
 9405   format %{
 9406     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9407     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9408   %}
 9409   ins_encode %{
 9410     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9411                Assembler::word, /*acquire*/ true, /*release*/ true,
 9412                /*weak*/ true, noreg);
 9413     __ csetw($res$$Register, Assembler::EQ);
 9414   %}
 9415   ins_pipe(pipe_slow);
 9416 %}
 9417 
 9418 // This pattern is generated automatically from cas.m4.
 9419 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9420 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9421   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9422   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9423   ins_cost(VOLATILE_REF_COST);
 9424   effect(KILL cr);
 9425   format %{
 9426     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9427     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9428   %}
 9429   ins_encode %{
 9430     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9431                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9432                /*weak*/ true, noreg);
 9433     __ csetw($res$$Register, Assembler::EQ);
 9434   %}
 9435   ins_pipe(pipe_slow);
 9436 %}
 9437 
 9438 // END This section of the file is automatically generated. Do not edit --------------
 9439 // ---------------------------------------------------------------------
 9440 
 9441 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9442   match(Set prev (GetAndSetI mem newv));
 9443   ins_cost(2 * VOLATILE_REF_COST);
 9444   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9445   ins_encode %{
 9446     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9447   %}
 9448   ins_pipe(pipe_serial);
 9449 %}
 9450 
 9451 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9452   match(Set prev (GetAndSetL mem newv));
 9453   ins_cost(2 * VOLATILE_REF_COST);
 9454   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9455   ins_encode %{
 9456     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9457   %}
 9458   ins_pipe(pipe_serial);
 9459 %}
 9460 
 9461 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9462   match(Set prev (GetAndSetN mem newv));
 9463   ins_cost(2 * VOLATILE_REF_COST);
 9464   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9465   ins_encode %{
 9466     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9467   %}
 9468   ins_pipe(pipe_serial);
 9469 %}
 9470 
 9471 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9472   predicate(n->as_LoadStore()->barrier_data() == 0);
 9473   match(Set prev (GetAndSetP mem newv));
 9474   ins_cost(2 * VOLATILE_REF_COST);
 9475   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9476   ins_encode %{
 9477     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9478   %}
 9479   ins_pipe(pipe_serial);
 9480 %}
 9481 
 9482 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9483   predicate(needs_acquiring_load_exclusive(n));
 9484   match(Set prev (GetAndSetI mem newv));
 9485   ins_cost(VOLATILE_REF_COST);
 9486   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9487   ins_encode %{
 9488     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9489   %}
 9490   ins_pipe(pipe_serial);
 9491 %}
 9492 
 9493 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9494   predicate(needs_acquiring_load_exclusive(n));
 9495   match(Set prev (GetAndSetL mem newv));
 9496   ins_cost(VOLATILE_REF_COST);
 9497   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9498   ins_encode %{
 9499     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9500   %}
 9501   ins_pipe(pipe_serial);
 9502 %}
 9503 
 9504 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9505   predicate(needs_acquiring_load_exclusive(n));
 9506   match(Set prev (GetAndSetN mem newv));
 9507   ins_cost(VOLATILE_REF_COST);
 9508   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9509   ins_encode %{
 9510     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9511   %}
 9512   ins_pipe(pipe_serial);
 9513 %}
 9514 
 9515 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9516   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9517   match(Set prev (GetAndSetP mem newv));
 9518   ins_cost(VOLATILE_REF_COST);
 9519   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9520   ins_encode %{
 9521     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9522   %}
 9523   ins_pipe(pipe_serial);
 9524 %}
 9525 
 9526 
 9527 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9528   match(Set newval (GetAndAddL mem incr));
 9529   ins_cost(2 * VOLATILE_REF_COST + 1);
 9530   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9531   ins_encode %{
 9532     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9533   %}
 9534   ins_pipe(pipe_serial);
 9535 %}
 9536 
 9537 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9538   predicate(n->as_LoadStore()->result_not_used());
 9539   match(Set dummy (GetAndAddL mem incr));
 9540   ins_cost(2 * VOLATILE_REF_COST);
 9541   format %{ "get_and_addL [$mem], $incr" %}
 9542   ins_encode %{
 9543     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9544   %}
 9545   ins_pipe(pipe_serial);
 9546 %}
 9547 
 9548 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9549   match(Set newval (GetAndAddL mem incr));
 9550   ins_cost(2 * VOLATILE_REF_COST + 1);
 9551   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9552   ins_encode %{
 9553     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9554   %}
 9555   ins_pipe(pipe_serial);
 9556 %}
 9557 
 9558 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9559   predicate(n->as_LoadStore()->result_not_used());
 9560   match(Set dummy (GetAndAddL mem incr));
 9561   ins_cost(2 * VOLATILE_REF_COST);
 9562   format %{ "get_and_addL [$mem], $incr" %}
 9563   ins_encode %{
 9564     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9565   %}
 9566   ins_pipe(pipe_serial);
 9567 %}
 9568 
 9569 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9570   match(Set newval (GetAndAddI mem incr));
 9571   ins_cost(2 * VOLATILE_REF_COST + 1);
 9572   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9573   ins_encode %{
 9574     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9575   %}
 9576   ins_pipe(pipe_serial);
 9577 %}
 9578 
 9579 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9580   predicate(n->as_LoadStore()->result_not_used());
 9581   match(Set dummy (GetAndAddI mem incr));
 9582   ins_cost(2 * VOLATILE_REF_COST);
 9583   format %{ "get_and_addI [$mem], $incr" %}
 9584   ins_encode %{
 9585     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9586   %}
 9587   ins_pipe(pipe_serial);
 9588 %}
 9589 
 9590 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9591   match(Set newval (GetAndAddI mem incr));
 9592   ins_cost(2 * VOLATILE_REF_COST + 1);
 9593   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9594   ins_encode %{
 9595     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9596   %}
 9597   ins_pipe(pipe_serial);
 9598 %}
 9599 
 9600 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9601   predicate(n->as_LoadStore()->result_not_used());
 9602   match(Set dummy (GetAndAddI mem incr));
 9603   ins_cost(2 * VOLATILE_REF_COST);
 9604   format %{ "get_and_addI [$mem], $incr" %}
 9605   ins_encode %{
 9606     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9607   %}
 9608   ins_pipe(pipe_serial);
 9609 %}
 9610 
 9611 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9612   predicate(needs_acquiring_load_exclusive(n));
 9613   match(Set newval (GetAndAddL mem incr));
 9614   ins_cost(VOLATILE_REF_COST + 1);
 9615   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9616   ins_encode %{
 9617     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9618   %}
 9619   ins_pipe(pipe_serial);
 9620 %}
 9621 
 9622 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9623   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9624   match(Set dummy (GetAndAddL mem incr));
 9625   ins_cost(VOLATILE_REF_COST);
 9626   format %{ "get_and_addL_acq [$mem], $incr" %}
 9627   ins_encode %{
 9628     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9629   %}
 9630   ins_pipe(pipe_serial);
 9631 %}
 9632 
 9633 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9634   predicate(needs_acquiring_load_exclusive(n));
 9635   match(Set newval (GetAndAddL mem incr));
 9636   ins_cost(VOLATILE_REF_COST + 1);
 9637   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9638   ins_encode %{
 9639     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9640   %}
 9641   ins_pipe(pipe_serial);
 9642 %}
 9643 
 9644 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9645   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9646   match(Set dummy (GetAndAddL mem incr));
 9647   ins_cost(VOLATILE_REF_COST);
 9648   format %{ "get_and_addL_acq [$mem], $incr" %}
 9649   ins_encode %{
 9650     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9651   %}
 9652   ins_pipe(pipe_serial);
 9653 %}
 9654 
 9655 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9656   predicate(needs_acquiring_load_exclusive(n));
 9657   match(Set newval (GetAndAddI mem incr));
 9658   ins_cost(VOLATILE_REF_COST + 1);
 9659   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9660   ins_encode %{
 9661     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9662   %}
 9663   ins_pipe(pipe_serial);
 9664 %}
 9665 
 9666 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9667   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9668   match(Set dummy (GetAndAddI mem incr));
 9669   ins_cost(VOLATILE_REF_COST);
 9670   format %{ "get_and_addI_acq [$mem], $incr" %}
 9671   ins_encode %{
 9672     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9673   %}
 9674   ins_pipe(pipe_serial);
 9675 %}
 9676 
 9677 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9678   predicate(needs_acquiring_load_exclusive(n));
 9679   match(Set newval (GetAndAddI mem incr));
 9680   ins_cost(VOLATILE_REF_COST + 1);
 9681   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9682   ins_encode %{
 9683     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9684   %}
 9685   ins_pipe(pipe_serial);
 9686 %}
 9687 
 9688 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9689   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9690   match(Set dummy (GetAndAddI mem incr));
 9691   ins_cost(VOLATILE_REF_COST);
 9692   format %{ "get_and_addI_acq [$mem], $incr" %}
 9693   ins_encode %{
 9694     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9695   %}
 9696   ins_pipe(pipe_serial);
 9697 %}
 9698 
 9699 // Manifest a CmpL result in an integer register.
 9700 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9701 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9702 %{
 9703   match(Set dst (CmpL3 src1 src2));
 9704   effect(KILL flags);
 9705 
 9706   ins_cost(INSN_COST * 6);
 9707   format %{
 9708       "cmp $src1, $src2"
 9709       "csetw $dst, ne"
 9710       "cnegw $dst, lt"
 9711   %}
 9712   // format %{ "CmpL3 $dst, $src1, $src2" %}
 9713   ins_encode %{
 9714     __ cmp($src1$$Register, $src2$$Register);
 9715     __ csetw($dst$$Register, Assembler::NE);
 9716     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9717   %}
 9718 
 9719   ins_pipe(pipe_class_default);
 9720 %}
 9721 
 9722 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9723 %{
 9724   match(Set dst (CmpL3 src1 src2));
 9725   effect(KILL flags);
 9726 
 9727   ins_cost(INSN_COST * 6);
 9728   format %{
 9729       "cmp $src1, $src2"
 9730       "csetw $dst, ne"
 9731       "cnegw $dst, lt"
 9732   %}
 9733   ins_encode %{
 9734     int32_t con = (int32_t)$src2$$constant;
 9735      if (con < 0) {
 9736       __ adds(zr, $src1$$Register, -con);
 9737     } else {
 9738       __ subs(zr, $src1$$Register, con);
 9739     }
 9740     __ csetw($dst$$Register, Assembler::NE);
 9741     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9742   %}
 9743 
 9744   ins_pipe(pipe_class_default);
 9745 %}
 9746 
 9747 // ============================================================================
 9748 // Conditional Move Instructions
 9749 
 9750 // n.b. we have identical rules for both a signed compare op (cmpOp)
 9751 // and an unsigned compare op (cmpOpU). it would be nice if we could
 9752 // define an op class which merged both inputs and use it to type the
 9753 // argument to a single rule. unfortunatelyt his fails because the
 9754 // opclass does not live up to the COND_INTER interface of its
 9755 // component operands. When the generic code tries to negate the
 9756 // operand it ends up running the generci Machoper::negate method
 9757 // which throws a ShouldNotHappen. So, we have to provide two flavours
 9758 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
 9759 
 9760 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9761   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9762 
 9763   ins_cost(INSN_COST * 2);
 9764   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
 9765 
 9766   ins_encode %{
 9767     __ cselw(as_Register($dst$$reg),
 9768              as_Register($src2$$reg),
 9769              as_Register($src1$$reg),
 9770              (Assembler::Condition)$cmp$$cmpcode);
 9771   %}
 9772 
 9773   ins_pipe(icond_reg_reg);
 9774 %}
 9775 
 9776 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9777   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9778 
 9779   ins_cost(INSN_COST * 2);
 9780   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
 9781 
 9782   ins_encode %{
 9783     __ cselw(as_Register($dst$$reg),
 9784              as_Register($src2$$reg),
 9785              as_Register($src1$$reg),
 9786              (Assembler::Condition)$cmp$$cmpcode);
 9787   %}
 9788 
 9789   ins_pipe(icond_reg_reg);
 9790 %}
 9791 
 9792 // special cases where one arg is zero
 9793 
 9794 // n.b. this is selected in preference to the rule above because it
 9795 // avoids loading constant 0 into a source register
 9796 
 9797 // TODO
 9798 // we ought only to be able to cull one of these variants as the ideal
 9799 // transforms ought always to order the zero consistently (to left/right?)
 9800 
 9801 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9802   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9803 
 9804   ins_cost(INSN_COST * 2);
 9805   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
 9806 
 9807   ins_encode %{
 9808     __ cselw(as_Register($dst$$reg),
 9809              as_Register($src$$reg),
 9810              zr,
 9811              (Assembler::Condition)$cmp$$cmpcode);
 9812   %}
 9813 
 9814   ins_pipe(icond_reg);
 9815 %}
 9816 
 9817 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9818   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9819 
 9820   ins_cost(INSN_COST * 2);
 9821   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
 9822 
 9823   ins_encode %{
 9824     __ cselw(as_Register($dst$$reg),
 9825              as_Register($src$$reg),
 9826              zr,
 9827              (Assembler::Condition)$cmp$$cmpcode);
 9828   %}
 9829 
 9830   ins_pipe(icond_reg);
 9831 %}
 9832 
 9833 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9834   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9835 
 9836   ins_cost(INSN_COST * 2);
 9837   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
 9838 
 9839   ins_encode %{
 9840     __ cselw(as_Register($dst$$reg),
 9841              zr,
 9842              as_Register($src$$reg),
 9843              (Assembler::Condition)$cmp$$cmpcode);
 9844   %}
 9845 
 9846   ins_pipe(icond_reg);
 9847 %}
 9848 
 9849 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9850   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9851 
 9852   ins_cost(INSN_COST * 2);
 9853   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
 9854 
 9855   ins_encode %{
 9856     __ cselw(as_Register($dst$$reg),
 9857              zr,
 9858              as_Register($src$$reg),
 9859              (Assembler::Condition)$cmp$$cmpcode);
 9860   %}
 9861 
 9862   ins_pipe(icond_reg);
 9863 %}
 9864 
 9865 // special case for creating a boolean 0 or 1
 9866 
 9867 // n.b. this is selected in preference to the rule above because it
 9868 // avoids loading constants 0 and 1 into a source register
 9869 
 9870 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9871   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9872 
 9873   ins_cost(INSN_COST * 2);
 9874   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
 9875 
 9876   ins_encode %{
 9877     // equivalently
 9878     // cset(as_Register($dst$$reg),
 9879     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9880     __ csincw(as_Register($dst$$reg),
 9881              zr,
 9882              zr,
 9883              (Assembler::Condition)$cmp$$cmpcode);
 9884   %}
 9885 
 9886   ins_pipe(icond_none);
 9887 %}
 9888 
 9889 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9890   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9891 
 9892   ins_cost(INSN_COST * 2);
 9893   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
 9894 
 9895   ins_encode %{
 9896     // equivalently
 9897     // cset(as_Register($dst$$reg),
 9898     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9899     __ csincw(as_Register($dst$$reg),
 9900              zr,
 9901              zr,
 9902              (Assembler::Condition)$cmp$$cmpcode);
 9903   %}
 9904 
 9905   ins_pipe(icond_none);
 9906 %}
 9907 
 9908 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9909   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9910 
 9911   ins_cost(INSN_COST * 2);
 9912   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
 9913 
 9914   ins_encode %{
 9915     __ csel(as_Register($dst$$reg),
 9916             as_Register($src2$$reg),
 9917             as_Register($src1$$reg),
 9918             (Assembler::Condition)$cmp$$cmpcode);
 9919   %}
 9920 
 9921   ins_pipe(icond_reg_reg);
 9922 %}
 9923 
 9924 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9925   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9926 
 9927   ins_cost(INSN_COST * 2);
 9928   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
 9929 
 9930   ins_encode %{
 9931     __ csel(as_Register($dst$$reg),
 9932             as_Register($src2$$reg),
 9933             as_Register($src1$$reg),
 9934             (Assembler::Condition)$cmp$$cmpcode);
 9935   %}
 9936 
 9937   ins_pipe(icond_reg_reg);
 9938 %}
 9939 
 9940 // special cases where one arg is zero
 9941 
 9942 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9943   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9944 
 9945   ins_cost(INSN_COST * 2);
 9946   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
 9947 
 9948   ins_encode %{
 9949     __ csel(as_Register($dst$$reg),
 9950             zr,
 9951             as_Register($src$$reg),
 9952             (Assembler::Condition)$cmp$$cmpcode);
 9953   %}
 9954 
 9955   ins_pipe(icond_reg);
 9956 %}
 9957 
 9958 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9959   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9960 
 9961   ins_cost(INSN_COST * 2);
 9962   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
 9963 
 9964   ins_encode %{
 9965     __ csel(as_Register($dst$$reg),
 9966             zr,
 9967             as_Register($src$$reg),
 9968             (Assembler::Condition)$cmp$$cmpcode);
 9969   %}
 9970 
 9971   ins_pipe(icond_reg);
 9972 %}
 9973 
 9974 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9975   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9976 
 9977   ins_cost(INSN_COST * 2);
 9978   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
 9979 
 9980   ins_encode %{
 9981     __ csel(as_Register($dst$$reg),
 9982             as_Register($src$$reg),
 9983             zr,
 9984             (Assembler::Condition)$cmp$$cmpcode);
 9985   %}
 9986 
 9987   ins_pipe(icond_reg);
 9988 %}
 9989 
 9990 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9991   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9992 
 9993   ins_cost(INSN_COST * 2);
 9994   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
 9995 
 9996   ins_encode %{
 9997     __ csel(as_Register($dst$$reg),
 9998             as_Register($src$$reg),
 9999             zr,
10000             (Assembler::Condition)$cmp$$cmpcode);
10001   %}
10002 
10003   ins_pipe(icond_reg);
10004 %}
10005 
10006 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10007   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10008 
10009   ins_cost(INSN_COST * 2);
10010   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10011 
10012   ins_encode %{
10013     __ csel(as_Register($dst$$reg),
10014             as_Register($src2$$reg),
10015             as_Register($src1$$reg),
10016             (Assembler::Condition)$cmp$$cmpcode);
10017   %}
10018 
10019   ins_pipe(icond_reg_reg);
10020 %}
10021 
10022 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10023   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10024 
10025   ins_cost(INSN_COST * 2);
10026   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10027 
10028   ins_encode %{
10029     __ csel(as_Register($dst$$reg),
10030             as_Register($src2$$reg),
10031             as_Register($src1$$reg),
10032             (Assembler::Condition)$cmp$$cmpcode);
10033   %}
10034 
10035   ins_pipe(icond_reg_reg);
10036 %}
10037 
10038 // special cases where one arg is zero
10039 
10040 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10041   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10042 
10043   ins_cost(INSN_COST * 2);
10044   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10045 
10046   ins_encode %{
10047     __ csel(as_Register($dst$$reg),
10048             zr,
10049             as_Register($src$$reg),
10050             (Assembler::Condition)$cmp$$cmpcode);
10051   %}
10052 
10053   ins_pipe(icond_reg);
10054 %}
10055 
10056 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10057   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10058 
10059   ins_cost(INSN_COST * 2);
10060   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10061 
10062   ins_encode %{
10063     __ csel(as_Register($dst$$reg),
10064             zr,
10065             as_Register($src$$reg),
10066             (Assembler::Condition)$cmp$$cmpcode);
10067   %}
10068 
10069   ins_pipe(icond_reg);
10070 %}
10071 
10072 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10073   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10074 
10075   ins_cost(INSN_COST * 2);
10076   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10077 
10078   ins_encode %{
10079     __ csel(as_Register($dst$$reg),
10080             as_Register($src$$reg),
10081             zr,
10082             (Assembler::Condition)$cmp$$cmpcode);
10083   %}
10084 
10085   ins_pipe(icond_reg);
10086 %}
10087 
10088 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10089   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10090 
10091   ins_cost(INSN_COST * 2);
10092   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10093 
10094   ins_encode %{
10095     __ csel(as_Register($dst$$reg),
10096             as_Register($src$$reg),
10097             zr,
10098             (Assembler::Condition)$cmp$$cmpcode);
10099   %}
10100 
10101   ins_pipe(icond_reg);
10102 %}
10103 
10104 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10105   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10106 
10107   ins_cost(INSN_COST * 2);
10108   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10109 
10110   ins_encode %{
10111     __ cselw(as_Register($dst$$reg),
10112              as_Register($src2$$reg),
10113              as_Register($src1$$reg),
10114              (Assembler::Condition)$cmp$$cmpcode);
10115   %}
10116 
10117   ins_pipe(icond_reg_reg);
10118 %}
10119 
10120 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10121   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10122 
10123   ins_cost(INSN_COST * 2);
10124   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10125 
10126   ins_encode %{
10127     __ cselw(as_Register($dst$$reg),
10128              as_Register($src2$$reg),
10129              as_Register($src1$$reg),
10130              (Assembler::Condition)$cmp$$cmpcode);
10131   %}
10132 
10133   ins_pipe(icond_reg_reg);
10134 %}
10135 
10136 // special cases where one arg is zero
10137 
10138 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10139   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10140 
10141   ins_cost(INSN_COST * 2);
10142   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10143 
10144   ins_encode %{
10145     __ cselw(as_Register($dst$$reg),
10146              zr,
10147              as_Register($src$$reg),
10148              (Assembler::Condition)$cmp$$cmpcode);
10149   %}
10150 
10151   ins_pipe(icond_reg);
10152 %}
10153 
10154 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10155   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10156 
10157   ins_cost(INSN_COST * 2);
10158   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10159 
10160   ins_encode %{
10161     __ cselw(as_Register($dst$$reg),
10162              zr,
10163              as_Register($src$$reg),
10164              (Assembler::Condition)$cmp$$cmpcode);
10165   %}
10166 
10167   ins_pipe(icond_reg);
10168 %}
10169 
10170 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10171   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10172 
10173   ins_cost(INSN_COST * 2);
10174   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10175 
10176   ins_encode %{
10177     __ cselw(as_Register($dst$$reg),
10178              as_Register($src$$reg),
10179              zr,
10180              (Assembler::Condition)$cmp$$cmpcode);
10181   %}
10182 
10183   ins_pipe(icond_reg);
10184 %}
10185 
10186 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10187   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10188 
10189   ins_cost(INSN_COST * 2);
10190   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10191 
10192   ins_encode %{
10193     __ cselw(as_Register($dst$$reg),
10194              as_Register($src$$reg),
10195              zr,
10196              (Assembler::Condition)$cmp$$cmpcode);
10197   %}
10198 
10199   ins_pipe(icond_reg);
10200 %}
10201 
10202 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10203 %{
10204   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10205 
10206   ins_cost(INSN_COST * 3);
10207 
10208   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10209   ins_encode %{
10210     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10211     __ fcsels(as_FloatRegister($dst$$reg),
10212               as_FloatRegister($src2$$reg),
10213               as_FloatRegister($src1$$reg),
10214               cond);
10215   %}
10216 
10217   ins_pipe(fp_cond_reg_reg_s);
10218 %}
10219 
10220 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10221 %{
10222   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10223 
10224   ins_cost(INSN_COST * 3);
10225 
10226   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10227   ins_encode %{
10228     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10229     __ fcsels(as_FloatRegister($dst$$reg),
10230               as_FloatRegister($src2$$reg),
10231               as_FloatRegister($src1$$reg),
10232               cond);
10233   %}
10234 
10235   ins_pipe(fp_cond_reg_reg_s);
10236 %}
10237 
10238 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10239 %{
10240   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10241 
10242   ins_cost(INSN_COST * 3);
10243 
10244   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10245   ins_encode %{
10246     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10247     __ fcseld(as_FloatRegister($dst$$reg),
10248               as_FloatRegister($src2$$reg),
10249               as_FloatRegister($src1$$reg),
10250               cond);
10251   %}
10252 
10253   ins_pipe(fp_cond_reg_reg_d);
10254 %}
10255 
10256 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10257 %{
10258   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10259 
10260   ins_cost(INSN_COST * 3);
10261 
10262   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10263   ins_encode %{
10264     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10265     __ fcseld(as_FloatRegister($dst$$reg),
10266               as_FloatRegister($src2$$reg),
10267               as_FloatRegister($src1$$reg),
10268               cond);
10269   %}
10270 
10271   ins_pipe(fp_cond_reg_reg_d);
10272 %}
10273 
10274 // ============================================================================
10275 // Arithmetic Instructions
10276 //
10277 
10278 // Integer Addition
10279 
10280 // TODO
10281 // these currently employ operations which do not set CR and hence are
10282 // not flagged as killing CR but we would like to isolate the cases
10283 // where we want to set flags from those where we don't. need to work
10284 // out how to do that.
10285 
10286 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10287   match(Set dst (AddI src1 src2));
10288 
10289   ins_cost(INSN_COST);
10290   format %{ "addw  $dst, $src1, $src2" %}
10291 
10292   ins_encode %{
10293     __ addw(as_Register($dst$$reg),
10294             as_Register($src1$$reg),
10295             as_Register($src2$$reg));
10296   %}
10297 
10298   ins_pipe(ialu_reg_reg);
10299 %}
10300 
10301 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10302   match(Set dst (AddI src1 src2));
10303 
10304   ins_cost(INSN_COST);
10305   format %{ "addw $dst, $src1, $src2" %}
10306 
10307   // use opcode to indicate that this is an add not a sub
10308   opcode(0x0);
10309 
10310   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10311 
10312   ins_pipe(ialu_reg_imm);
10313 %}
10314 
10315 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10316   match(Set dst (AddI (ConvL2I src1) src2));
10317 
10318   ins_cost(INSN_COST);
10319   format %{ "addw $dst, $src1, $src2" %}
10320 
10321   // use opcode to indicate that this is an add not a sub
10322   opcode(0x0);
10323 
10324   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10325 
10326   ins_pipe(ialu_reg_imm);
10327 %}
10328 
10329 // Pointer Addition
10330 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10331   match(Set dst (AddP src1 src2));
10332 
10333   ins_cost(INSN_COST);
10334   format %{ "add $dst, $src1, $src2\t# ptr" %}
10335 
10336   ins_encode %{
10337     __ add(as_Register($dst$$reg),
10338            as_Register($src1$$reg),
10339            as_Register($src2$$reg));
10340   %}
10341 
10342   ins_pipe(ialu_reg_reg);
10343 %}
10344 
10345 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10346   match(Set dst (AddP src1 (ConvI2L src2)));
10347 
10348   ins_cost(1.9 * INSN_COST);
10349   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10350 
10351   ins_encode %{
10352     __ add(as_Register($dst$$reg),
10353            as_Register($src1$$reg),
10354            as_Register($src2$$reg), ext::sxtw);
10355   %}
10356 
10357   ins_pipe(ialu_reg_reg);
10358 %}
10359 
10360 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10361   match(Set dst (AddP src1 (LShiftL src2 scale)));
10362 
10363   ins_cost(1.9 * INSN_COST);
10364   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10365 
10366   ins_encode %{
10367     __ lea(as_Register($dst$$reg),
10368            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10369                    Address::lsl($scale$$constant)));
10370   %}
10371 
10372   ins_pipe(ialu_reg_reg_shift);
10373 %}
10374 
10375 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10376   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10377 
10378   ins_cost(1.9 * INSN_COST);
10379   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10380 
10381   ins_encode %{
10382     __ lea(as_Register($dst$$reg),
10383            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10384                    Address::sxtw($scale$$constant)));
10385   %}
10386 
10387   ins_pipe(ialu_reg_reg_shift);
10388 %}
10389 
10390 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10391   match(Set dst (LShiftL (ConvI2L src) scale));
10392 
10393   ins_cost(INSN_COST);
10394   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10395 
10396   ins_encode %{
10397     __ sbfiz(as_Register($dst$$reg),
10398           as_Register($src$$reg),
10399           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10400   %}
10401 
10402   ins_pipe(ialu_reg_shift);
10403 %}
10404 
10405 // Pointer Immediate Addition
10406 // n.b. this needs to be more expensive than using an indirect memory
10407 // operand
10408 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10409   match(Set dst (AddP src1 src2));
10410 
10411   ins_cost(INSN_COST);
10412   format %{ "add $dst, $src1, $src2\t# ptr" %}
10413 
10414   // use opcode to indicate that this is an add not a sub
10415   opcode(0x0);
10416 
10417   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10418 
10419   ins_pipe(ialu_reg_imm);
10420 %}
10421 
10422 // Long Addition
10423 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10424 
10425   match(Set dst (AddL src1 src2));
10426 
10427   ins_cost(INSN_COST);
10428   format %{ "add  $dst, $src1, $src2" %}
10429 
10430   ins_encode %{
10431     __ add(as_Register($dst$$reg),
10432            as_Register($src1$$reg),
10433            as_Register($src2$$reg));
10434   %}
10435 
10436   ins_pipe(ialu_reg_reg);
10437 %}
10438 
10439 // No constant pool entries requiredLong Immediate Addition.
10440 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10441   match(Set dst (AddL src1 src2));
10442 
10443   ins_cost(INSN_COST);
10444   format %{ "add $dst, $src1, $src2" %}
10445 
10446   // use opcode to indicate that this is an add not a sub
10447   opcode(0x0);
10448 
10449   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10450 
10451   ins_pipe(ialu_reg_imm);
10452 %}
10453 
10454 // Integer Subtraction
10455 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10456   match(Set dst (SubI src1 src2));
10457 
10458   ins_cost(INSN_COST);
10459   format %{ "subw  $dst, $src1, $src2" %}
10460 
10461   ins_encode %{
10462     __ subw(as_Register($dst$$reg),
10463             as_Register($src1$$reg),
10464             as_Register($src2$$reg));
10465   %}
10466 
10467   ins_pipe(ialu_reg_reg);
10468 %}
10469 
10470 // Immediate Subtraction
10471 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10472   match(Set dst (SubI src1 src2));
10473 
10474   ins_cost(INSN_COST);
10475   format %{ "subw $dst, $src1, $src2" %}
10476 
10477   // use opcode to indicate that this is a sub not an add
10478   opcode(0x1);
10479 
10480   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10481 
10482   ins_pipe(ialu_reg_imm);
10483 %}
10484 
10485 // Long Subtraction
10486 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10487 
10488   match(Set dst (SubL src1 src2));
10489 
10490   ins_cost(INSN_COST);
10491   format %{ "sub  $dst, $src1, $src2" %}
10492 
10493   ins_encode %{
10494     __ sub(as_Register($dst$$reg),
10495            as_Register($src1$$reg),
10496            as_Register($src2$$reg));
10497   %}
10498 
10499   ins_pipe(ialu_reg_reg);
10500 %}
10501 
10502 // No constant pool entries requiredLong Immediate Subtraction.
10503 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10504   match(Set dst (SubL src1 src2));
10505 
10506   ins_cost(INSN_COST);
10507   format %{ "sub$dst, $src1, $src2" %}
10508 
10509   // use opcode to indicate that this is a sub not an add
10510   opcode(0x1);
10511 
10512   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10513 
10514   ins_pipe(ialu_reg_imm);
10515 %}
10516 
10517 // Integer Negation (special case for sub)
10518 
10519 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10520   match(Set dst (SubI zero src));
10521 
10522   ins_cost(INSN_COST);
10523   format %{ "negw $dst, $src\t# int" %}
10524 
10525   ins_encode %{
10526     __ negw(as_Register($dst$$reg),
10527             as_Register($src$$reg));
10528   %}
10529 
10530   ins_pipe(ialu_reg);
10531 %}
10532 
10533 // Long Negation
10534 
10535 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10536   match(Set dst (SubL zero src));
10537 
10538   ins_cost(INSN_COST);
10539   format %{ "neg $dst, $src\t# long" %}
10540 
10541   ins_encode %{
10542     __ neg(as_Register($dst$$reg),
10543            as_Register($src$$reg));
10544   %}
10545 
10546   ins_pipe(ialu_reg);
10547 %}
10548 
10549 // Integer Multiply
10550 
10551 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10552   match(Set dst (MulI src1 src2));
10553 
10554   ins_cost(INSN_COST * 3);
10555   format %{ "mulw  $dst, $src1, $src2" %}
10556 
10557   ins_encode %{
10558     __ mulw(as_Register($dst$$reg),
10559             as_Register($src1$$reg),
10560             as_Register($src2$$reg));
10561   %}
10562 
10563   ins_pipe(imul_reg_reg);
10564 %}
10565 
10566 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10567   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10568 
10569   ins_cost(INSN_COST * 3);
10570   format %{ "smull  $dst, $src1, $src2" %}
10571 
10572   ins_encode %{
10573     __ smull(as_Register($dst$$reg),
10574              as_Register($src1$$reg),
10575              as_Register($src2$$reg));
10576   %}
10577 
10578   ins_pipe(imul_reg_reg);
10579 %}
10580 
10581 // Long Multiply
10582 
10583 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10584   match(Set dst (MulL src1 src2));
10585 
10586   ins_cost(INSN_COST * 5);
10587   format %{ "mul  $dst, $src1, $src2" %}
10588 
10589   ins_encode %{
10590     __ mul(as_Register($dst$$reg),
10591            as_Register($src1$$reg),
10592            as_Register($src2$$reg));
10593   %}
10594 
10595   ins_pipe(lmul_reg_reg);
10596 %}
10597 
10598 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10599 %{
10600   match(Set dst (MulHiL src1 src2));
10601 
10602   ins_cost(INSN_COST * 7);
10603   format %{ "smulh   $dst, $src1, $src2\t# mulhi" %}
10604 
10605   ins_encode %{
10606     __ smulh(as_Register($dst$$reg),
10607              as_Register($src1$$reg),
10608              as_Register($src2$$reg));
10609   %}
10610 
10611   ins_pipe(lmul_reg_reg);
10612 %}
10613 
10614 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10615 %{
10616   match(Set dst (UMulHiL src1 src2));
10617 
10618   ins_cost(INSN_COST * 7);
10619   format %{ "umulh   $dst, $src1, $src2\t# umulhi" %}
10620 
10621   ins_encode %{
10622     __ umulh(as_Register($dst$$reg),
10623              as_Register($src1$$reg),
10624              as_Register($src2$$reg));
10625   %}
10626 
10627   ins_pipe(lmul_reg_reg);
10628 %}
10629 
10630 // Combined Integer Multiply & Add/Sub
10631 
10632 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10633   match(Set dst (AddI src3 (MulI src1 src2)));
10634 
10635   ins_cost(INSN_COST * 3);
10636   format %{ "madd  $dst, $src1, $src2, $src3" %}
10637 
10638   ins_encode %{
10639     __ maddw(as_Register($dst$$reg),
10640              as_Register($src1$$reg),
10641              as_Register($src2$$reg),
10642              as_Register($src3$$reg));
10643   %}
10644 
10645   ins_pipe(imac_reg_reg);
10646 %}
10647 
10648 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10649   match(Set dst (SubI src3 (MulI src1 src2)));
10650 
10651   ins_cost(INSN_COST * 3);
10652   format %{ "msub  $dst, $src1, $src2, $src3" %}
10653 
10654   ins_encode %{
10655     __ msubw(as_Register($dst$$reg),
10656              as_Register($src1$$reg),
10657              as_Register($src2$$reg),
10658              as_Register($src3$$reg));
10659   %}
10660 
10661   ins_pipe(imac_reg_reg);
10662 %}
10663 
10664 // Combined Integer Multiply & Neg
10665 
10666 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10667   match(Set dst (MulI (SubI zero src1) src2));
10668 
10669   ins_cost(INSN_COST * 3);
10670   format %{ "mneg  $dst, $src1, $src2" %}
10671 
10672   ins_encode %{
10673     __ mnegw(as_Register($dst$$reg),
10674              as_Register($src1$$reg),
10675              as_Register($src2$$reg));
10676   %}
10677 
10678   ins_pipe(imac_reg_reg);
10679 %}
10680 
10681 // Combined Long Multiply & Add/Sub
10682 
10683 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10684   match(Set dst (AddL src3 (MulL src1 src2)));
10685 
10686   ins_cost(INSN_COST * 5);
10687   format %{ "madd  $dst, $src1, $src2, $src3" %}
10688 
10689   ins_encode %{
10690     __ madd(as_Register($dst$$reg),
10691             as_Register($src1$$reg),
10692             as_Register($src2$$reg),
10693             as_Register($src3$$reg));
10694   %}
10695 
10696   ins_pipe(lmac_reg_reg);
10697 %}
10698 
10699 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10700   match(Set dst (SubL src3 (MulL src1 src2)));
10701 
10702   ins_cost(INSN_COST * 5);
10703   format %{ "msub  $dst, $src1, $src2, $src3" %}
10704 
10705   ins_encode %{
10706     __ msub(as_Register($dst$$reg),
10707             as_Register($src1$$reg),
10708             as_Register($src2$$reg),
10709             as_Register($src3$$reg));
10710   %}
10711 
10712   ins_pipe(lmac_reg_reg);
10713 %}
10714 
10715 // Combined Long Multiply & Neg
10716 
10717 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10718   match(Set dst (MulL (SubL zero src1) src2));
10719 
10720   ins_cost(INSN_COST * 5);
10721   format %{ "mneg  $dst, $src1, $src2" %}
10722 
10723   ins_encode %{
10724     __ mneg(as_Register($dst$$reg),
10725             as_Register($src1$$reg),
10726             as_Register($src2$$reg));
10727   %}
10728 
10729   ins_pipe(lmac_reg_reg);
10730 %}
10731 
10732 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10733 
10734 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10735   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10736 
10737   ins_cost(INSN_COST * 3);
10738   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10739 
10740   ins_encode %{
10741     __ smaddl(as_Register($dst$$reg),
10742               as_Register($src1$$reg),
10743               as_Register($src2$$reg),
10744               as_Register($src3$$reg));
10745   %}
10746 
10747   ins_pipe(imac_reg_reg);
10748 %}
10749 
10750 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10751   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10752 
10753   ins_cost(INSN_COST * 3);
10754   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10755 
10756   ins_encode %{
10757     __ smsubl(as_Register($dst$$reg),
10758               as_Register($src1$$reg),
10759               as_Register($src2$$reg),
10760               as_Register($src3$$reg));
10761   %}
10762 
10763   ins_pipe(imac_reg_reg);
10764 %}
10765 
10766 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10767   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10768 
10769   ins_cost(INSN_COST * 3);
10770   format %{ "smnegl  $dst, $src1, $src2" %}
10771 
10772   ins_encode %{
10773     __ smnegl(as_Register($dst$$reg),
10774               as_Register($src1$$reg),
10775               as_Register($src2$$reg));
10776   %}
10777 
10778   ins_pipe(imac_reg_reg);
10779 %}
10780 
10781 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10782 
10783 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10784   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10785 
10786   ins_cost(INSN_COST * 5);
10787   format %{ "mulw  rscratch1, $src1, $src2\n\t"
10788             "maddw $dst, $src3, $src4, rscratch1" %}
10789 
10790   ins_encode %{
10791     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10792     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10793 
10794   ins_pipe(imac_reg_reg);
10795 %}
10796 
10797 // Integer Divide
10798 
10799 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10800   match(Set dst (DivI src1 src2));
10801 
10802   ins_cost(INSN_COST * 19);
10803   format %{ "sdivw  $dst, $src1, $src2" %}
10804 
10805   ins_encode(aarch64_enc_divw(dst, src1, src2));
10806   ins_pipe(idiv_reg_reg);
10807 %}
10808 
10809 // Long Divide
10810 
10811 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10812   match(Set dst (DivL src1 src2));
10813 
10814   ins_cost(INSN_COST * 35);
10815   format %{ "sdiv   $dst, $src1, $src2" %}
10816 
10817   ins_encode(aarch64_enc_div(dst, src1, src2));
10818   ins_pipe(ldiv_reg_reg);
10819 %}
10820 
10821 // Integer Remainder
10822 
10823 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10824   match(Set dst (ModI src1 src2));
10825 
10826   ins_cost(INSN_COST * 22);
10827   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
10828             "msubw  $dst, rscratch1, $src2, $src1" %}
10829 
10830   ins_encode(aarch64_enc_modw(dst, src1, src2));
10831   ins_pipe(idiv_reg_reg);
10832 %}
10833 
10834 // Long Remainder
10835 
10836 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10837   match(Set dst (ModL src1 src2));
10838 
10839   ins_cost(INSN_COST * 38);
10840   format %{ "sdiv   rscratch1, $src1, $src2\n"
10841             "msub   $dst, rscratch1, $src2, $src1" %}
10842 
10843   ins_encode(aarch64_enc_mod(dst, src1, src2));
10844   ins_pipe(ldiv_reg_reg);
10845 %}
10846 
10847 // Unsigned Integer Divide
10848 
10849 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10850   match(Set dst (UDivI src1 src2));
10851 
10852   ins_cost(INSN_COST * 19);
10853   format %{ "udivw  $dst, $src1, $src2" %}
10854 
10855   ins_encode %{
10856     __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10857   %}
10858 
10859   ins_pipe(idiv_reg_reg);
10860 %}
10861 
10862 //  Unsigned Long Divide
10863 
10864 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10865   match(Set dst (UDivL src1 src2));
10866 
10867   ins_cost(INSN_COST * 35);
10868   format %{ "udiv   $dst, $src1, $src2" %}
10869 
10870   ins_encode %{
10871     __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10872   %}
10873 
10874   ins_pipe(ldiv_reg_reg);
10875 %}
10876 
10877 // Unsigned Integer Remainder
10878 
10879 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10880   match(Set dst (UModI src1 src2));
10881 
10882   ins_cost(INSN_COST * 22);
10883   format %{ "udivw  rscratch1, $src1, $src2\n\t"
10884             "msubw  $dst, rscratch1, $src2, $src1" %}
10885 
10886   ins_encode %{
10887     __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10888     __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10889   %}
10890 
10891   ins_pipe(idiv_reg_reg);
10892 %}
10893 
10894 // Unsigned Long Remainder
10895 
10896 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10897   match(Set dst (UModL src1 src2));
10898 
10899   ins_cost(INSN_COST * 38);
10900   format %{ "udiv   rscratch1, $src1, $src2\n"
10901             "msub   $dst, rscratch1, $src2, $src1" %}
10902 
10903   ins_encode %{
10904     __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10905     __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10906   %}
10907 
10908   ins_pipe(ldiv_reg_reg);
10909 %}
10910 
10911 // Integer Shifts
10912 
10913 // Shift Left Register
10914 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10915   match(Set dst (LShiftI src1 src2));
10916 
10917   ins_cost(INSN_COST * 2);
10918   format %{ "lslvw  $dst, $src1, $src2" %}
10919 
10920   ins_encode %{
10921     __ lslvw(as_Register($dst$$reg),
10922              as_Register($src1$$reg),
10923              as_Register($src2$$reg));
10924   %}
10925 
10926   ins_pipe(ialu_reg_reg_vshift);
10927 %}
10928 
10929 // Shift Left Immediate
10930 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10931   match(Set dst (LShiftI src1 src2));
10932 
10933   ins_cost(INSN_COST);
10934   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10935 
10936   ins_encode %{
10937     __ lslw(as_Register($dst$$reg),
10938             as_Register($src1$$reg),
10939             $src2$$constant & 0x1f);
10940   %}
10941 
10942   ins_pipe(ialu_reg_shift);
10943 %}
10944 
10945 // Shift Right Logical Register
10946 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10947   match(Set dst (URShiftI src1 src2));
10948 
10949   ins_cost(INSN_COST * 2);
10950   format %{ "lsrvw  $dst, $src1, $src2" %}
10951 
10952   ins_encode %{
10953     __ lsrvw(as_Register($dst$$reg),
10954              as_Register($src1$$reg),
10955              as_Register($src2$$reg));
10956   %}
10957 
10958   ins_pipe(ialu_reg_reg_vshift);
10959 %}
10960 
10961 // Shift Right Logical Immediate
10962 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10963   match(Set dst (URShiftI src1 src2));
10964 
10965   ins_cost(INSN_COST);
10966   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10967 
10968   ins_encode %{
10969     __ lsrw(as_Register($dst$$reg),
10970             as_Register($src1$$reg),
10971             $src2$$constant & 0x1f);
10972   %}
10973 
10974   ins_pipe(ialu_reg_shift);
10975 %}
10976 
10977 // Shift Right Arithmetic Register
10978 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10979   match(Set dst (RShiftI src1 src2));
10980 
10981   ins_cost(INSN_COST * 2);
10982   format %{ "asrvw  $dst, $src1, $src2" %}
10983 
10984   ins_encode %{
10985     __ asrvw(as_Register($dst$$reg),
10986              as_Register($src1$$reg),
10987              as_Register($src2$$reg));
10988   %}
10989 
10990   ins_pipe(ialu_reg_reg_vshift);
10991 %}
10992 
10993 // Shift Right Arithmetic Immediate
10994 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10995   match(Set dst (RShiftI src1 src2));
10996 
10997   ins_cost(INSN_COST);
10998   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10999 
11000   ins_encode %{
11001     __ asrw(as_Register($dst$$reg),
11002             as_Register($src1$$reg),
11003             $src2$$constant & 0x1f);
11004   %}
11005 
11006   ins_pipe(ialu_reg_shift);
11007 %}
11008 
11009 // Combined Int Mask and Right Shift (using UBFM)
11010 // TODO
11011 
11012 // Long Shifts
11013 
11014 // Shift Left Register
11015 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11016   match(Set dst (LShiftL src1 src2));
11017 
11018   ins_cost(INSN_COST * 2);
11019   format %{ "lslv  $dst, $src1, $src2" %}
11020 
11021   ins_encode %{
11022     __ lslv(as_Register($dst$$reg),
11023             as_Register($src1$$reg),
11024             as_Register($src2$$reg));
11025   %}
11026 
11027   ins_pipe(ialu_reg_reg_vshift);
11028 %}
11029 
11030 // Shift Left Immediate
11031 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11032   match(Set dst (LShiftL src1 src2));
11033 
11034   ins_cost(INSN_COST);
11035   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11036 
11037   ins_encode %{
11038     __ lsl(as_Register($dst$$reg),
11039             as_Register($src1$$reg),
11040             $src2$$constant & 0x3f);
11041   %}
11042 
11043   ins_pipe(ialu_reg_shift);
11044 %}
11045 
11046 // Shift Right Logical Register
11047 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11048   match(Set dst (URShiftL src1 src2));
11049 
11050   ins_cost(INSN_COST * 2);
11051   format %{ "lsrv  $dst, $src1, $src2" %}
11052 
11053   ins_encode %{
11054     __ lsrv(as_Register($dst$$reg),
11055             as_Register($src1$$reg),
11056             as_Register($src2$$reg));
11057   %}
11058 
11059   ins_pipe(ialu_reg_reg_vshift);
11060 %}
11061 
11062 // Shift Right Logical Immediate
11063 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11064   match(Set dst (URShiftL src1 src2));
11065 
11066   ins_cost(INSN_COST);
11067   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11068 
11069   ins_encode %{
11070     __ lsr(as_Register($dst$$reg),
11071            as_Register($src1$$reg),
11072            $src2$$constant & 0x3f);
11073   %}
11074 
11075   ins_pipe(ialu_reg_shift);
11076 %}
11077 
11078 // A special-case pattern for card table stores.
11079 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11080   match(Set dst (URShiftL (CastP2X src1) src2));
11081 
11082   ins_cost(INSN_COST);
11083   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11084 
11085   ins_encode %{
11086     __ lsr(as_Register($dst$$reg),
11087            as_Register($src1$$reg),
11088            $src2$$constant & 0x3f);
11089   %}
11090 
11091   ins_pipe(ialu_reg_shift);
11092 %}
11093 
11094 // Shift Right Arithmetic Register
11095 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11096   match(Set dst (RShiftL src1 src2));
11097 
11098   ins_cost(INSN_COST * 2);
11099   format %{ "asrv  $dst, $src1, $src2" %}
11100 
11101   ins_encode %{
11102     __ asrv(as_Register($dst$$reg),
11103             as_Register($src1$$reg),
11104             as_Register($src2$$reg));
11105   %}
11106 
11107   ins_pipe(ialu_reg_reg_vshift);
11108 %}
11109 
11110 // Shift Right Arithmetic Immediate
11111 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11112   match(Set dst (RShiftL src1 src2));
11113 
11114   ins_cost(INSN_COST);
11115   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11116 
11117   ins_encode %{
11118     __ asr(as_Register($dst$$reg),
11119            as_Register($src1$$reg),
11120            $src2$$constant & 0x3f);
11121   %}
11122 
11123   ins_pipe(ialu_reg_shift);
11124 %}
11125 
11126 // BEGIN This section of the file is automatically generated. Do not edit --------------
11127 // This section is generated from aarch64_ad.m4
11128 
11129 
11130 // This pattern is automatically generated from aarch64_ad.m4
11131 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11132 instruct regL_not_reg(iRegLNoSp dst,
11133                          iRegL src1, immL_M1 m1,
11134                          rFlagsReg cr) %{
11135   match(Set dst (XorL src1 m1));
11136   ins_cost(INSN_COST);
11137   format %{ "eon  $dst, $src1, zr" %}
11138 
11139   ins_encode %{
11140     __ eon(as_Register($dst$$reg),
11141               as_Register($src1$$reg),
11142               zr,
11143               Assembler::LSL, 0);
11144   %}
11145 
11146   ins_pipe(ialu_reg);
11147 %}
11148 
11149 // This pattern is automatically generated from aarch64_ad.m4
11150 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11151 instruct regI_not_reg(iRegINoSp dst,
11152                          iRegIorL2I src1, immI_M1 m1,
11153                          rFlagsReg cr) %{
11154   match(Set dst (XorI src1 m1));
11155   ins_cost(INSN_COST);
11156   format %{ "eonw  $dst, $src1, zr" %}
11157 
11158   ins_encode %{
11159     __ eonw(as_Register($dst$$reg),
11160               as_Register($src1$$reg),
11161               zr,
11162               Assembler::LSL, 0);
11163   %}
11164 
11165   ins_pipe(ialu_reg);
11166 %}
11167 
11168 // This pattern is automatically generated from aarch64_ad.m4
11169 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11170 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11171                               immI0 zero, iRegIorL2I src1, immI src2) %{
11172   match(Set dst (SubI zero (URShiftI src1 src2)));
11173 
11174   ins_cost(1.9 * INSN_COST);
11175   format %{ "negw  $dst, $src1, LSR $src2" %}
11176 
11177   ins_encode %{
11178     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11179             Assembler::LSR, $src2$$constant & 0x1f);
11180   %}
11181 
11182   ins_pipe(ialu_reg_shift);
11183 %}
11184 
11185 // This pattern is automatically generated from aarch64_ad.m4
11186 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11187 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11188                               immI0 zero, iRegIorL2I src1, immI src2) %{
11189   match(Set dst (SubI zero (RShiftI src1 src2)));
11190 
11191   ins_cost(1.9 * INSN_COST);
11192   format %{ "negw  $dst, $src1, ASR $src2" %}
11193 
11194   ins_encode %{
11195     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11196             Assembler::ASR, $src2$$constant & 0x1f);
11197   %}
11198 
11199   ins_pipe(ialu_reg_shift);
11200 %}
11201 
11202 // This pattern is automatically generated from aarch64_ad.m4
11203 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11204 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11205                               immI0 zero, iRegIorL2I src1, immI src2) %{
11206   match(Set dst (SubI zero (LShiftI src1 src2)));
11207 
11208   ins_cost(1.9 * INSN_COST);
11209   format %{ "negw  $dst, $src1, LSL $src2" %}
11210 
11211   ins_encode %{
11212     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11213             Assembler::LSL, $src2$$constant & 0x1f);
11214   %}
11215 
11216   ins_pipe(ialu_reg_shift);
11217 %}
11218 
11219 // This pattern is automatically generated from aarch64_ad.m4
11220 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11221 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11222                               immL0 zero, iRegL src1, immI src2) %{
11223   match(Set dst (SubL zero (URShiftL src1 src2)));
11224 
11225   ins_cost(1.9 * INSN_COST);
11226   format %{ "neg  $dst, $src1, LSR $src2" %}
11227 
11228   ins_encode %{
11229     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11230             Assembler::LSR, $src2$$constant & 0x3f);
11231   %}
11232 
11233   ins_pipe(ialu_reg_shift);
11234 %}
11235 
11236 // This pattern is automatically generated from aarch64_ad.m4
11237 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11238 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11239                               immL0 zero, iRegL src1, immI src2) %{
11240   match(Set dst (SubL zero (RShiftL src1 src2)));
11241 
11242   ins_cost(1.9 * INSN_COST);
11243   format %{ "neg  $dst, $src1, ASR $src2" %}
11244 
11245   ins_encode %{
11246     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11247             Assembler::ASR, $src2$$constant & 0x3f);
11248   %}
11249 
11250   ins_pipe(ialu_reg_shift);
11251 %}
11252 
11253 // This pattern is automatically generated from aarch64_ad.m4
11254 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11255 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11256                               immL0 zero, iRegL src1, immI src2) %{
11257   match(Set dst (SubL zero (LShiftL src1 src2)));
11258 
11259   ins_cost(1.9 * INSN_COST);
11260   format %{ "neg  $dst, $src1, LSL $src2" %}
11261 
11262   ins_encode %{
11263     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11264             Assembler::LSL, $src2$$constant & 0x3f);
11265   %}
11266 
11267   ins_pipe(ialu_reg_shift);
11268 %}
11269 
11270 // This pattern is automatically generated from aarch64_ad.m4
11271 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11272 instruct AndI_reg_not_reg(iRegINoSp dst,
11273                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11274   match(Set dst (AndI src1 (XorI src2 m1)));
11275   ins_cost(INSN_COST);
11276   format %{ "bicw  $dst, $src1, $src2" %}
11277 
11278   ins_encode %{
11279     __ bicw(as_Register($dst$$reg),
11280               as_Register($src1$$reg),
11281               as_Register($src2$$reg),
11282               Assembler::LSL, 0);
11283   %}
11284 
11285   ins_pipe(ialu_reg_reg);
11286 %}
11287 
11288 // This pattern is automatically generated from aarch64_ad.m4
11289 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11290 instruct AndL_reg_not_reg(iRegLNoSp dst,
11291                          iRegL src1, iRegL src2, immL_M1 m1) %{
11292   match(Set dst (AndL src1 (XorL src2 m1)));
11293   ins_cost(INSN_COST);
11294   format %{ "bic  $dst, $src1, $src2" %}
11295 
11296   ins_encode %{
11297     __ bic(as_Register($dst$$reg),
11298               as_Register($src1$$reg),
11299               as_Register($src2$$reg),
11300               Assembler::LSL, 0);
11301   %}
11302 
11303   ins_pipe(ialu_reg_reg);
11304 %}
11305 
11306 // This pattern is automatically generated from aarch64_ad.m4
11307 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11308 instruct OrI_reg_not_reg(iRegINoSp dst,
11309                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11310   match(Set dst (OrI src1 (XorI src2 m1)));
11311   ins_cost(INSN_COST);
11312   format %{ "ornw  $dst, $src1, $src2" %}
11313 
11314   ins_encode %{
11315     __ ornw(as_Register($dst$$reg),
11316               as_Register($src1$$reg),
11317               as_Register($src2$$reg),
11318               Assembler::LSL, 0);
11319   %}
11320 
11321   ins_pipe(ialu_reg_reg);
11322 %}
11323 
11324 // This pattern is automatically generated from aarch64_ad.m4
11325 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11326 instruct OrL_reg_not_reg(iRegLNoSp dst,
11327                          iRegL src1, iRegL src2, immL_M1 m1) %{
11328   match(Set dst (OrL src1 (XorL src2 m1)));
11329   ins_cost(INSN_COST);
11330   format %{ "orn  $dst, $src1, $src2" %}
11331 
11332   ins_encode %{
11333     __ orn(as_Register($dst$$reg),
11334               as_Register($src1$$reg),
11335               as_Register($src2$$reg),
11336               Assembler::LSL, 0);
11337   %}
11338 
11339   ins_pipe(ialu_reg_reg);
11340 %}
11341 
11342 // This pattern is automatically generated from aarch64_ad.m4
11343 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11344 instruct XorI_reg_not_reg(iRegINoSp dst,
11345                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11346   match(Set dst (XorI m1 (XorI src2 src1)));
11347   ins_cost(INSN_COST);
11348   format %{ "eonw  $dst, $src1, $src2" %}
11349 
11350   ins_encode %{
11351     __ eonw(as_Register($dst$$reg),
11352               as_Register($src1$$reg),
11353               as_Register($src2$$reg),
11354               Assembler::LSL, 0);
11355   %}
11356 
11357   ins_pipe(ialu_reg_reg);
11358 %}
11359 
11360 // This pattern is automatically generated from aarch64_ad.m4
11361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11362 instruct XorL_reg_not_reg(iRegLNoSp dst,
11363                          iRegL src1, iRegL src2, immL_M1 m1) %{
11364   match(Set dst (XorL m1 (XorL src2 src1)));
11365   ins_cost(INSN_COST);
11366   format %{ "eon  $dst, $src1, $src2" %}
11367 
11368   ins_encode %{
11369     __ eon(as_Register($dst$$reg),
11370               as_Register($src1$$reg),
11371               as_Register($src2$$reg),
11372               Assembler::LSL, 0);
11373   %}
11374 
11375   ins_pipe(ialu_reg_reg);
11376 %}
11377 
11378 // This pattern is automatically generated from aarch64_ad.m4
11379 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11380 // val & (-1 ^ (val >>> shift)) ==> bicw
11381 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11382                          iRegIorL2I src1, iRegIorL2I src2,
11383                          immI src3, immI_M1 src4) %{
11384   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11385   ins_cost(1.9 * INSN_COST);
11386   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11387 
11388   ins_encode %{
11389     __ bicw(as_Register($dst$$reg),
11390               as_Register($src1$$reg),
11391               as_Register($src2$$reg),
11392               Assembler::LSR,
11393               $src3$$constant & 0x1f);
11394   %}
11395 
11396   ins_pipe(ialu_reg_reg_shift);
11397 %}
11398 
11399 // This pattern is automatically generated from aarch64_ad.m4
11400 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11401 // val & (-1 ^ (val >>> shift)) ==> bic
11402 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11403                          iRegL src1, iRegL src2,
11404                          immI src3, immL_M1 src4) %{
11405   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11406   ins_cost(1.9 * INSN_COST);
11407   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11408 
11409   ins_encode %{
11410     __ bic(as_Register($dst$$reg),
11411               as_Register($src1$$reg),
11412               as_Register($src2$$reg),
11413               Assembler::LSR,
11414               $src3$$constant & 0x3f);
11415   %}
11416 
11417   ins_pipe(ialu_reg_reg_shift);
11418 %}
11419 
11420 // This pattern is automatically generated from aarch64_ad.m4
11421 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11422 // val & (-1 ^ (val >> shift)) ==> bicw
11423 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11424                          iRegIorL2I src1, iRegIorL2I src2,
11425                          immI src3, immI_M1 src4) %{
11426   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11427   ins_cost(1.9 * INSN_COST);
11428   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11429 
11430   ins_encode %{
11431     __ bicw(as_Register($dst$$reg),
11432               as_Register($src1$$reg),
11433               as_Register($src2$$reg),
11434               Assembler::ASR,
11435               $src3$$constant & 0x1f);
11436   %}
11437 
11438   ins_pipe(ialu_reg_reg_shift);
11439 %}
11440 
11441 // This pattern is automatically generated from aarch64_ad.m4
11442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11443 // val & (-1 ^ (val >> shift)) ==> bic
11444 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11445                          iRegL src1, iRegL src2,
11446                          immI src3, immL_M1 src4) %{
11447   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11448   ins_cost(1.9 * INSN_COST);
11449   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11450 
11451   ins_encode %{
11452     __ bic(as_Register($dst$$reg),
11453               as_Register($src1$$reg),
11454               as_Register($src2$$reg),
11455               Assembler::ASR,
11456               $src3$$constant & 0x3f);
11457   %}
11458 
11459   ins_pipe(ialu_reg_reg_shift);
11460 %}
11461 
11462 // This pattern is automatically generated from aarch64_ad.m4
11463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11464 // val & (-1 ^ (val ror shift)) ==> bicw
11465 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11466                          iRegIorL2I src1, iRegIorL2I src2,
11467                          immI src3, immI_M1 src4) %{
11468   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11469   ins_cost(1.9 * INSN_COST);
11470   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11471 
11472   ins_encode %{
11473     __ bicw(as_Register($dst$$reg),
11474               as_Register($src1$$reg),
11475               as_Register($src2$$reg),
11476               Assembler::ROR,
11477               $src3$$constant & 0x1f);
11478   %}
11479 
11480   ins_pipe(ialu_reg_reg_shift);
11481 %}
11482 
11483 // This pattern is automatically generated from aarch64_ad.m4
11484 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11485 // val & (-1 ^ (val ror shift)) ==> bic
11486 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11487                          iRegL src1, iRegL src2,
11488                          immI src3, immL_M1 src4) %{
11489   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11490   ins_cost(1.9 * INSN_COST);
11491   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11492 
11493   ins_encode %{
11494     __ bic(as_Register($dst$$reg),
11495               as_Register($src1$$reg),
11496               as_Register($src2$$reg),
11497               Assembler::ROR,
11498               $src3$$constant & 0x3f);
11499   %}
11500 
11501   ins_pipe(ialu_reg_reg_shift);
11502 %}
11503 
11504 // This pattern is automatically generated from aarch64_ad.m4
11505 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11506 // val & (-1 ^ (val << shift)) ==> bicw
11507 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11508                          iRegIorL2I src1, iRegIorL2I src2,
11509                          immI src3, immI_M1 src4) %{
11510   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11511   ins_cost(1.9 * INSN_COST);
11512   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11513 
11514   ins_encode %{
11515     __ bicw(as_Register($dst$$reg),
11516               as_Register($src1$$reg),
11517               as_Register($src2$$reg),
11518               Assembler::LSL,
11519               $src3$$constant & 0x1f);
11520   %}
11521 
11522   ins_pipe(ialu_reg_reg_shift);
11523 %}
11524 
11525 // This pattern is automatically generated from aarch64_ad.m4
11526 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11527 // val & (-1 ^ (val << shift)) ==> bic
11528 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11529                          iRegL src1, iRegL src2,
11530                          immI src3, immL_M1 src4) %{
11531   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11532   ins_cost(1.9 * INSN_COST);
11533   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11534 
11535   ins_encode %{
11536     __ bic(as_Register($dst$$reg),
11537               as_Register($src1$$reg),
11538               as_Register($src2$$reg),
11539               Assembler::LSL,
11540               $src3$$constant & 0x3f);
11541   %}
11542 
11543   ins_pipe(ialu_reg_reg_shift);
11544 %}
11545 
11546 // This pattern is automatically generated from aarch64_ad.m4
11547 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11548 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11549 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11550                          iRegIorL2I src1, iRegIorL2I src2,
11551                          immI src3, immI_M1 src4) %{
11552   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11553   ins_cost(1.9 * INSN_COST);
11554   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11555 
11556   ins_encode %{
11557     __ eonw(as_Register($dst$$reg),
11558               as_Register($src1$$reg),
11559               as_Register($src2$$reg),
11560               Assembler::LSR,
11561               $src3$$constant & 0x1f);
11562   %}
11563 
11564   ins_pipe(ialu_reg_reg_shift);
11565 %}
11566 
11567 // This pattern is automatically generated from aarch64_ad.m4
11568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11569 // val ^ (-1 ^ (val >>> shift)) ==> eon
11570 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11571                          iRegL src1, iRegL src2,
11572                          immI src3, immL_M1 src4) %{
11573   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11574   ins_cost(1.9 * INSN_COST);
11575   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11576 
11577   ins_encode %{
11578     __ eon(as_Register($dst$$reg),
11579               as_Register($src1$$reg),
11580               as_Register($src2$$reg),
11581               Assembler::LSR,
11582               $src3$$constant & 0x3f);
11583   %}
11584 
11585   ins_pipe(ialu_reg_reg_shift);
11586 %}
11587 
11588 // This pattern is automatically generated from aarch64_ad.m4
11589 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11590 // val ^ (-1 ^ (val >> shift)) ==> eonw
11591 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11592                          iRegIorL2I src1, iRegIorL2I src2,
11593                          immI src3, immI_M1 src4) %{
11594   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11595   ins_cost(1.9 * INSN_COST);
11596   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11597 
11598   ins_encode %{
11599     __ eonw(as_Register($dst$$reg),
11600               as_Register($src1$$reg),
11601               as_Register($src2$$reg),
11602               Assembler::ASR,
11603               $src3$$constant & 0x1f);
11604   %}
11605 
11606   ins_pipe(ialu_reg_reg_shift);
11607 %}
11608 
11609 // This pattern is automatically generated from aarch64_ad.m4
11610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11611 // val ^ (-1 ^ (val >> shift)) ==> eon
11612 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11613                          iRegL src1, iRegL src2,
11614                          immI src3, immL_M1 src4) %{
11615   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11616   ins_cost(1.9 * INSN_COST);
11617   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11618 
11619   ins_encode %{
11620     __ eon(as_Register($dst$$reg),
11621               as_Register($src1$$reg),
11622               as_Register($src2$$reg),
11623               Assembler::ASR,
11624               $src3$$constant & 0x3f);
11625   %}
11626 
11627   ins_pipe(ialu_reg_reg_shift);
11628 %}
11629 
11630 // This pattern is automatically generated from aarch64_ad.m4
11631 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11632 // val ^ (-1 ^ (val ror shift)) ==> eonw
11633 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11634                          iRegIorL2I src1, iRegIorL2I src2,
11635                          immI src3, immI_M1 src4) %{
11636   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11637   ins_cost(1.9 * INSN_COST);
11638   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11639 
11640   ins_encode %{
11641     __ eonw(as_Register($dst$$reg),
11642               as_Register($src1$$reg),
11643               as_Register($src2$$reg),
11644               Assembler::ROR,
11645               $src3$$constant & 0x1f);
11646   %}
11647 
11648   ins_pipe(ialu_reg_reg_shift);
11649 %}
11650 
11651 // This pattern is automatically generated from aarch64_ad.m4
11652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11653 // val ^ (-1 ^ (val ror shift)) ==> eon
11654 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11655                          iRegL src1, iRegL src2,
11656                          immI src3, immL_M1 src4) %{
11657   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11658   ins_cost(1.9 * INSN_COST);
11659   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11660 
11661   ins_encode %{
11662     __ eon(as_Register($dst$$reg),
11663               as_Register($src1$$reg),
11664               as_Register($src2$$reg),
11665               Assembler::ROR,
11666               $src3$$constant & 0x3f);
11667   %}
11668 
11669   ins_pipe(ialu_reg_reg_shift);
11670 %}
11671 
11672 // This pattern is automatically generated from aarch64_ad.m4
11673 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11674 // val ^ (-1 ^ (val << shift)) ==> eonw
11675 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11676                          iRegIorL2I src1, iRegIorL2I src2,
11677                          immI src3, immI_M1 src4) %{
11678   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11679   ins_cost(1.9 * INSN_COST);
11680   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11681 
11682   ins_encode %{
11683     __ eonw(as_Register($dst$$reg),
11684               as_Register($src1$$reg),
11685               as_Register($src2$$reg),
11686               Assembler::LSL,
11687               $src3$$constant & 0x1f);
11688   %}
11689 
11690   ins_pipe(ialu_reg_reg_shift);
11691 %}
11692 
11693 // This pattern is automatically generated from aarch64_ad.m4
11694 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11695 // val ^ (-1 ^ (val << shift)) ==> eon
11696 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11697                          iRegL src1, iRegL src2,
11698                          immI src3, immL_M1 src4) %{
11699   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11700   ins_cost(1.9 * INSN_COST);
11701   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11702 
11703   ins_encode %{
11704     __ eon(as_Register($dst$$reg),
11705               as_Register($src1$$reg),
11706               as_Register($src2$$reg),
11707               Assembler::LSL,
11708               $src3$$constant & 0x3f);
11709   %}
11710 
11711   ins_pipe(ialu_reg_reg_shift);
11712 %}
11713 
11714 // This pattern is automatically generated from aarch64_ad.m4
11715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11716 // val | (-1 ^ (val >>> shift)) ==> ornw
11717 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11718                          iRegIorL2I src1, iRegIorL2I src2,
11719                          immI src3, immI_M1 src4) %{
11720   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11721   ins_cost(1.9 * INSN_COST);
11722   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11723 
11724   ins_encode %{
11725     __ ornw(as_Register($dst$$reg),
11726               as_Register($src1$$reg),
11727               as_Register($src2$$reg),
11728               Assembler::LSR,
11729               $src3$$constant & 0x1f);
11730   %}
11731 
11732   ins_pipe(ialu_reg_reg_shift);
11733 %}
11734 
11735 // This pattern is automatically generated from aarch64_ad.m4
11736 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11737 // val | (-1 ^ (val >>> shift)) ==> orn
11738 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11739                          iRegL src1, iRegL src2,
11740                          immI src3, immL_M1 src4) %{
11741   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11742   ins_cost(1.9 * INSN_COST);
11743   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11744 
11745   ins_encode %{
11746     __ orn(as_Register($dst$$reg),
11747               as_Register($src1$$reg),
11748               as_Register($src2$$reg),
11749               Assembler::LSR,
11750               $src3$$constant & 0x3f);
11751   %}
11752 
11753   ins_pipe(ialu_reg_reg_shift);
11754 %}
11755 
11756 // This pattern is automatically generated from aarch64_ad.m4
11757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11758 // val | (-1 ^ (val >> shift)) ==> ornw
11759 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11760                          iRegIorL2I src1, iRegIorL2I src2,
11761                          immI src3, immI_M1 src4) %{
11762   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11763   ins_cost(1.9 * INSN_COST);
11764   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11765 
11766   ins_encode %{
11767     __ ornw(as_Register($dst$$reg),
11768               as_Register($src1$$reg),
11769               as_Register($src2$$reg),
11770               Assembler::ASR,
11771               $src3$$constant & 0x1f);
11772   %}
11773 
11774   ins_pipe(ialu_reg_reg_shift);
11775 %}
11776 
11777 // This pattern is automatically generated from aarch64_ad.m4
11778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11779 // val | (-1 ^ (val >> shift)) ==> orn
11780 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11781                          iRegL src1, iRegL src2,
11782                          immI src3, immL_M1 src4) %{
11783   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11784   ins_cost(1.9 * INSN_COST);
11785   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11786 
11787   ins_encode %{
11788     __ orn(as_Register($dst$$reg),
11789               as_Register($src1$$reg),
11790               as_Register($src2$$reg),
11791               Assembler::ASR,
11792               $src3$$constant & 0x3f);
11793   %}
11794 
11795   ins_pipe(ialu_reg_reg_shift);
11796 %}
11797 
11798 // This pattern is automatically generated from aarch64_ad.m4
11799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11800 // val | (-1 ^ (val ror shift)) ==> ornw
11801 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11802                          iRegIorL2I src1, iRegIorL2I src2,
11803                          immI src3, immI_M1 src4) %{
11804   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11805   ins_cost(1.9 * INSN_COST);
11806   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11807 
11808   ins_encode %{
11809     __ ornw(as_Register($dst$$reg),
11810               as_Register($src1$$reg),
11811               as_Register($src2$$reg),
11812               Assembler::ROR,
11813               $src3$$constant & 0x1f);
11814   %}
11815 
11816   ins_pipe(ialu_reg_reg_shift);
11817 %}
11818 
11819 // This pattern is automatically generated from aarch64_ad.m4
11820 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11821 // val | (-1 ^ (val ror shift)) ==> orn
11822 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11823                          iRegL src1, iRegL src2,
11824                          immI src3, immL_M1 src4) %{
11825   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11826   ins_cost(1.9 * INSN_COST);
11827   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11828 
11829   ins_encode %{
11830     __ orn(as_Register($dst$$reg),
11831               as_Register($src1$$reg),
11832               as_Register($src2$$reg),
11833               Assembler::ROR,
11834               $src3$$constant & 0x3f);
11835   %}
11836 
11837   ins_pipe(ialu_reg_reg_shift);
11838 %}
11839 
11840 // This pattern is automatically generated from aarch64_ad.m4
11841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11842 // val | (-1 ^ (val << shift)) ==> ornw
11843 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11844                          iRegIorL2I src1, iRegIorL2I src2,
11845                          immI src3, immI_M1 src4) %{
11846   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11847   ins_cost(1.9 * INSN_COST);
11848   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11849 
11850   ins_encode %{
11851     __ ornw(as_Register($dst$$reg),
11852               as_Register($src1$$reg),
11853               as_Register($src2$$reg),
11854               Assembler::LSL,
11855               $src3$$constant & 0x1f);
11856   %}
11857 
11858   ins_pipe(ialu_reg_reg_shift);
11859 %}
11860 
11861 // This pattern is automatically generated from aarch64_ad.m4
11862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11863 // val | (-1 ^ (val << shift)) ==> orn
11864 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11865                          iRegL src1, iRegL src2,
11866                          immI src3, immL_M1 src4) %{
11867   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11868   ins_cost(1.9 * INSN_COST);
11869   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11870 
11871   ins_encode %{
11872     __ orn(as_Register($dst$$reg),
11873               as_Register($src1$$reg),
11874               as_Register($src2$$reg),
11875               Assembler::LSL,
11876               $src3$$constant & 0x3f);
11877   %}
11878 
11879   ins_pipe(ialu_reg_reg_shift);
11880 %}
11881 
11882 // This pattern is automatically generated from aarch64_ad.m4
11883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11884 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11885                          iRegIorL2I src1, iRegIorL2I src2,
11886                          immI src3) %{
11887   match(Set dst (AndI src1 (URShiftI src2 src3)));
11888 
11889   ins_cost(1.9 * INSN_COST);
11890   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
11891 
11892   ins_encode %{
11893     __ andw(as_Register($dst$$reg),
11894               as_Register($src1$$reg),
11895               as_Register($src2$$reg),
11896               Assembler::LSR,
11897               $src3$$constant & 0x1f);
11898   %}
11899 
11900   ins_pipe(ialu_reg_reg_shift);
11901 %}
11902 
11903 // This pattern is automatically generated from aarch64_ad.m4
11904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11905 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11906                          iRegL src1, iRegL src2,
11907                          immI src3) %{
11908   match(Set dst (AndL src1 (URShiftL src2 src3)));
11909 
11910   ins_cost(1.9 * INSN_COST);
11911   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
11912 
11913   ins_encode %{
11914     __ andr(as_Register($dst$$reg),
11915               as_Register($src1$$reg),
11916               as_Register($src2$$reg),
11917               Assembler::LSR,
11918               $src3$$constant & 0x3f);
11919   %}
11920 
11921   ins_pipe(ialu_reg_reg_shift);
11922 %}
11923 
11924 // This pattern is automatically generated from aarch64_ad.m4
11925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11926 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11927                          iRegIorL2I src1, iRegIorL2I src2,
11928                          immI src3) %{
11929   match(Set dst (AndI src1 (RShiftI src2 src3)));
11930 
11931   ins_cost(1.9 * INSN_COST);
11932   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
11933 
11934   ins_encode %{
11935     __ andw(as_Register($dst$$reg),
11936               as_Register($src1$$reg),
11937               as_Register($src2$$reg),
11938               Assembler::ASR,
11939               $src3$$constant & 0x1f);
11940   %}
11941 
11942   ins_pipe(ialu_reg_reg_shift);
11943 %}
11944 
11945 // This pattern is automatically generated from aarch64_ad.m4
11946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11947 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11948                          iRegL src1, iRegL src2,
11949                          immI src3) %{
11950   match(Set dst (AndL src1 (RShiftL src2 src3)));
11951 
11952   ins_cost(1.9 * INSN_COST);
11953   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
11954 
11955   ins_encode %{
11956     __ andr(as_Register($dst$$reg),
11957               as_Register($src1$$reg),
11958               as_Register($src2$$reg),
11959               Assembler::ASR,
11960               $src3$$constant & 0x3f);
11961   %}
11962 
11963   ins_pipe(ialu_reg_reg_shift);
11964 %}
11965 
11966 // This pattern is automatically generated from aarch64_ad.m4
11967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11968 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11969                          iRegIorL2I src1, iRegIorL2I src2,
11970                          immI src3) %{
11971   match(Set dst (AndI src1 (LShiftI src2 src3)));
11972 
11973   ins_cost(1.9 * INSN_COST);
11974   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
11975 
11976   ins_encode %{
11977     __ andw(as_Register($dst$$reg),
11978               as_Register($src1$$reg),
11979               as_Register($src2$$reg),
11980               Assembler::LSL,
11981               $src3$$constant & 0x1f);
11982   %}
11983 
11984   ins_pipe(ialu_reg_reg_shift);
11985 %}
11986 
11987 // This pattern is automatically generated from aarch64_ad.m4
11988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11989 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11990                          iRegL src1, iRegL src2,
11991                          immI src3) %{
11992   match(Set dst (AndL src1 (LShiftL src2 src3)));
11993 
11994   ins_cost(1.9 * INSN_COST);
11995   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
11996 
11997   ins_encode %{
11998     __ andr(as_Register($dst$$reg),
11999               as_Register($src1$$reg),
12000               as_Register($src2$$reg),
12001               Assembler::LSL,
12002               $src3$$constant & 0x3f);
12003   %}
12004 
12005   ins_pipe(ialu_reg_reg_shift);
12006 %}
12007 
12008 // This pattern is automatically generated from aarch64_ad.m4
12009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12010 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12011                          iRegIorL2I src1, iRegIorL2I src2,
12012                          immI src3) %{
12013   match(Set dst (AndI src1 (RotateRight src2 src3)));
12014 
12015   ins_cost(1.9 * INSN_COST);
12016   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12017 
12018   ins_encode %{
12019     __ andw(as_Register($dst$$reg),
12020               as_Register($src1$$reg),
12021               as_Register($src2$$reg),
12022               Assembler::ROR,
12023               $src3$$constant & 0x1f);
12024   %}
12025 
12026   ins_pipe(ialu_reg_reg_shift);
12027 %}
12028 
12029 // This pattern is automatically generated from aarch64_ad.m4
12030 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12031 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12032                          iRegL src1, iRegL src2,
12033                          immI src3) %{
12034   match(Set dst (AndL src1 (RotateRight src2 src3)));
12035 
12036   ins_cost(1.9 * INSN_COST);
12037   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12038 
12039   ins_encode %{
12040     __ andr(as_Register($dst$$reg),
12041               as_Register($src1$$reg),
12042               as_Register($src2$$reg),
12043               Assembler::ROR,
12044               $src3$$constant & 0x3f);
12045   %}
12046 
12047   ins_pipe(ialu_reg_reg_shift);
12048 %}
12049 
12050 // This pattern is automatically generated from aarch64_ad.m4
12051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12052 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12053                          iRegIorL2I src1, iRegIorL2I src2,
12054                          immI src3) %{
12055   match(Set dst (XorI src1 (URShiftI src2 src3)));
12056 
12057   ins_cost(1.9 * INSN_COST);
12058   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12059 
12060   ins_encode %{
12061     __ eorw(as_Register($dst$$reg),
12062               as_Register($src1$$reg),
12063               as_Register($src2$$reg),
12064               Assembler::LSR,
12065               $src3$$constant & 0x1f);
12066   %}
12067 
12068   ins_pipe(ialu_reg_reg_shift);
12069 %}
12070 
12071 // This pattern is automatically generated from aarch64_ad.m4
12072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12073 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12074                          iRegL src1, iRegL src2,
12075                          immI src3) %{
12076   match(Set dst (XorL src1 (URShiftL src2 src3)));
12077 
12078   ins_cost(1.9 * INSN_COST);
12079   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12080 
12081   ins_encode %{
12082     __ eor(as_Register($dst$$reg),
12083               as_Register($src1$$reg),
12084               as_Register($src2$$reg),
12085               Assembler::LSR,
12086               $src3$$constant & 0x3f);
12087   %}
12088 
12089   ins_pipe(ialu_reg_reg_shift);
12090 %}
12091 
12092 // This pattern is automatically generated from aarch64_ad.m4
12093 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12094 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12095                          iRegIorL2I src1, iRegIorL2I src2,
12096                          immI src3) %{
12097   match(Set dst (XorI src1 (RShiftI src2 src3)));
12098 
12099   ins_cost(1.9 * INSN_COST);
12100   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12101 
12102   ins_encode %{
12103     __ eorw(as_Register($dst$$reg),
12104               as_Register($src1$$reg),
12105               as_Register($src2$$reg),
12106               Assembler::ASR,
12107               $src3$$constant & 0x1f);
12108   %}
12109 
12110   ins_pipe(ialu_reg_reg_shift);
12111 %}
12112 
12113 // This pattern is automatically generated from aarch64_ad.m4
12114 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12115 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12116                          iRegL src1, iRegL src2,
12117                          immI src3) %{
12118   match(Set dst (XorL src1 (RShiftL src2 src3)));
12119 
12120   ins_cost(1.9 * INSN_COST);
12121   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12122 
12123   ins_encode %{
12124     __ eor(as_Register($dst$$reg),
12125               as_Register($src1$$reg),
12126               as_Register($src2$$reg),
12127               Assembler::ASR,
12128               $src3$$constant & 0x3f);
12129   %}
12130 
12131   ins_pipe(ialu_reg_reg_shift);
12132 %}
12133 
12134 // This pattern is automatically generated from aarch64_ad.m4
12135 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12136 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12137                          iRegIorL2I src1, iRegIorL2I src2,
12138                          immI src3) %{
12139   match(Set dst (XorI src1 (LShiftI src2 src3)));
12140 
12141   ins_cost(1.9 * INSN_COST);
12142   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12143 
12144   ins_encode %{
12145     __ eorw(as_Register($dst$$reg),
12146               as_Register($src1$$reg),
12147               as_Register($src2$$reg),
12148               Assembler::LSL,
12149               $src3$$constant & 0x1f);
12150   %}
12151 
12152   ins_pipe(ialu_reg_reg_shift);
12153 %}
12154 
12155 // This pattern is automatically generated from aarch64_ad.m4
12156 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12157 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12158                          iRegL src1, iRegL src2,
12159                          immI src3) %{
12160   match(Set dst (XorL src1 (LShiftL src2 src3)));
12161 
12162   ins_cost(1.9 * INSN_COST);
12163   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12164 
12165   ins_encode %{
12166     __ eor(as_Register($dst$$reg),
12167               as_Register($src1$$reg),
12168               as_Register($src2$$reg),
12169               Assembler::LSL,
12170               $src3$$constant & 0x3f);
12171   %}
12172 
12173   ins_pipe(ialu_reg_reg_shift);
12174 %}
12175 
12176 // This pattern is automatically generated from aarch64_ad.m4
12177 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12178 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12179                          iRegIorL2I src1, iRegIorL2I src2,
12180                          immI src3) %{
12181   match(Set dst (XorI src1 (RotateRight src2 src3)));
12182 
12183   ins_cost(1.9 * INSN_COST);
12184   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12185 
12186   ins_encode %{
12187     __ eorw(as_Register($dst$$reg),
12188               as_Register($src1$$reg),
12189               as_Register($src2$$reg),
12190               Assembler::ROR,
12191               $src3$$constant & 0x1f);
12192   %}
12193 
12194   ins_pipe(ialu_reg_reg_shift);
12195 %}
12196 
12197 // This pattern is automatically generated from aarch64_ad.m4
12198 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12199 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12200                          iRegL src1, iRegL src2,
12201                          immI src3) %{
12202   match(Set dst (XorL src1 (RotateRight src2 src3)));
12203 
12204   ins_cost(1.9 * INSN_COST);
12205   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12206 
12207   ins_encode %{
12208     __ eor(as_Register($dst$$reg),
12209               as_Register($src1$$reg),
12210               as_Register($src2$$reg),
12211               Assembler::ROR,
12212               $src3$$constant & 0x3f);
12213   %}
12214 
12215   ins_pipe(ialu_reg_reg_shift);
12216 %}
12217 
12218 // This pattern is automatically generated from aarch64_ad.m4
12219 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12220 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12221                          iRegIorL2I src1, iRegIorL2I src2,
12222                          immI src3) %{
12223   match(Set dst (OrI src1 (URShiftI src2 src3)));
12224 
12225   ins_cost(1.9 * INSN_COST);
12226   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12227 
12228   ins_encode %{
12229     __ orrw(as_Register($dst$$reg),
12230               as_Register($src1$$reg),
12231               as_Register($src2$$reg),
12232               Assembler::LSR,
12233               $src3$$constant & 0x1f);
12234   %}
12235 
12236   ins_pipe(ialu_reg_reg_shift);
12237 %}
12238 
12239 // This pattern is automatically generated from aarch64_ad.m4
12240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12241 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12242                          iRegL src1, iRegL src2,
12243                          immI src3) %{
12244   match(Set dst (OrL src1 (URShiftL src2 src3)));
12245 
12246   ins_cost(1.9 * INSN_COST);
12247   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12248 
12249   ins_encode %{
12250     __ orr(as_Register($dst$$reg),
12251               as_Register($src1$$reg),
12252               as_Register($src2$$reg),
12253               Assembler::LSR,
12254               $src3$$constant & 0x3f);
12255   %}
12256 
12257   ins_pipe(ialu_reg_reg_shift);
12258 %}
12259 
12260 // This pattern is automatically generated from aarch64_ad.m4
12261 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12262 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12263                          iRegIorL2I src1, iRegIorL2I src2,
12264                          immI src3) %{
12265   match(Set dst (OrI src1 (RShiftI src2 src3)));
12266 
12267   ins_cost(1.9 * INSN_COST);
12268   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12269 
12270   ins_encode %{
12271     __ orrw(as_Register($dst$$reg),
12272               as_Register($src1$$reg),
12273               as_Register($src2$$reg),
12274               Assembler::ASR,
12275               $src3$$constant & 0x1f);
12276   %}
12277 
12278   ins_pipe(ialu_reg_reg_shift);
12279 %}
12280 
12281 // This pattern is automatically generated from aarch64_ad.m4
12282 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12283 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12284                          iRegL src1, iRegL src2,
12285                          immI src3) %{
12286   match(Set dst (OrL src1 (RShiftL src2 src3)));
12287 
12288   ins_cost(1.9 * INSN_COST);
12289   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12290 
12291   ins_encode %{
12292     __ orr(as_Register($dst$$reg),
12293               as_Register($src1$$reg),
12294               as_Register($src2$$reg),
12295               Assembler::ASR,
12296               $src3$$constant & 0x3f);
12297   %}
12298 
12299   ins_pipe(ialu_reg_reg_shift);
12300 %}
12301 
12302 // This pattern is automatically generated from aarch64_ad.m4
12303 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12304 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12305                          iRegIorL2I src1, iRegIorL2I src2,
12306                          immI src3) %{
12307   match(Set dst (OrI src1 (LShiftI src2 src3)));
12308 
12309   ins_cost(1.9 * INSN_COST);
12310   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12311 
12312   ins_encode %{
12313     __ orrw(as_Register($dst$$reg),
12314               as_Register($src1$$reg),
12315               as_Register($src2$$reg),
12316               Assembler::LSL,
12317               $src3$$constant & 0x1f);
12318   %}
12319 
12320   ins_pipe(ialu_reg_reg_shift);
12321 %}
12322 
12323 // This pattern is automatically generated from aarch64_ad.m4
12324 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12325 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12326                          iRegL src1, iRegL src2,
12327                          immI src3) %{
12328   match(Set dst (OrL src1 (LShiftL src2 src3)));
12329 
12330   ins_cost(1.9 * INSN_COST);
12331   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12332 
12333   ins_encode %{
12334     __ orr(as_Register($dst$$reg),
12335               as_Register($src1$$reg),
12336               as_Register($src2$$reg),
12337               Assembler::LSL,
12338               $src3$$constant & 0x3f);
12339   %}
12340 
12341   ins_pipe(ialu_reg_reg_shift);
12342 %}
12343 
12344 // This pattern is automatically generated from aarch64_ad.m4
12345 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12346 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12347                          iRegIorL2I src1, iRegIorL2I src2,
12348                          immI src3) %{
12349   match(Set dst (OrI src1 (RotateRight src2 src3)));
12350 
12351   ins_cost(1.9 * INSN_COST);
12352   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12353 
12354   ins_encode %{
12355     __ orrw(as_Register($dst$$reg),
12356               as_Register($src1$$reg),
12357               as_Register($src2$$reg),
12358               Assembler::ROR,
12359               $src3$$constant & 0x1f);
12360   %}
12361 
12362   ins_pipe(ialu_reg_reg_shift);
12363 %}
12364 
12365 // This pattern is automatically generated from aarch64_ad.m4
12366 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12367 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12368                          iRegL src1, iRegL src2,
12369                          immI src3) %{
12370   match(Set dst (OrL src1 (RotateRight src2 src3)));
12371 
12372   ins_cost(1.9 * INSN_COST);
12373   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12374 
12375   ins_encode %{
12376     __ orr(as_Register($dst$$reg),
12377               as_Register($src1$$reg),
12378               as_Register($src2$$reg),
12379               Assembler::ROR,
12380               $src3$$constant & 0x3f);
12381   %}
12382 
12383   ins_pipe(ialu_reg_reg_shift);
12384 %}
12385 
12386 // This pattern is automatically generated from aarch64_ad.m4
12387 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12388 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12389                          iRegIorL2I src1, iRegIorL2I src2,
12390                          immI src3) %{
12391   match(Set dst (AddI src1 (URShiftI src2 src3)));
12392 
12393   ins_cost(1.9 * INSN_COST);
12394   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12395 
12396   ins_encode %{
12397     __ addw(as_Register($dst$$reg),
12398               as_Register($src1$$reg),
12399               as_Register($src2$$reg),
12400               Assembler::LSR,
12401               $src3$$constant & 0x1f);
12402   %}
12403 
12404   ins_pipe(ialu_reg_reg_shift);
12405 %}
12406 
12407 // This pattern is automatically generated from aarch64_ad.m4
12408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12409 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12410                          iRegL src1, iRegL src2,
12411                          immI src3) %{
12412   match(Set dst (AddL src1 (URShiftL src2 src3)));
12413 
12414   ins_cost(1.9 * INSN_COST);
12415   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12416 
12417   ins_encode %{
12418     __ add(as_Register($dst$$reg),
12419               as_Register($src1$$reg),
12420               as_Register($src2$$reg),
12421               Assembler::LSR,
12422               $src3$$constant & 0x3f);
12423   %}
12424 
12425   ins_pipe(ialu_reg_reg_shift);
12426 %}
12427 
12428 // This pattern is automatically generated from aarch64_ad.m4
12429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12430 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12431                          iRegIorL2I src1, iRegIorL2I src2,
12432                          immI src3) %{
12433   match(Set dst (AddI src1 (RShiftI src2 src3)));
12434 
12435   ins_cost(1.9 * INSN_COST);
12436   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12437 
12438   ins_encode %{
12439     __ addw(as_Register($dst$$reg),
12440               as_Register($src1$$reg),
12441               as_Register($src2$$reg),
12442               Assembler::ASR,
12443               $src3$$constant & 0x1f);
12444   %}
12445 
12446   ins_pipe(ialu_reg_reg_shift);
12447 %}
12448 
12449 // This pattern is automatically generated from aarch64_ad.m4
12450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12451 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12452                          iRegL src1, iRegL src2,
12453                          immI src3) %{
12454   match(Set dst (AddL src1 (RShiftL src2 src3)));
12455 
12456   ins_cost(1.9 * INSN_COST);
12457   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12458 
12459   ins_encode %{
12460     __ add(as_Register($dst$$reg),
12461               as_Register($src1$$reg),
12462               as_Register($src2$$reg),
12463               Assembler::ASR,
12464               $src3$$constant & 0x3f);
12465   %}
12466 
12467   ins_pipe(ialu_reg_reg_shift);
12468 %}
12469 
12470 // This pattern is automatically generated from aarch64_ad.m4
12471 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12472 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12473                          iRegIorL2I src1, iRegIorL2I src2,
12474                          immI src3) %{
12475   match(Set dst (AddI src1 (LShiftI src2 src3)));
12476 
12477   ins_cost(1.9 * INSN_COST);
12478   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12479 
12480   ins_encode %{
12481     __ addw(as_Register($dst$$reg),
12482               as_Register($src1$$reg),
12483               as_Register($src2$$reg),
12484               Assembler::LSL,
12485               $src3$$constant & 0x1f);
12486   %}
12487 
12488   ins_pipe(ialu_reg_reg_shift);
12489 %}
12490 
12491 // This pattern is automatically generated from aarch64_ad.m4
12492 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12493 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12494                          iRegL src1, iRegL src2,
12495                          immI src3) %{
12496   match(Set dst (AddL src1 (LShiftL src2 src3)));
12497 
12498   ins_cost(1.9 * INSN_COST);
12499   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12500 
12501   ins_encode %{
12502     __ add(as_Register($dst$$reg),
12503               as_Register($src1$$reg),
12504               as_Register($src2$$reg),
12505               Assembler::LSL,
12506               $src3$$constant & 0x3f);
12507   %}
12508 
12509   ins_pipe(ialu_reg_reg_shift);
12510 %}
12511 
12512 // This pattern is automatically generated from aarch64_ad.m4
12513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12514 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12515                          iRegIorL2I src1, iRegIorL2I src2,
12516                          immI src3) %{
12517   match(Set dst (SubI src1 (URShiftI src2 src3)));
12518 
12519   ins_cost(1.9 * INSN_COST);
12520   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12521 
12522   ins_encode %{
12523     __ subw(as_Register($dst$$reg),
12524               as_Register($src1$$reg),
12525               as_Register($src2$$reg),
12526               Assembler::LSR,
12527               $src3$$constant & 0x1f);
12528   %}
12529 
12530   ins_pipe(ialu_reg_reg_shift);
12531 %}
12532 
12533 // This pattern is automatically generated from aarch64_ad.m4
12534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12535 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12536                          iRegL src1, iRegL src2,
12537                          immI src3) %{
12538   match(Set dst (SubL src1 (URShiftL src2 src3)));
12539 
12540   ins_cost(1.9 * INSN_COST);
12541   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12542 
12543   ins_encode %{
12544     __ sub(as_Register($dst$$reg),
12545               as_Register($src1$$reg),
12546               as_Register($src2$$reg),
12547               Assembler::LSR,
12548               $src3$$constant & 0x3f);
12549   %}
12550 
12551   ins_pipe(ialu_reg_reg_shift);
12552 %}
12553 
12554 // This pattern is automatically generated from aarch64_ad.m4
12555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12556 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12557                          iRegIorL2I src1, iRegIorL2I src2,
12558                          immI src3) %{
12559   match(Set dst (SubI src1 (RShiftI src2 src3)));
12560 
12561   ins_cost(1.9 * INSN_COST);
12562   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12563 
12564   ins_encode %{
12565     __ subw(as_Register($dst$$reg),
12566               as_Register($src1$$reg),
12567               as_Register($src2$$reg),
12568               Assembler::ASR,
12569               $src3$$constant & 0x1f);
12570   %}
12571 
12572   ins_pipe(ialu_reg_reg_shift);
12573 %}
12574 
12575 // This pattern is automatically generated from aarch64_ad.m4
12576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12577 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12578                          iRegL src1, iRegL src2,
12579                          immI src3) %{
12580   match(Set dst (SubL src1 (RShiftL src2 src3)));
12581 
12582   ins_cost(1.9 * INSN_COST);
12583   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12584 
12585   ins_encode %{
12586     __ sub(as_Register($dst$$reg),
12587               as_Register($src1$$reg),
12588               as_Register($src2$$reg),
12589               Assembler::ASR,
12590               $src3$$constant & 0x3f);
12591   %}
12592 
12593   ins_pipe(ialu_reg_reg_shift);
12594 %}
12595 
12596 // This pattern is automatically generated from aarch64_ad.m4
12597 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12598 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12599                          iRegIorL2I src1, iRegIorL2I src2,
12600                          immI src3) %{
12601   match(Set dst (SubI src1 (LShiftI src2 src3)));
12602 
12603   ins_cost(1.9 * INSN_COST);
12604   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12605 
12606   ins_encode %{
12607     __ subw(as_Register($dst$$reg),
12608               as_Register($src1$$reg),
12609               as_Register($src2$$reg),
12610               Assembler::LSL,
12611               $src3$$constant & 0x1f);
12612   %}
12613 
12614   ins_pipe(ialu_reg_reg_shift);
12615 %}
12616 
12617 // This pattern is automatically generated from aarch64_ad.m4
12618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12619 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12620                          iRegL src1, iRegL src2,
12621                          immI src3) %{
12622   match(Set dst (SubL src1 (LShiftL src2 src3)));
12623 
12624   ins_cost(1.9 * INSN_COST);
12625   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12626 
12627   ins_encode %{
12628     __ sub(as_Register($dst$$reg),
12629               as_Register($src1$$reg),
12630               as_Register($src2$$reg),
12631               Assembler::LSL,
12632               $src3$$constant & 0x3f);
12633   %}
12634 
12635   ins_pipe(ialu_reg_reg_shift);
12636 %}
12637 
12638 // This pattern is automatically generated from aarch64_ad.m4
12639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12640 
12641 // Shift Left followed by Shift Right.
12642 // This idiom is used by the compiler for the i2b bytecode etc.
12643 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12644 %{
12645   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12646   ins_cost(INSN_COST * 2);
12647   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12648   ins_encode %{
12649     int lshift = $lshift_count$$constant & 63;
12650     int rshift = $rshift_count$$constant & 63;
12651     int s = 63 - lshift;
12652     int r = (rshift - lshift) & 63;
12653     __ sbfm(as_Register($dst$$reg),
12654             as_Register($src$$reg),
12655             r, s);
12656   %}
12657 
12658   ins_pipe(ialu_reg_shift);
12659 %}
12660 
12661 // This pattern is automatically generated from aarch64_ad.m4
12662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12663 
12664 // Shift Left followed by Shift Right.
12665 // This idiom is used by the compiler for the i2b bytecode etc.
12666 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12667 %{
12668   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12669   ins_cost(INSN_COST * 2);
12670   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12671   ins_encode %{
12672     int lshift = $lshift_count$$constant & 31;
12673     int rshift = $rshift_count$$constant & 31;
12674     int s = 31 - lshift;
12675     int r = (rshift - lshift) & 31;
12676     __ sbfmw(as_Register($dst$$reg),
12677             as_Register($src$$reg),
12678             r, s);
12679   %}
12680 
12681   ins_pipe(ialu_reg_shift);
12682 %}
12683 
12684 // This pattern is automatically generated from aarch64_ad.m4
12685 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12686 
12687 // Shift Left followed by Shift Right.
12688 // This idiom is used by the compiler for the i2b bytecode etc.
12689 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12690 %{
12691   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12692   ins_cost(INSN_COST * 2);
12693   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12694   ins_encode %{
12695     int lshift = $lshift_count$$constant & 63;
12696     int rshift = $rshift_count$$constant & 63;
12697     int s = 63 - lshift;
12698     int r = (rshift - lshift) & 63;
12699     __ ubfm(as_Register($dst$$reg),
12700             as_Register($src$$reg),
12701             r, s);
12702   %}
12703 
12704   ins_pipe(ialu_reg_shift);
12705 %}
12706 
12707 // This pattern is automatically generated from aarch64_ad.m4
12708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12709 
12710 // Shift Left followed by Shift Right.
12711 // This idiom is used by the compiler for the i2b bytecode etc.
12712 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12713 %{
12714   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12715   ins_cost(INSN_COST * 2);
12716   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12717   ins_encode %{
12718     int lshift = $lshift_count$$constant & 31;
12719     int rshift = $rshift_count$$constant & 31;
12720     int s = 31 - lshift;
12721     int r = (rshift - lshift) & 31;
12722     __ ubfmw(as_Register($dst$$reg),
12723             as_Register($src$$reg),
12724             r, s);
12725   %}
12726 
12727   ins_pipe(ialu_reg_shift);
12728 %}
12729 
12730 // Bitfield extract with shift & mask
12731 
12732 // This pattern is automatically generated from aarch64_ad.m4
12733 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12734 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12735 %{
12736   match(Set dst (AndI (URShiftI src rshift) mask));
12737   // Make sure we are not going to exceed what ubfxw can do.
12738   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12739 
12740   ins_cost(INSN_COST);
12741   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12742   ins_encode %{
12743     int rshift = $rshift$$constant & 31;
12744     intptr_t mask = $mask$$constant;
12745     int width = exact_log2(mask+1);
12746     __ ubfxw(as_Register($dst$$reg),
12747             as_Register($src$$reg), rshift, width);
12748   %}
12749   ins_pipe(ialu_reg_shift);
12750 %}
12751 
12752 // This pattern is automatically generated from aarch64_ad.m4
12753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12754 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12755 %{
12756   match(Set dst (AndL (URShiftL src rshift) mask));
12757   // Make sure we are not going to exceed what ubfx can do.
12758   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12759 
12760   ins_cost(INSN_COST);
12761   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12762   ins_encode %{
12763     int rshift = $rshift$$constant & 63;
12764     intptr_t mask = $mask$$constant;
12765     int width = exact_log2_long(mask+1);
12766     __ ubfx(as_Register($dst$$reg),
12767             as_Register($src$$reg), rshift, width);
12768   %}
12769   ins_pipe(ialu_reg_shift);
12770 %}
12771 
12772 
12773 // This pattern is automatically generated from aarch64_ad.m4
12774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12775 
12776 // We can use ubfx when extending an And with a mask when we know mask
12777 // is positive.  We know that because immI_bitmask guarantees it.
12778 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12779 %{
12780   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12781   // Make sure we are not going to exceed what ubfxw can do.
12782   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12783 
12784   ins_cost(INSN_COST * 2);
12785   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12786   ins_encode %{
12787     int rshift = $rshift$$constant & 31;
12788     intptr_t mask = $mask$$constant;
12789     int width = exact_log2(mask+1);
12790     __ ubfx(as_Register($dst$$reg),
12791             as_Register($src$$reg), rshift, width);
12792   %}
12793   ins_pipe(ialu_reg_shift);
12794 %}
12795 
12796 
12797 // This pattern is automatically generated from aarch64_ad.m4
12798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12799 
12800 // We can use ubfiz when masking by a positive number and then left shifting the result.
12801 // We know that the mask is positive because immI_bitmask guarantees it.
12802 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12803 %{
12804   match(Set dst (LShiftI (AndI src mask) lshift));
12805   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12806 
12807   ins_cost(INSN_COST);
12808   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12809   ins_encode %{
12810     int lshift = $lshift$$constant & 31;
12811     intptr_t mask = $mask$$constant;
12812     int width = exact_log2(mask+1);
12813     __ ubfizw(as_Register($dst$$reg),
12814           as_Register($src$$reg), lshift, width);
12815   %}
12816   ins_pipe(ialu_reg_shift);
12817 %}
12818 
12819 // This pattern is automatically generated from aarch64_ad.m4
12820 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12821 
12822 // We can use ubfiz when masking by a positive number and then left shifting the result.
12823 // We know that the mask is positive because immL_bitmask guarantees it.
12824 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12825 %{
12826   match(Set dst (LShiftL (AndL src mask) lshift));
12827   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12828 
12829   ins_cost(INSN_COST);
12830   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12831   ins_encode %{
12832     int lshift = $lshift$$constant & 63;
12833     intptr_t mask = $mask$$constant;
12834     int width = exact_log2_long(mask+1);
12835     __ ubfiz(as_Register($dst$$reg),
12836           as_Register($src$$reg), lshift, width);
12837   %}
12838   ins_pipe(ialu_reg_shift);
12839 %}
12840 
12841 // This pattern is automatically generated from aarch64_ad.m4
12842 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12843 
12844 // We can use ubfiz when masking by a positive number and then left shifting the result.
12845 // We know that the mask is positive because immI_bitmask guarantees it.
12846 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12847 %{
12848   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12849   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12850 
12851   ins_cost(INSN_COST);
12852   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12853   ins_encode %{
12854     int lshift = $lshift$$constant & 31;
12855     intptr_t mask = $mask$$constant;
12856     int width = exact_log2(mask+1);
12857     __ ubfizw(as_Register($dst$$reg),
12858           as_Register($src$$reg), lshift, width);
12859   %}
12860   ins_pipe(ialu_reg_shift);
12861 %}
12862 
12863 // This pattern is automatically generated from aarch64_ad.m4
12864 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12865 
12866 // We can use ubfiz when masking by a positive number and then left shifting the result.
12867 // We know that the mask is positive because immL_bitmask guarantees it.
12868 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12869 %{
12870   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12871   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12872 
12873   ins_cost(INSN_COST);
12874   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12875   ins_encode %{
12876     int lshift = $lshift$$constant & 63;
12877     intptr_t mask = $mask$$constant;
12878     int width = exact_log2_long(mask+1);
12879     __ ubfiz(as_Register($dst$$reg),
12880           as_Register($src$$reg), lshift, width);
12881   %}
12882   ins_pipe(ialu_reg_shift);
12883 %}
12884 
12885 
12886 // This pattern is automatically generated from aarch64_ad.m4
12887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12888 
12889 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12890 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12891 %{
12892   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12893   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12894 
12895   ins_cost(INSN_COST);
12896   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12897   ins_encode %{
12898     int lshift = $lshift$$constant & 63;
12899     intptr_t mask = $mask$$constant;
12900     int width = exact_log2(mask+1);
12901     __ ubfiz(as_Register($dst$$reg),
12902              as_Register($src$$reg), lshift, width);
12903   %}
12904   ins_pipe(ialu_reg_shift);
12905 %}
12906 
12907 // This pattern is automatically generated from aarch64_ad.m4
12908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12909 
12910 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12911 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12912 %{
12913   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12914   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12915 
12916   ins_cost(INSN_COST);
12917   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12918   ins_encode %{
12919     int lshift = $lshift$$constant & 31;
12920     intptr_t mask = $mask$$constant;
12921     int width = exact_log2(mask+1);
12922     __ ubfiz(as_Register($dst$$reg),
12923              as_Register($src$$reg), lshift, width);
12924   %}
12925   ins_pipe(ialu_reg_shift);
12926 %}
12927 
12928 // This pattern is automatically generated from aarch64_ad.m4
12929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12930 
12931 // Can skip int2long conversions after AND with small bitmask
12932 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12933 %{
12934   match(Set dst (ConvI2L (AndI src msk)));
12935   ins_cost(INSN_COST);
12936   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12937   ins_encode %{
12938     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12939   %}
12940   ins_pipe(ialu_reg_shift);
12941 %}
12942 
12943 
12944 // Rotations
12945 // This pattern is automatically generated from aarch64_ad.m4
12946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12947 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12948 %{
12949   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12950   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12951 
12952   ins_cost(INSN_COST);
12953   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12954 
12955   ins_encode %{
12956     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12957             $rshift$$constant & 63);
12958   %}
12959   ins_pipe(ialu_reg_reg_extr);
12960 %}
12961 
12962 
12963 // This pattern is automatically generated from aarch64_ad.m4
12964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12965 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12966 %{
12967   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12968   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12969 
12970   ins_cost(INSN_COST);
12971   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12972 
12973   ins_encode %{
12974     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12975             $rshift$$constant & 31);
12976   %}
12977   ins_pipe(ialu_reg_reg_extr);
12978 %}
12979 
12980 
12981 // This pattern is automatically generated from aarch64_ad.m4
12982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12983 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12984 %{
12985   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12986   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12987 
12988   ins_cost(INSN_COST);
12989   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12990 
12991   ins_encode %{
12992     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12993             $rshift$$constant & 63);
12994   %}
12995   ins_pipe(ialu_reg_reg_extr);
12996 %}
12997 
12998 
12999 // This pattern is automatically generated from aarch64_ad.m4
13000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13001 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13002 %{
13003   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13004   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13005 
13006   ins_cost(INSN_COST);
13007   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13008 
13009   ins_encode %{
13010     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13011             $rshift$$constant & 31);
13012   %}
13013   ins_pipe(ialu_reg_reg_extr);
13014 %}
13015 
13016 
13017 // This pattern is automatically generated from aarch64_ad.m4
13018 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13019 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13020 %{
13021   match(Set dst (RotateRight src shift));
13022 
13023   ins_cost(INSN_COST);
13024   format %{ "ror    $dst, $src, $shift" %}
13025 
13026   ins_encode %{
13027      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13028                $shift$$constant & 0x1f);
13029   %}
13030   ins_pipe(ialu_reg_reg_vshift);
13031 %}
13032 
13033 // This pattern is automatically generated from aarch64_ad.m4
13034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13035 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13036 %{
13037   match(Set dst (RotateRight src shift));
13038 
13039   ins_cost(INSN_COST);
13040   format %{ "ror    $dst, $src, $shift" %}
13041 
13042   ins_encode %{
13043      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13044                $shift$$constant & 0x3f);
13045   %}
13046   ins_pipe(ialu_reg_reg_vshift);
13047 %}
13048 
13049 // This pattern is automatically generated from aarch64_ad.m4
13050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13051 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13052 %{
13053   match(Set dst (RotateRight src shift));
13054 
13055   ins_cost(INSN_COST);
13056   format %{ "ror    $dst, $src, $shift" %}
13057 
13058   ins_encode %{
13059      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13060   %}
13061   ins_pipe(ialu_reg_reg_vshift);
13062 %}
13063 
13064 // This pattern is automatically generated from aarch64_ad.m4
13065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13066 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13067 %{
13068   match(Set dst (RotateRight src shift));
13069 
13070   ins_cost(INSN_COST);
13071   format %{ "ror    $dst, $src, $shift" %}
13072 
13073   ins_encode %{
13074      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13075   %}
13076   ins_pipe(ialu_reg_reg_vshift);
13077 %}
13078 
13079 // This pattern is automatically generated from aarch64_ad.m4
13080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13081 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13082 %{
13083   match(Set dst (RotateLeft src shift));
13084 
13085   ins_cost(INSN_COST);
13086   format %{ "rol    $dst, $src, $shift" %}
13087 
13088   ins_encode %{
13089      __ subw(rscratch1, zr, as_Register($shift$$reg));
13090      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13091   %}
13092   ins_pipe(ialu_reg_reg_vshift);
13093 %}
13094 
13095 // This pattern is automatically generated from aarch64_ad.m4
13096 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13097 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13098 %{
13099   match(Set dst (RotateLeft src shift));
13100 
13101   ins_cost(INSN_COST);
13102   format %{ "rol    $dst, $src, $shift" %}
13103 
13104   ins_encode %{
13105      __ subw(rscratch1, zr, as_Register($shift$$reg));
13106      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13107   %}
13108   ins_pipe(ialu_reg_reg_vshift);
13109 %}
13110 
13111 
13112 // Add/subtract (extended)
13113 
13114 // This pattern is automatically generated from aarch64_ad.m4
13115 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13116 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13117 %{
13118   match(Set dst (AddL src1 (ConvI2L src2)));
13119   ins_cost(INSN_COST);
13120   format %{ "add  $dst, $src1, $src2, sxtw" %}
13121 
13122    ins_encode %{
13123      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13124             as_Register($src2$$reg), ext::sxtw);
13125    %}
13126   ins_pipe(ialu_reg_reg);
13127 %}
13128 
13129 // This pattern is automatically generated from aarch64_ad.m4
13130 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13131 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13132 %{
13133   match(Set dst (SubL src1 (ConvI2L src2)));
13134   ins_cost(INSN_COST);
13135   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13136 
13137    ins_encode %{
13138      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13139             as_Register($src2$$reg), ext::sxtw);
13140    %}
13141   ins_pipe(ialu_reg_reg);
13142 %}
13143 
13144 // This pattern is automatically generated from aarch64_ad.m4
13145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13146 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13147 %{
13148   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13149   ins_cost(INSN_COST);
13150   format %{ "add  $dst, $src1, $src2, sxth" %}
13151 
13152    ins_encode %{
13153      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13154             as_Register($src2$$reg), ext::sxth);
13155    %}
13156   ins_pipe(ialu_reg_reg);
13157 %}
13158 
13159 // This pattern is automatically generated from aarch64_ad.m4
13160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13161 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13162 %{
13163   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13164   ins_cost(INSN_COST);
13165   format %{ "add  $dst, $src1, $src2, sxtb" %}
13166 
13167    ins_encode %{
13168      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13169             as_Register($src2$$reg), ext::sxtb);
13170    %}
13171   ins_pipe(ialu_reg_reg);
13172 %}
13173 
13174 // This pattern is automatically generated from aarch64_ad.m4
13175 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13176 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13177 %{
13178   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13179   ins_cost(INSN_COST);
13180   format %{ "add  $dst, $src1, $src2, uxtb" %}
13181 
13182    ins_encode %{
13183      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13184             as_Register($src2$$reg), ext::uxtb);
13185    %}
13186   ins_pipe(ialu_reg_reg);
13187 %}
13188 
13189 // This pattern is automatically generated from aarch64_ad.m4
13190 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13191 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13192 %{
13193   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13194   ins_cost(INSN_COST);
13195   format %{ "add  $dst, $src1, $src2, sxth" %}
13196 
13197    ins_encode %{
13198      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13199             as_Register($src2$$reg), ext::sxth);
13200    %}
13201   ins_pipe(ialu_reg_reg);
13202 %}
13203 
13204 // This pattern is automatically generated from aarch64_ad.m4
13205 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13206 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13207 %{
13208   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13209   ins_cost(INSN_COST);
13210   format %{ "add  $dst, $src1, $src2, sxtw" %}
13211 
13212    ins_encode %{
13213      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13214             as_Register($src2$$reg), ext::sxtw);
13215    %}
13216   ins_pipe(ialu_reg_reg);
13217 %}
13218 
13219 // This pattern is automatically generated from aarch64_ad.m4
13220 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13221 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13222 %{
13223   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13224   ins_cost(INSN_COST);
13225   format %{ "add  $dst, $src1, $src2, sxtb" %}
13226 
13227    ins_encode %{
13228      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13229             as_Register($src2$$reg), ext::sxtb);
13230    %}
13231   ins_pipe(ialu_reg_reg);
13232 %}
13233 
13234 // This pattern is automatically generated from aarch64_ad.m4
13235 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13236 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13237 %{
13238   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13239   ins_cost(INSN_COST);
13240   format %{ "add  $dst, $src1, $src2, uxtb" %}
13241 
13242    ins_encode %{
13243      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13244             as_Register($src2$$reg), ext::uxtb);
13245    %}
13246   ins_pipe(ialu_reg_reg);
13247 %}
13248 
13249 // This pattern is automatically generated from aarch64_ad.m4
13250 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13251 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13252 %{
13253   match(Set dst (AddI src1 (AndI src2 mask)));
13254   ins_cost(INSN_COST);
13255   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13256 
13257    ins_encode %{
13258      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13259             as_Register($src2$$reg), ext::uxtb);
13260    %}
13261   ins_pipe(ialu_reg_reg);
13262 %}
13263 
13264 // This pattern is automatically generated from aarch64_ad.m4
13265 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13266 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13267 %{
13268   match(Set dst (AddI src1 (AndI src2 mask)));
13269   ins_cost(INSN_COST);
13270   format %{ "addw  $dst, $src1, $src2, uxth" %}
13271 
13272    ins_encode %{
13273      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13274             as_Register($src2$$reg), ext::uxth);
13275    %}
13276   ins_pipe(ialu_reg_reg);
13277 %}
13278 
13279 // This pattern is automatically generated from aarch64_ad.m4
13280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13281 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13282 %{
13283   match(Set dst (AddL src1 (AndL src2 mask)));
13284   ins_cost(INSN_COST);
13285   format %{ "add  $dst, $src1, $src2, uxtb" %}
13286 
13287    ins_encode %{
13288      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13289             as_Register($src2$$reg), ext::uxtb);
13290    %}
13291   ins_pipe(ialu_reg_reg);
13292 %}
13293 
13294 // This pattern is automatically generated from aarch64_ad.m4
13295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13296 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13297 %{
13298   match(Set dst (AddL src1 (AndL src2 mask)));
13299   ins_cost(INSN_COST);
13300   format %{ "add  $dst, $src1, $src2, uxth" %}
13301 
13302    ins_encode %{
13303      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13304             as_Register($src2$$reg), ext::uxth);
13305    %}
13306   ins_pipe(ialu_reg_reg);
13307 %}
13308 
13309 // This pattern is automatically generated from aarch64_ad.m4
13310 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13311 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13312 %{
13313   match(Set dst (AddL src1 (AndL src2 mask)));
13314   ins_cost(INSN_COST);
13315   format %{ "add  $dst, $src1, $src2, uxtw" %}
13316 
13317    ins_encode %{
13318      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13319             as_Register($src2$$reg), ext::uxtw);
13320    %}
13321   ins_pipe(ialu_reg_reg);
13322 %}
13323 
13324 // This pattern is automatically generated from aarch64_ad.m4
13325 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13326 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13327 %{
13328   match(Set dst (SubI src1 (AndI src2 mask)));
13329   ins_cost(INSN_COST);
13330   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13331 
13332    ins_encode %{
13333      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13334             as_Register($src2$$reg), ext::uxtb);
13335    %}
13336   ins_pipe(ialu_reg_reg);
13337 %}
13338 
13339 // This pattern is automatically generated from aarch64_ad.m4
13340 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13341 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13342 %{
13343   match(Set dst (SubI src1 (AndI src2 mask)));
13344   ins_cost(INSN_COST);
13345   format %{ "subw  $dst, $src1, $src2, uxth" %}
13346 
13347    ins_encode %{
13348      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13349             as_Register($src2$$reg), ext::uxth);
13350    %}
13351   ins_pipe(ialu_reg_reg);
13352 %}
13353 
13354 // This pattern is automatically generated from aarch64_ad.m4
13355 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13356 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13357 %{
13358   match(Set dst (SubL src1 (AndL src2 mask)));
13359   ins_cost(INSN_COST);
13360   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13361 
13362    ins_encode %{
13363      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13364             as_Register($src2$$reg), ext::uxtb);
13365    %}
13366   ins_pipe(ialu_reg_reg);
13367 %}
13368 
13369 // This pattern is automatically generated from aarch64_ad.m4
13370 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13371 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13372 %{
13373   match(Set dst (SubL src1 (AndL src2 mask)));
13374   ins_cost(INSN_COST);
13375   format %{ "sub  $dst, $src1, $src2, uxth" %}
13376 
13377    ins_encode %{
13378      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13379             as_Register($src2$$reg), ext::uxth);
13380    %}
13381   ins_pipe(ialu_reg_reg);
13382 %}
13383 
13384 // This pattern is automatically generated from aarch64_ad.m4
13385 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13386 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13387 %{
13388   match(Set dst (SubL src1 (AndL src2 mask)));
13389   ins_cost(INSN_COST);
13390   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13391 
13392    ins_encode %{
13393      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13394             as_Register($src2$$reg), ext::uxtw);
13395    %}
13396   ins_pipe(ialu_reg_reg);
13397 %}
13398 
13399 
13400 // This pattern is automatically generated from aarch64_ad.m4
13401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13402 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13403 %{
13404   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13405   ins_cost(1.9 * INSN_COST);
13406   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13407 
13408    ins_encode %{
13409      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13410             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13411    %}
13412   ins_pipe(ialu_reg_reg_shift);
13413 %}
13414 
13415 // This pattern is automatically generated from aarch64_ad.m4
13416 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13417 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13418 %{
13419   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13420   ins_cost(1.9 * INSN_COST);
13421   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13422 
13423    ins_encode %{
13424      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13425             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13426    %}
13427   ins_pipe(ialu_reg_reg_shift);
13428 %}
13429 
13430 // This pattern is automatically generated from aarch64_ad.m4
13431 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13432 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13433 %{
13434   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13435   ins_cost(1.9 * INSN_COST);
13436   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13437 
13438    ins_encode %{
13439      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13440             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13441    %}
13442   ins_pipe(ialu_reg_reg_shift);
13443 %}
13444 
13445 // This pattern is automatically generated from aarch64_ad.m4
13446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13447 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13448 %{
13449   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13450   ins_cost(1.9 * INSN_COST);
13451   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13452 
13453    ins_encode %{
13454      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13455             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13456    %}
13457   ins_pipe(ialu_reg_reg_shift);
13458 %}
13459 
13460 // This pattern is automatically generated from aarch64_ad.m4
13461 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13462 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13463 %{
13464   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13465   ins_cost(1.9 * INSN_COST);
13466   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13467 
13468    ins_encode %{
13469      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13470             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13471    %}
13472   ins_pipe(ialu_reg_reg_shift);
13473 %}
13474 
13475 // This pattern is automatically generated from aarch64_ad.m4
13476 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13477 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13478 %{
13479   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13480   ins_cost(1.9 * INSN_COST);
13481   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13482 
13483    ins_encode %{
13484      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13485             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13486    %}
13487   ins_pipe(ialu_reg_reg_shift);
13488 %}
13489 
13490 // This pattern is automatically generated from aarch64_ad.m4
13491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13492 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13493 %{
13494   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13495   ins_cost(1.9 * INSN_COST);
13496   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13497 
13498    ins_encode %{
13499      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13500             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13501    %}
13502   ins_pipe(ialu_reg_reg_shift);
13503 %}
13504 
13505 // This pattern is automatically generated from aarch64_ad.m4
13506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13507 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13508 %{
13509   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13510   ins_cost(1.9 * INSN_COST);
13511   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13512 
13513    ins_encode %{
13514      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13515             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13516    %}
13517   ins_pipe(ialu_reg_reg_shift);
13518 %}
13519 
13520 // This pattern is automatically generated from aarch64_ad.m4
13521 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13522 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13523 %{
13524   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13525   ins_cost(1.9 * INSN_COST);
13526   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13527 
13528    ins_encode %{
13529      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13530             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13531    %}
13532   ins_pipe(ialu_reg_reg_shift);
13533 %}
13534 
13535 // This pattern is automatically generated from aarch64_ad.m4
13536 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13537 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13538 %{
13539   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13540   ins_cost(1.9 * INSN_COST);
13541   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13542 
13543    ins_encode %{
13544      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13545             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13546    %}
13547   ins_pipe(ialu_reg_reg_shift);
13548 %}
13549 
13550 // This pattern is automatically generated from aarch64_ad.m4
13551 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13552 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13553 %{
13554   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13555   ins_cost(1.9 * INSN_COST);
13556   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13557 
13558    ins_encode %{
13559      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13560             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13561    %}
13562   ins_pipe(ialu_reg_reg_shift);
13563 %}
13564 
13565 // This pattern is automatically generated from aarch64_ad.m4
13566 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13567 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13568 %{
13569   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13570   ins_cost(1.9 * INSN_COST);
13571   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13572 
13573    ins_encode %{
13574      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13575             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13576    %}
13577   ins_pipe(ialu_reg_reg_shift);
13578 %}
13579 
13580 // This pattern is automatically generated from aarch64_ad.m4
13581 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13582 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13583 %{
13584   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13585   ins_cost(1.9 * INSN_COST);
13586   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13587 
13588    ins_encode %{
13589      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13590             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13591    %}
13592   ins_pipe(ialu_reg_reg_shift);
13593 %}
13594 
13595 // This pattern is automatically generated from aarch64_ad.m4
13596 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13597 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13598 %{
13599   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13600   ins_cost(1.9 * INSN_COST);
13601   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13602 
13603    ins_encode %{
13604      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13605             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13606    %}
13607   ins_pipe(ialu_reg_reg_shift);
13608 %}
13609 
13610 // This pattern is automatically generated from aarch64_ad.m4
13611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13612 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13613 %{
13614   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13615   ins_cost(1.9 * INSN_COST);
13616   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13617 
13618    ins_encode %{
13619      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13620             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13621    %}
13622   ins_pipe(ialu_reg_reg_shift);
13623 %}
13624 
13625 // This pattern is automatically generated from aarch64_ad.m4
13626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13627 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13628 %{
13629   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13630   ins_cost(1.9 * INSN_COST);
13631   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13632 
13633    ins_encode %{
13634      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13635             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13636    %}
13637   ins_pipe(ialu_reg_reg_shift);
13638 %}
13639 
13640 // This pattern is automatically generated from aarch64_ad.m4
13641 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13642 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13643 %{
13644   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13645   ins_cost(1.9 * INSN_COST);
13646   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13647 
13648    ins_encode %{
13649      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13650             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13651    %}
13652   ins_pipe(ialu_reg_reg_shift);
13653 %}
13654 
13655 // This pattern is automatically generated from aarch64_ad.m4
13656 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13657 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13658 %{
13659   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13660   ins_cost(1.9 * INSN_COST);
13661   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13662 
13663    ins_encode %{
13664      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13665             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13666    %}
13667   ins_pipe(ialu_reg_reg_shift);
13668 %}
13669 
13670 // This pattern is automatically generated from aarch64_ad.m4
13671 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13672 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13673 %{
13674   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13675   ins_cost(1.9 * INSN_COST);
13676   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13677 
13678    ins_encode %{
13679      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13680             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13681    %}
13682   ins_pipe(ialu_reg_reg_shift);
13683 %}
13684 
13685 // This pattern is automatically generated from aarch64_ad.m4
13686 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13687 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13688 %{
13689   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13690   ins_cost(1.9 * INSN_COST);
13691   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13692 
13693    ins_encode %{
13694      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13695             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13696    %}
13697   ins_pipe(ialu_reg_reg_shift);
13698 %}
13699 
13700 // This pattern is automatically generated from aarch64_ad.m4
13701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13702 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13703 %{
13704   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13705   ins_cost(1.9 * INSN_COST);
13706   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13707 
13708    ins_encode %{
13709      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13710             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13711    %}
13712   ins_pipe(ialu_reg_reg_shift);
13713 %}
13714 
13715 // This pattern is automatically generated from aarch64_ad.m4
13716 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13717 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13718 %{
13719   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13720   ins_cost(1.9 * INSN_COST);
13721   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13722 
13723    ins_encode %{
13724      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13725             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13726    %}
13727   ins_pipe(ialu_reg_reg_shift);
13728 %}
13729 
13730 
13731 
13732 // END This section of the file is automatically generated. Do not edit --------------
13733 
13734 
13735 // ============================================================================
13736 // Floating Point Arithmetic Instructions
13737 
13738 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13739   match(Set dst (AddF src1 src2));
13740 
13741   ins_cost(INSN_COST * 5);
13742   format %{ "fadds   $dst, $src1, $src2" %}
13743 
13744   ins_encode %{
13745     __ fadds(as_FloatRegister($dst$$reg),
13746              as_FloatRegister($src1$$reg),
13747              as_FloatRegister($src2$$reg));
13748   %}
13749 
13750   ins_pipe(fp_dop_reg_reg_s);
13751 %}
13752 
13753 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13754   match(Set dst (AddD src1 src2));
13755 
13756   ins_cost(INSN_COST * 5);
13757   format %{ "faddd   $dst, $src1, $src2" %}
13758 
13759   ins_encode %{
13760     __ faddd(as_FloatRegister($dst$$reg),
13761              as_FloatRegister($src1$$reg),
13762              as_FloatRegister($src2$$reg));
13763   %}
13764 
13765   ins_pipe(fp_dop_reg_reg_d);
13766 %}
13767 
13768 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13769   match(Set dst (SubF src1 src2));
13770 
13771   ins_cost(INSN_COST * 5);
13772   format %{ "fsubs   $dst, $src1, $src2" %}
13773 
13774   ins_encode %{
13775     __ fsubs(as_FloatRegister($dst$$reg),
13776              as_FloatRegister($src1$$reg),
13777              as_FloatRegister($src2$$reg));
13778   %}
13779 
13780   ins_pipe(fp_dop_reg_reg_s);
13781 %}
13782 
13783 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13784   match(Set dst (SubD src1 src2));
13785 
13786   ins_cost(INSN_COST * 5);
13787   format %{ "fsubd   $dst, $src1, $src2" %}
13788 
13789   ins_encode %{
13790     __ fsubd(as_FloatRegister($dst$$reg),
13791              as_FloatRegister($src1$$reg),
13792              as_FloatRegister($src2$$reg));
13793   %}
13794 
13795   ins_pipe(fp_dop_reg_reg_d);
13796 %}
13797 
13798 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13799   match(Set dst (MulF src1 src2));
13800 
13801   ins_cost(INSN_COST * 6);
13802   format %{ "fmuls   $dst, $src1, $src2" %}
13803 
13804   ins_encode %{
13805     __ fmuls(as_FloatRegister($dst$$reg),
13806              as_FloatRegister($src1$$reg),
13807              as_FloatRegister($src2$$reg));
13808   %}
13809 
13810   ins_pipe(fp_dop_reg_reg_s);
13811 %}
13812 
13813 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13814   match(Set dst (MulD src1 src2));
13815 
13816   ins_cost(INSN_COST * 6);
13817   format %{ "fmuld   $dst, $src1, $src2" %}
13818 
13819   ins_encode %{
13820     __ fmuld(as_FloatRegister($dst$$reg),
13821              as_FloatRegister($src1$$reg),
13822              as_FloatRegister($src2$$reg));
13823   %}
13824 
13825   ins_pipe(fp_dop_reg_reg_d);
13826 %}
13827 
13828 // src1 * src2 + src3
13829 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13830   predicate(UseFMA);
13831   match(Set dst (FmaF src3 (Binary src1 src2)));
13832 
13833   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
13834 
13835   ins_encode %{
13836     __ fmadds(as_FloatRegister($dst$$reg),
13837              as_FloatRegister($src1$$reg),
13838              as_FloatRegister($src2$$reg),
13839              as_FloatRegister($src3$$reg));
13840   %}
13841 
13842   ins_pipe(pipe_class_default);
13843 %}
13844 
13845 // src1 * src2 + src3
13846 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13847   predicate(UseFMA);
13848   match(Set dst (FmaD src3 (Binary src1 src2)));
13849 
13850   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
13851 
13852   ins_encode %{
13853     __ fmaddd(as_FloatRegister($dst$$reg),
13854              as_FloatRegister($src1$$reg),
13855              as_FloatRegister($src2$$reg),
13856              as_FloatRegister($src3$$reg));
13857   %}
13858 
13859   ins_pipe(pipe_class_default);
13860 %}
13861 
13862 // -src1 * src2 + src3
13863 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13864   predicate(UseFMA);
13865   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
13866   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13867 
13868   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
13869 
13870   ins_encode %{
13871     __ fmsubs(as_FloatRegister($dst$$reg),
13872               as_FloatRegister($src1$$reg),
13873               as_FloatRegister($src2$$reg),
13874               as_FloatRegister($src3$$reg));
13875   %}
13876 
13877   ins_pipe(pipe_class_default);
13878 %}
13879 
13880 // -src1 * src2 + src3
13881 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13882   predicate(UseFMA);
13883   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
13884   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13885 
13886   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
13887 
13888   ins_encode %{
13889     __ fmsubd(as_FloatRegister($dst$$reg),
13890               as_FloatRegister($src1$$reg),
13891               as_FloatRegister($src2$$reg),
13892               as_FloatRegister($src3$$reg));
13893   %}
13894 
13895   ins_pipe(pipe_class_default);
13896 %}
13897 
13898 // -src1 * src2 - src3
13899 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13900   predicate(UseFMA);
13901   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
13902   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13903 
13904   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
13905 
13906   ins_encode %{
13907     __ fnmadds(as_FloatRegister($dst$$reg),
13908                as_FloatRegister($src1$$reg),
13909                as_FloatRegister($src2$$reg),
13910                as_FloatRegister($src3$$reg));
13911   %}
13912 
13913   ins_pipe(pipe_class_default);
13914 %}
13915 
13916 // -src1 * src2 - src3
13917 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13918   predicate(UseFMA);
13919   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
13920   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13921 
13922   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
13923 
13924   ins_encode %{
13925     __ fnmaddd(as_FloatRegister($dst$$reg),
13926                as_FloatRegister($src1$$reg),
13927                as_FloatRegister($src2$$reg),
13928                as_FloatRegister($src3$$reg));
13929   %}
13930 
13931   ins_pipe(pipe_class_default);
13932 %}
13933 
13934 // src1 * src2 - src3
13935 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13936   predicate(UseFMA);
13937   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13938 
13939   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
13940 
13941   ins_encode %{
13942     __ fnmsubs(as_FloatRegister($dst$$reg),
13943                as_FloatRegister($src1$$reg),
13944                as_FloatRegister($src2$$reg),
13945                as_FloatRegister($src3$$reg));
13946   %}
13947 
13948   ins_pipe(pipe_class_default);
13949 %}
13950 
13951 // src1 * src2 - src3
13952 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13953   predicate(UseFMA);
13954   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13955 
13956   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
13957 
13958   ins_encode %{
13959   // n.b. insn name should be fnmsubd
13960     __ fnmsub(as_FloatRegister($dst$$reg),
13961               as_FloatRegister($src1$$reg),
13962               as_FloatRegister($src2$$reg),
13963               as_FloatRegister($src3$$reg));
13964   %}
13965 
13966   ins_pipe(pipe_class_default);
13967 %}
13968 
13969 
13970 // Math.max(FF)F
13971 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13972   match(Set dst (MaxF src1 src2));
13973 
13974   format %{ "fmaxs   $dst, $src1, $src2" %}
13975   ins_encode %{
13976     __ fmaxs(as_FloatRegister($dst$$reg),
13977              as_FloatRegister($src1$$reg),
13978              as_FloatRegister($src2$$reg));
13979   %}
13980 
13981   ins_pipe(fp_dop_reg_reg_s);
13982 %}
13983 
13984 // Math.min(FF)F
13985 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13986   match(Set dst (MinF src1 src2));
13987 
13988   format %{ "fmins   $dst, $src1, $src2" %}
13989   ins_encode %{
13990     __ fmins(as_FloatRegister($dst$$reg),
13991              as_FloatRegister($src1$$reg),
13992              as_FloatRegister($src2$$reg));
13993   %}
13994 
13995   ins_pipe(fp_dop_reg_reg_s);
13996 %}
13997 
13998 // Math.max(DD)D
13999 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14000   match(Set dst (MaxD src1 src2));
14001 
14002   format %{ "fmaxd   $dst, $src1, $src2" %}
14003   ins_encode %{
14004     __ fmaxd(as_FloatRegister($dst$$reg),
14005              as_FloatRegister($src1$$reg),
14006              as_FloatRegister($src2$$reg));
14007   %}
14008 
14009   ins_pipe(fp_dop_reg_reg_d);
14010 %}
14011 
14012 // Math.min(DD)D
14013 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14014   match(Set dst (MinD src1 src2));
14015 
14016   format %{ "fmind   $dst, $src1, $src2" %}
14017   ins_encode %{
14018     __ fmind(as_FloatRegister($dst$$reg),
14019              as_FloatRegister($src1$$reg),
14020              as_FloatRegister($src2$$reg));
14021   %}
14022 
14023   ins_pipe(fp_dop_reg_reg_d);
14024 %}
14025 
14026 
14027 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14028   match(Set dst (DivF src1  src2));
14029 
14030   ins_cost(INSN_COST * 18);
14031   format %{ "fdivs   $dst, $src1, $src2" %}
14032 
14033   ins_encode %{
14034     __ fdivs(as_FloatRegister($dst$$reg),
14035              as_FloatRegister($src1$$reg),
14036              as_FloatRegister($src2$$reg));
14037   %}
14038 
14039   ins_pipe(fp_div_s);
14040 %}
14041 
14042 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14043   match(Set dst (DivD src1  src2));
14044 
14045   ins_cost(INSN_COST * 32);
14046   format %{ "fdivd   $dst, $src1, $src2" %}
14047 
14048   ins_encode %{
14049     __ fdivd(as_FloatRegister($dst$$reg),
14050              as_FloatRegister($src1$$reg),
14051              as_FloatRegister($src2$$reg));
14052   %}
14053 
14054   ins_pipe(fp_div_d);
14055 %}
14056 
14057 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14058   match(Set dst (NegF src));
14059 
14060   ins_cost(INSN_COST * 3);
14061   format %{ "fneg   $dst, $src" %}
14062 
14063   ins_encode %{
14064     __ fnegs(as_FloatRegister($dst$$reg),
14065              as_FloatRegister($src$$reg));
14066   %}
14067 
14068   ins_pipe(fp_uop_s);
14069 %}
14070 
14071 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14072   match(Set dst (NegD src));
14073 
14074   ins_cost(INSN_COST * 3);
14075   format %{ "fnegd   $dst, $src" %}
14076 
14077   ins_encode %{
14078     __ fnegd(as_FloatRegister($dst$$reg),
14079              as_FloatRegister($src$$reg));
14080   %}
14081 
14082   ins_pipe(fp_uop_d);
14083 %}
14084 
14085 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14086 %{
14087   match(Set dst (AbsI src));
14088 
14089   effect(KILL cr);
14090   ins_cost(INSN_COST * 2);
14091   format %{ "cmpw  $src, zr\n\t"
14092             "cnegw $dst, $src, Assembler::LT\t# int abs"
14093   %}
14094 
14095   ins_encode %{
14096     __ cmpw(as_Register($src$$reg), zr);
14097     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14098   %}
14099   ins_pipe(pipe_class_default);
14100 %}
14101 
14102 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14103 %{
14104   match(Set dst (AbsL src));
14105 
14106   effect(KILL cr);
14107   ins_cost(INSN_COST * 2);
14108   format %{ "cmp  $src, zr\n\t"
14109             "cneg $dst, $src, Assembler::LT\t# long abs"
14110   %}
14111 
14112   ins_encode %{
14113     __ cmp(as_Register($src$$reg), zr);
14114     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14115   %}
14116   ins_pipe(pipe_class_default);
14117 %}
14118 
14119 instruct absF_reg(vRegF dst, vRegF src) %{
14120   match(Set dst (AbsF src));
14121 
14122   ins_cost(INSN_COST * 3);
14123   format %{ "fabss   $dst, $src" %}
14124   ins_encode %{
14125     __ fabss(as_FloatRegister($dst$$reg),
14126              as_FloatRegister($src$$reg));
14127   %}
14128 
14129   ins_pipe(fp_uop_s);
14130 %}
14131 
14132 instruct absD_reg(vRegD dst, vRegD src) %{
14133   match(Set dst (AbsD src));
14134 
14135   ins_cost(INSN_COST * 3);
14136   format %{ "fabsd   $dst, $src" %}
14137   ins_encode %{
14138     __ fabsd(as_FloatRegister($dst$$reg),
14139              as_FloatRegister($src$$reg));
14140   %}
14141 
14142   ins_pipe(fp_uop_d);
14143 %}
14144 
14145 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14146   match(Set dst (AbsF (SubF src1 src2)));
14147 
14148   ins_cost(INSN_COST * 3);
14149   format %{ "fabds   $dst, $src1, $src2" %}
14150   ins_encode %{
14151     __ fabds(as_FloatRegister($dst$$reg),
14152              as_FloatRegister($src1$$reg),
14153              as_FloatRegister($src2$$reg));
14154   %}
14155 
14156   ins_pipe(fp_uop_s);
14157 %}
14158 
14159 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14160   match(Set dst (AbsD (SubD src1 src2)));
14161 
14162   ins_cost(INSN_COST * 3);
14163   format %{ "fabdd   $dst, $src1, $src2" %}
14164   ins_encode %{
14165     __ fabdd(as_FloatRegister($dst$$reg),
14166              as_FloatRegister($src1$$reg),
14167              as_FloatRegister($src2$$reg));
14168   %}
14169 
14170   ins_pipe(fp_uop_d);
14171 %}
14172 
14173 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14174   match(Set dst (SqrtD src));
14175 
14176   ins_cost(INSN_COST * 50);
14177   format %{ "fsqrtd  $dst, $src" %}
14178   ins_encode %{
14179     __ fsqrtd(as_FloatRegister($dst$$reg),
14180              as_FloatRegister($src$$reg));
14181   %}
14182 
14183   ins_pipe(fp_div_s);
14184 %}
14185 
14186 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14187   match(Set dst (SqrtF src));
14188 
14189   ins_cost(INSN_COST * 50);
14190   format %{ "fsqrts  $dst, $src" %}
14191   ins_encode %{
14192     __ fsqrts(as_FloatRegister($dst$$reg),
14193              as_FloatRegister($src$$reg));
14194   %}
14195 
14196   ins_pipe(fp_div_d);
14197 %}
14198 
14199 // Math.rint, floor, ceil
14200 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14201   match(Set dst (RoundDoubleMode src rmode));
14202   format %{ "frint  $dst, $src, $rmode" %}
14203   ins_encode %{
14204     switch ($rmode$$constant) {
14205       case RoundDoubleModeNode::rmode_rint:
14206         __ frintnd(as_FloatRegister($dst$$reg),
14207                    as_FloatRegister($src$$reg));
14208         break;
14209       case RoundDoubleModeNode::rmode_floor:
14210         __ frintmd(as_FloatRegister($dst$$reg),
14211                    as_FloatRegister($src$$reg));
14212         break;
14213       case RoundDoubleModeNode::rmode_ceil:
14214         __ frintpd(as_FloatRegister($dst$$reg),
14215                    as_FloatRegister($src$$reg));
14216         break;
14217     }
14218   %}
14219   ins_pipe(fp_uop_d);
14220 %}
14221 
14222 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14223   match(Set dst (CopySignD src1 (Binary src2 zero)));
14224   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14225   format %{ "CopySignD  $dst $src1 $src2" %}
14226   ins_encode %{
14227     FloatRegister dst = as_FloatRegister($dst$$reg),
14228                   src1 = as_FloatRegister($src1$$reg),
14229                   src2 = as_FloatRegister($src2$$reg),
14230                   zero = as_FloatRegister($zero$$reg);
14231     __ fnegd(dst, zero);
14232     __ bsl(dst, __ T8B, src2, src1);
14233   %}
14234   ins_pipe(fp_uop_d);
14235 %}
14236 
14237 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14238   match(Set dst (CopySignF src1 src2));
14239   effect(TEMP_DEF dst, USE src1, USE src2);
14240   format %{ "CopySignF  $dst $src1 $src2" %}
14241   ins_encode %{
14242     FloatRegister dst = as_FloatRegister($dst$$reg),
14243                   src1 = as_FloatRegister($src1$$reg),
14244                   src2 = as_FloatRegister($src2$$reg);
14245     __ movi(dst, __ T2S, 0x80, 24);
14246     __ bsl(dst, __ T8B, src2, src1);
14247   %}
14248   ins_pipe(fp_uop_d);
14249 %}
14250 
14251 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14252   match(Set dst (SignumD src (Binary zero one)));
14253   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14254   format %{ "signumD  $dst, $src" %}
14255   ins_encode %{
14256     FloatRegister src = as_FloatRegister($src$$reg),
14257                   dst = as_FloatRegister($dst$$reg),
14258                   zero = as_FloatRegister($zero$$reg),
14259                   one = as_FloatRegister($one$$reg);
14260     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14261     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14262     // Bit selection instruction gets bit from "one" for each enabled bit in
14263     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14264     // NaN the whole "src" will be copied because "dst" is zero. For all other
14265     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14266     // from "src", and all other bits are copied from 1.0.
14267     __ bsl(dst, __ T8B, one, src);
14268   %}
14269   ins_pipe(fp_uop_d);
14270 %}
14271 
14272 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14273   match(Set dst (SignumF src (Binary zero one)));
14274   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14275   format %{ "signumF  $dst, $src" %}
14276   ins_encode %{
14277     FloatRegister src = as_FloatRegister($src$$reg),
14278                   dst = as_FloatRegister($dst$$reg),
14279                   zero = as_FloatRegister($zero$$reg),
14280                   one = as_FloatRegister($one$$reg);
14281     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14282     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14283     // Bit selection instruction gets bit from "one" for each enabled bit in
14284     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14285     // NaN the whole "src" will be copied because "dst" is zero. For all other
14286     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14287     // from "src", and all other bits are copied from 1.0.
14288     __ bsl(dst, __ T8B, one, src);
14289   %}
14290   ins_pipe(fp_uop_d);
14291 %}
14292 
14293 instruct onspinwait() %{
14294   match(OnSpinWait);
14295   ins_cost(INSN_COST);
14296 
14297   format %{ "onspinwait" %}
14298 
14299   ins_encode %{
14300     __ spin_wait();
14301   %}
14302   ins_pipe(pipe_class_empty);
14303 %}
14304 
14305 // ============================================================================
14306 // Logical Instructions
14307 
14308 // Integer Logical Instructions
14309 
14310 // And Instructions
14311 
14312 
14313 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14314   match(Set dst (AndI src1 src2));
14315 
14316   format %{ "andw  $dst, $src1, $src2\t# int" %}
14317 
14318   ins_cost(INSN_COST);
14319   ins_encode %{
14320     __ andw(as_Register($dst$$reg),
14321             as_Register($src1$$reg),
14322             as_Register($src2$$reg));
14323   %}
14324 
14325   ins_pipe(ialu_reg_reg);
14326 %}
14327 
14328 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14329   match(Set dst (AndI src1 src2));
14330 
14331   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14332 
14333   ins_cost(INSN_COST);
14334   ins_encode %{
14335     __ andw(as_Register($dst$$reg),
14336             as_Register($src1$$reg),
14337             (uint64_t)($src2$$constant));
14338   %}
14339 
14340   ins_pipe(ialu_reg_imm);
14341 %}
14342 
14343 // Or Instructions
14344 
14345 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14346   match(Set dst (OrI src1 src2));
14347 
14348   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14349 
14350   ins_cost(INSN_COST);
14351   ins_encode %{
14352     __ orrw(as_Register($dst$$reg),
14353             as_Register($src1$$reg),
14354             as_Register($src2$$reg));
14355   %}
14356 
14357   ins_pipe(ialu_reg_reg);
14358 %}
14359 
14360 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14361   match(Set dst (OrI src1 src2));
14362 
14363   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14364 
14365   ins_cost(INSN_COST);
14366   ins_encode %{
14367     __ orrw(as_Register($dst$$reg),
14368             as_Register($src1$$reg),
14369             (uint64_t)($src2$$constant));
14370   %}
14371 
14372   ins_pipe(ialu_reg_imm);
14373 %}
14374 
14375 // Xor Instructions
14376 
14377 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14378   match(Set dst (XorI src1 src2));
14379 
14380   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14381 
14382   ins_cost(INSN_COST);
14383   ins_encode %{
14384     __ eorw(as_Register($dst$$reg),
14385             as_Register($src1$$reg),
14386             as_Register($src2$$reg));
14387   %}
14388 
14389   ins_pipe(ialu_reg_reg);
14390 %}
14391 
14392 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14393   match(Set dst (XorI src1 src2));
14394 
14395   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14396 
14397   ins_cost(INSN_COST);
14398   ins_encode %{
14399     __ eorw(as_Register($dst$$reg),
14400             as_Register($src1$$reg),
14401             (uint64_t)($src2$$constant));
14402   %}
14403 
14404   ins_pipe(ialu_reg_imm);
14405 %}
14406 
14407 // Long Logical Instructions
14408 // TODO
14409 
14410 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14411   match(Set dst (AndL src1 src2));
14412 
14413   format %{ "and  $dst, $src1, $src2\t# int" %}
14414 
14415   ins_cost(INSN_COST);
14416   ins_encode %{
14417     __ andr(as_Register($dst$$reg),
14418             as_Register($src1$$reg),
14419             as_Register($src2$$reg));
14420   %}
14421 
14422   ins_pipe(ialu_reg_reg);
14423 %}
14424 
14425 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14426   match(Set dst (AndL src1 src2));
14427 
14428   format %{ "and  $dst, $src1, $src2\t# int" %}
14429 
14430   ins_cost(INSN_COST);
14431   ins_encode %{
14432     __ andr(as_Register($dst$$reg),
14433             as_Register($src1$$reg),
14434             (uint64_t)($src2$$constant));
14435   %}
14436 
14437   ins_pipe(ialu_reg_imm);
14438 %}
14439 
14440 // Or Instructions
14441 
14442 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14443   match(Set dst (OrL src1 src2));
14444 
14445   format %{ "orr  $dst, $src1, $src2\t# int" %}
14446 
14447   ins_cost(INSN_COST);
14448   ins_encode %{
14449     __ orr(as_Register($dst$$reg),
14450            as_Register($src1$$reg),
14451            as_Register($src2$$reg));
14452   %}
14453 
14454   ins_pipe(ialu_reg_reg);
14455 %}
14456 
14457 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14458   match(Set dst (OrL src1 src2));
14459 
14460   format %{ "orr  $dst, $src1, $src2\t# int" %}
14461 
14462   ins_cost(INSN_COST);
14463   ins_encode %{
14464     __ orr(as_Register($dst$$reg),
14465            as_Register($src1$$reg),
14466            (uint64_t)($src2$$constant));
14467   %}
14468 
14469   ins_pipe(ialu_reg_imm);
14470 %}
14471 
14472 // Xor Instructions
14473 
14474 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14475   match(Set dst (XorL src1 src2));
14476 
14477   format %{ "eor  $dst, $src1, $src2\t# int" %}
14478 
14479   ins_cost(INSN_COST);
14480   ins_encode %{
14481     __ eor(as_Register($dst$$reg),
14482            as_Register($src1$$reg),
14483            as_Register($src2$$reg));
14484   %}
14485 
14486   ins_pipe(ialu_reg_reg);
14487 %}
14488 
14489 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14490   match(Set dst (XorL src1 src2));
14491 
14492   ins_cost(INSN_COST);
14493   format %{ "eor  $dst, $src1, $src2\t# int" %}
14494 
14495   ins_encode %{
14496     __ eor(as_Register($dst$$reg),
14497            as_Register($src1$$reg),
14498            (uint64_t)($src2$$constant));
14499   %}
14500 
14501   ins_pipe(ialu_reg_imm);
14502 %}
14503 
14504 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14505 %{
14506   match(Set dst (ConvI2L src));
14507 
14508   ins_cost(INSN_COST);
14509   format %{ "sxtw  $dst, $src\t# i2l" %}
14510   ins_encode %{
14511     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14512   %}
14513   ins_pipe(ialu_reg_shift);
14514 %}
14515 
14516 // this pattern occurs in bigmath arithmetic
14517 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14518 %{
14519   match(Set dst (AndL (ConvI2L src) mask));
14520 
14521   ins_cost(INSN_COST);
14522   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14523   ins_encode %{
14524     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14525   %}
14526 
14527   ins_pipe(ialu_reg_shift);
14528 %}
14529 
14530 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14531   match(Set dst (ConvL2I src));
14532 
14533   ins_cost(INSN_COST);
14534   format %{ "movw  $dst, $src \t// l2i" %}
14535 
14536   ins_encode %{
14537     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14538   %}
14539 
14540   ins_pipe(ialu_reg);
14541 %}
14542 
14543 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14544 %{
14545   match(Set dst (Conv2B src));
14546   effect(KILL cr);
14547 
14548   format %{
14549     "cmpw $src, zr\n\t"
14550     "cset $dst, ne"
14551   %}
14552 
14553   ins_encode %{
14554     __ cmpw(as_Register($src$$reg), zr);
14555     __ cset(as_Register($dst$$reg), Assembler::NE);
14556   %}
14557 
14558   ins_pipe(ialu_reg);
14559 %}
14560 
14561 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14562 %{
14563   match(Set dst (Conv2B src));
14564   effect(KILL cr);
14565 
14566   format %{
14567     "cmp  $src, zr\n\t"
14568     "cset $dst, ne"
14569   %}
14570 
14571   ins_encode %{
14572     __ cmp(as_Register($src$$reg), zr);
14573     __ cset(as_Register($dst$$reg), Assembler::NE);
14574   %}
14575 
14576   ins_pipe(ialu_reg);
14577 %}
14578 
14579 instruct convD2F_reg(vRegF dst, vRegD src) %{
14580   match(Set dst (ConvD2F src));
14581 
14582   ins_cost(INSN_COST * 5);
14583   format %{ "fcvtd  $dst, $src \t// d2f" %}
14584 
14585   ins_encode %{
14586     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14587   %}
14588 
14589   ins_pipe(fp_d2f);
14590 %}
14591 
14592 instruct convF2D_reg(vRegD dst, vRegF src) %{
14593   match(Set dst (ConvF2D src));
14594 
14595   ins_cost(INSN_COST * 5);
14596   format %{ "fcvts  $dst, $src \t// f2d" %}
14597 
14598   ins_encode %{
14599     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14600   %}
14601 
14602   ins_pipe(fp_f2d);
14603 %}
14604 
14605 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14606   match(Set dst (ConvF2I src));
14607 
14608   ins_cost(INSN_COST * 5);
14609   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14610 
14611   ins_encode %{
14612     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14613   %}
14614 
14615   ins_pipe(fp_f2i);
14616 %}
14617 
14618 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14619   match(Set dst (ConvF2L src));
14620 
14621   ins_cost(INSN_COST * 5);
14622   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14623 
14624   ins_encode %{
14625     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14626   %}
14627 
14628   ins_pipe(fp_f2l);
14629 %}
14630 
14631 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14632   match(Set dst (ConvI2F src));
14633 
14634   ins_cost(INSN_COST * 5);
14635   format %{ "scvtfws  $dst, $src \t// i2f" %}
14636 
14637   ins_encode %{
14638     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14639   %}
14640 
14641   ins_pipe(fp_i2f);
14642 %}
14643 
14644 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14645   match(Set dst (ConvL2F src));
14646 
14647   ins_cost(INSN_COST * 5);
14648   format %{ "scvtfs  $dst, $src \t// l2f" %}
14649 
14650   ins_encode %{
14651     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14652   %}
14653 
14654   ins_pipe(fp_l2f);
14655 %}
14656 
14657 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14658   match(Set dst (ConvD2I src));
14659 
14660   ins_cost(INSN_COST * 5);
14661   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14662 
14663   ins_encode %{
14664     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14665   %}
14666 
14667   ins_pipe(fp_d2i);
14668 %}
14669 
14670 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14671   match(Set dst (ConvD2L src));
14672 
14673   ins_cost(INSN_COST * 5);
14674   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14675 
14676   ins_encode %{
14677     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14678   %}
14679 
14680   ins_pipe(fp_d2l);
14681 %}
14682 
14683 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14684   match(Set dst (ConvI2D src));
14685 
14686   ins_cost(INSN_COST * 5);
14687   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14688 
14689   ins_encode %{
14690     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14691   %}
14692 
14693   ins_pipe(fp_i2d);
14694 %}
14695 
14696 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14697   match(Set dst (ConvL2D src));
14698 
14699   ins_cost(INSN_COST * 5);
14700   format %{ "scvtfd  $dst, $src \t// l2d" %}
14701 
14702   ins_encode %{
14703     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14704   %}
14705 
14706   ins_pipe(fp_l2d);
14707 %}
14708 
14709 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14710 %{
14711   match(Set dst (RoundD src));
14712   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14713   format %{ "java_round_double $dst,$src"%}
14714   ins_encode %{
14715     __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14716                          as_FloatRegister($ftmp$$reg));
14717   %}
14718   ins_pipe(pipe_slow);
14719 %}
14720 
14721 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14722 %{
14723   match(Set dst (RoundF src));
14724   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14725   format %{ "java_round_float $dst,$src"%}
14726   ins_encode %{
14727     __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14728                         as_FloatRegister($ftmp$$reg));
14729   %}
14730   ins_pipe(pipe_slow);
14731 %}
14732 
14733 // stack <-> reg and reg <-> reg shuffles with no conversion
14734 
14735 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14736 
14737   match(Set dst (MoveF2I src));
14738 
14739   effect(DEF dst, USE src);
14740 
14741   ins_cost(4 * INSN_COST);
14742 
14743   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14744 
14745   ins_encode %{
14746     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14747   %}
14748 
14749   ins_pipe(iload_reg_reg);
14750 
14751 %}
14752 
14753 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14754 
14755   match(Set dst (MoveI2F src));
14756 
14757   effect(DEF dst, USE src);
14758 
14759   ins_cost(4 * INSN_COST);
14760 
14761   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14762 
14763   ins_encode %{
14764     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14765   %}
14766 
14767   ins_pipe(pipe_class_memory);
14768 
14769 %}
14770 
14771 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14772 
14773   match(Set dst (MoveD2L src));
14774 
14775   effect(DEF dst, USE src);
14776 
14777   ins_cost(4 * INSN_COST);
14778 
14779   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14780 
14781   ins_encode %{
14782     __ ldr($dst$$Register, Address(sp, $src$$disp));
14783   %}
14784 
14785   ins_pipe(iload_reg_reg);
14786 
14787 %}
14788 
14789 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14790 
14791   match(Set dst (MoveL2D src));
14792 
14793   effect(DEF dst, USE src);
14794 
14795   ins_cost(4 * INSN_COST);
14796 
14797   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14798 
14799   ins_encode %{
14800     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14801   %}
14802 
14803   ins_pipe(pipe_class_memory);
14804 
14805 %}
14806 
14807 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14808 
14809   match(Set dst (MoveF2I src));
14810 
14811   effect(DEF dst, USE src);
14812 
14813   ins_cost(INSN_COST);
14814 
14815   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14816 
14817   ins_encode %{
14818     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14819   %}
14820 
14821   ins_pipe(pipe_class_memory);
14822 
14823 %}
14824 
14825 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14826 
14827   match(Set dst (MoveI2F src));
14828 
14829   effect(DEF dst, USE src);
14830 
14831   ins_cost(INSN_COST);
14832 
14833   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14834 
14835   ins_encode %{
14836     __ strw($src$$Register, Address(sp, $dst$$disp));
14837   %}
14838 
14839   ins_pipe(istore_reg_reg);
14840 
14841 %}
14842 
14843 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14844 
14845   match(Set dst (MoveD2L src));
14846 
14847   effect(DEF dst, USE src);
14848 
14849   ins_cost(INSN_COST);
14850 
14851   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14852 
14853   ins_encode %{
14854     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14855   %}
14856 
14857   ins_pipe(pipe_class_memory);
14858 
14859 %}
14860 
14861 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14862 
14863   match(Set dst (MoveL2D src));
14864 
14865   effect(DEF dst, USE src);
14866 
14867   ins_cost(INSN_COST);
14868 
14869   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14870 
14871   ins_encode %{
14872     __ str($src$$Register, Address(sp, $dst$$disp));
14873   %}
14874 
14875   ins_pipe(istore_reg_reg);
14876 
14877 %}
14878 
14879 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14880 
14881   match(Set dst (MoveF2I src));
14882 
14883   effect(DEF dst, USE src);
14884 
14885   ins_cost(INSN_COST);
14886 
14887   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14888 
14889   ins_encode %{
14890     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14891   %}
14892 
14893   ins_pipe(fp_f2i);
14894 
14895 %}
14896 
14897 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14898 
14899   match(Set dst (MoveI2F src));
14900 
14901   effect(DEF dst, USE src);
14902 
14903   ins_cost(INSN_COST);
14904 
14905   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14906 
14907   ins_encode %{
14908     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14909   %}
14910 
14911   ins_pipe(fp_i2f);
14912 
14913 %}
14914 
14915 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14916 
14917   match(Set dst (MoveD2L src));
14918 
14919   effect(DEF dst, USE src);
14920 
14921   ins_cost(INSN_COST);
14922 
14923   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14924 
14925   ins_encode %{
14926     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14927   %}
14928 
14929   ins_pipe(fp_d2l);
14930 
14931 %}
14932 
14933 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14934 
14935   match(Set dst (MoveL2D src));
14936 
14937   effect(DEF dst, USE src);
14938 
14939   ins_cost(INSN_COST);
14940 
14941   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14942 
14943   ins_encode %{
14944     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14945   %}
14946 
14947   ins_pipe(fp_l2d);
14948 
14949 %}
14950 
14951 // ============================================================================
14952 // clearing of an array
14953 
14954 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
14955 %{
14956   match(Set dummy (ClearArray (Binary cnt base) zero));
14957   effect(USE_KILL cnt, USE_KILL base, KILL cr);
14958 
14959   ins_cost(4 * INSN_COST);
14960   format %{ "ClearArray $cnt, $base" %}
14961 
14962   ins_encode %{
14963     address tpc = __ zero_words($base$$Register, $cnt$$Register);
14964     if (tpc == NULL) {
14965       ciEnv::current()->record_failure("CodeCache is full");
14966       return;
14967     }
14968   %}
14969 
14970   ins_pipe(pipe_class_memory);
14971 %}
14972 
14973 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
14974 %{
14975   predicate(((ClearArrayNode*)n)->word_copy_only());
14976   match(Set dummy (ClearArray (Binary cnt base) val));
14977   effect(USE_KILL cnt, USE_KILL base, KILL cr);
14978 
14979   ins_cost(4 * INSN_COST);
14980   format %{ "ClearArray $cnt, $base, $val" %}
14981 
14982   ins_encode %{
14983     __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
14984   %}
14985 
14986   ins_pipe(pipe_class_memory);
14987 %}
14988 
14989 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14990 %{
14991   predicate((uint64_t)n->in(2)->get_long()
14992             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
14993             && !((ClearArrayNode*)n)->word_copy_only());
14994   match(Set dummy (ClearArray cnt base));
14995   effect(TEMP temp, USE_KILL base, KILL cr);
14996 
14997   ins_cost(4 * INSN_COST);
14998   format %{ "ClearArray $cnt, $base" %}
14999 
15000   ins_encode %{
15001     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15002     if (tpc == NULL) {
15003       ciEnv::current()->record_failure("CodeCache is full");
15004       return;
15005     }
15006   %}
15007 
15008   ins_pipe(pipe_class_memory);
15009 %}
15010 
15011 // ============================================================================
15012 // Overflow Math Instructions
15013 
15014 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15015 %{
15016   match(Set cr (OverflowAddI op1 op2));
15017 
15018   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15019   ins_cost(INSN_COST);
15020   ins_encode %{
15021     __ cmnw($op1$$Register, $op2$$Register);
15022   %}
15023 
15024   ins_pipe(icmp_reg_reg);
15025 %}
15026 
15027 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15028 %{
15029   match(Set cr (OverflowAddI op1 op2));
15030 
15031   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15032   ins_cost(INSN_COST);
15033   ins_encode %{
15034     __ cmnw($op1$$Register, $op2$$constant);
15035   %}
15036 
15037   ins_pipe(icmp_reg_imm);
15038 %}
15039 
15040 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15041 %{
15042   match(Set cr (OverflowAddL op1 op2));
15043 
15044   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15045   ins_cost(INSN_COST);
15046   ins_encode %{
15047     __ cmn($op1$$Register, $op2$$Register);
15048   %}
15049 
15050   ins_pipe(icmp_reg_reg);
15051 %}
15052 
15053 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15054 %{
15055   match(Set cr (OverflowAddL op1 op2));
15056 
15057   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
15058   ins_cost(INSN_COST);
15059   ins_encode %{
15060     __ adds(zr, $op1$$Register, $op2$$constant);
15061   %}
15062 
15063   ins_pipe(icmp_reg_imm);
15064 %}
15065 
15066 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15067 %{
15068   match(Set cr (OverflowSubI op1 op2));
15069 
15070   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15071   ins_cost(INSN_COST);
15072   ins_encode %{
15073     __ cmpw($op1$$Register, $op2$$Register);
15074   %}
15075 
15076   ins_pipe(icmp_reg_reg);
15077 %}
15078 
15079 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15080 %{
15081   match(Set cr (OverflowSubI op1 op2));
15082 
15083   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15084   ins_cost(INSN_COST);
15085   ins_encode %{
15086     __ cmpw($op1$$Register, $op2$$constant);
15087   %}
15088 
15089   ins_pipe(icmp_reg_imm);
15090 %}
15091 
15092 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15093 %{
15094   match(Set cr (OverflowSubL op1 op2));
15095 
15096   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15097   ins_cost(INSN_COST);
15098   ins_encode %{
15099     __ cmp($op1$$Register, $op2$$Register);
15100   %}
15101 
15102   ins_pipe(icmp_reg_reg);
15103 %}
15104 
15105 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15106 %{
15107   match(Set cr (OverflowSubL op1 op2));
15108 
15109   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15110   ins_cost(INSN_COST);
15111   ins_encode %{
15112     __ subs(zr, $op1$$Register, $op2$$constant);
15113   %}
15114 
15115   ins_pipe(icmp_reg_imm);
15116 %}
15117 
15118 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15119 %{
15120   match(Set cr (OverflowSubI zero op1));
15121 
15122   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15123   ins_cost(INSN_COST);
15124   ins_encode %{
15125     __ cmpw(zr, $op1$$Register);
15126   %}
15127 
15128   ins_pipe(icmp_reg_imm);
15129 %}
15130 
15131 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15132 %{
15133   match(Set cr (OverflowSubL zero op1));
15134 
15135   format %{ "cmp   zr, $op1\t# overflow check long" %}
15136   ins_cost(INSN_COST);
15137   ins_encode %{
15138     __ cmp(zr, $op1$$Register);
15139   %}
15140 
15141   ins_pipe(icmp_reg_imm);
15142 %}
15143 
15144 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15145 %{
15146   match(Set cr (OverflowMulI op1 op2));
15147 
15148   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15149             "cmp   rscratch1, rscratch1, sxtw\n\t"
15150             "movw  rscratch1, #0x80000000\n\t"
15151             "cselw rscratch1, rscratch1, zr, NE\n\t"
15152             "cmpw  rscratch1, #1" %}
15153   ins_cost(5 * INSN_COST);
15154   ins_encode %{
15155     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15156     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15157     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15158     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15159     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15160   %}
15161 
15162   ins_pipe(pipe_slow);
15163 %}
15164 
15165 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15166 %{
15167   match(If cmp (OverflowMulI op1 op2));
15168   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15169             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15170   effect(USE labl, KILL cr);
15171 
15172   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15173             "cmp   rscratch1, rscratch1, sxtw\n\t"
15174             "b$cmp   $labl" %}
15175   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15176   ins_encode %{
15177     Label* L = $labl$$label;
15178     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15179     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15180     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15181     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15182   %}
15183 
15184   ins_pipe(pipe_serial);
15185 %}
15186 
15187 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15188 %{
15189   match(Set cr (OverflowMulL op1 op2));
15190 
15191   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15192             "smulh rscratch2, $op1, $op2\n\t"
15193             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15194             "movw  rscratch1, #0x80000000\n\t"
15195             "cselw rscratch1, rscratch1, zr, NE\n\t"
15196             "cmpw  rscratch1, #1" %}
15197   ins_cost(6 * INSN_COST);
15198   ins_encode %{
15199     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15200     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15201     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15202     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15203     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15204     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15205   %}
15206 
15207   ins_pipe(pipe_slow);
15208 %}
15209 
15210 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15211 %{
15212   match(If cmp (OverflowMulL op1 op2));
15213   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15214             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15215   effect(USE labl, KILL cr);
15216 
15217   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15218             "smulh rscratch2, $op1, $op2\n\t"
15219             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15220             "b$cmp $labl" %}
15221   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15222   ins_encode %{
15223     Label* L = $labl$$label;
15224     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15225     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15226     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15227     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15228     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15229   %}
15230 
15231   ins_pipe(pipe_serial);
15232 %}
15233 
15234 // ============================================================================
15235 // Compare Instructions
15236 
15237 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15238 %{
15239   match(Set cr (CmpI op1 op2));
15240 
15241   effect(DEF cr, USE op1, USE op2);
15242 
15243   ins_cost(INSN_COST);
15244   format %{ "cmpw  $op1, $op2" %}
15245 
15246   ins_encode(aarch64_enc_cmpw(op1, op2));
15247 
15248   ins_pipe(icmp_reg_reg);
15249 %}
15250 
15251 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15252 %{
15253   match(Set cr (CmpI op1 zero));
15254 
15255   effect(DEF cr, USE op1);
15256 
15257   ins_cost(INSN_COST);
15258   format %{ "cmpw $op1, 0" %}
15259 
15260   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15261 
15262   ins_pipe(icmp_reg_imm);
15263 %}
15264 
15265 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15266 %{
15267   match(Set cr (CmpI op1 op2));
15268 
15269   effect(DEF cr, USE op1);
15270 
15271   ins_cost(INSN_COST);
15272   format %{ "cmpw  $op1, $op2" %}
15273 
15274   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15275 
15276   ins_pipe(icmp_reg_imm);
15277 %}
15278 
15279 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15280 %{
15281   match(Set cr (CmpI op1 op2));
15282 
15283   effect(DEF cr, USE op1);
15284 
15285   ins_cost(INSN_COST * 2);
15286   format %{ "cmpw  $op1, $op2" %}
15287 
15288   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15289 
15290   ins_pipe(icmp_reg_imm);
15291 %}
15292 
15293 // Unsigned compare Instructions; really, same as signed compare
15294 // except it should only be used to feed an If or a CMovI which takes a
15295 // cmpOpU.
15296 
15297 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15298 %{
15299   match(Set cr (CmpU op1 op2));
15300 
15301   effect(DEF cr, USE op1, USE op2);
15302 
15303   ins_cost(INSN_COST);
15304   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15305 
15306   ins_encode(aarch64_enc_cmpw(op1, op2));
15307 
15308   ins_pipe(icmp_reg_reg);
15309 %}
15310 
15311 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15312 %{
15313   match(Set cr (CmpU op1 zero));
15314 
15315   effect(DEF cr, USE op1);
15316 
15317   ins_cost(INSN_COST);
15318   format %{ "cmpw $op1, #0\t# unsigned" %}
15319 
15320   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15321 
15322   ins_pipe(icmp_reg_imm);
15323 %}
15324 
15325 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15326 %{
15327   match(Set cr (CmpU op1 op2));
15328 
15329   effect(DEF cr, USE op1);
15330 
15331   ins_cost(INSN_COST);
15332   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15333 
15334   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15335 
15336   ins_pipe(icmp_reg_imm);
15337 %}
15338 
15339 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15340 %{
15341   match(Set cr (CmpU op1 op2));
15342 
15343   effect(DEF cr, USE op1);
15344 
15345   ins_cost(INSN_COST * 2);
15346   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15347 
15348   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15349 
15350   ins_pipe(icmp_reg_imm);
15351 %}
15352 
15353 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15354 %{
15355   match(Set cr (CmpL op1 op2));
15356 
15357   effect(DEF cr, USE op1, USE op2);
15358 
15359   ins_cost(INSN_COST);
15360   format %{ "cmp  $op1, $op2" %}
15361 
15362   ins_encode(aarch64_enc_cmp(op1, op2));
15363 
15364   ins_pipe(icmp_reg_reg);
15365 %}
15366 
15367 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15368 %{
15369   match(Set cr (CmpL op1 zero));
15370 
15371   effect(DEF cr, USE op1);
15372 
15373   ins_cost(INSN_COST);
15374   format %{ "tst  $op1" %}
15375 
15376   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15377 
15378   ins_pipe(icmp_reg_imm);
15379 %}
15380 
15381 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15382 %{
15383   match(Set cr (CmpL op1 op2));
15384 
15385   effect(DEF cr, USE op1);
15386 
15387   ins_cost(INSN_COST);
15388   format %{ "cmp  $op1, $op2" %}
15389 
15390   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15391 
15392   ins_pipe(icmp_reg_imm);
15393 %}
15394 
15395 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15396 %{
15397   match(Set cr (CmpL op1 op2));
15398 
15399   effect(DEF cr, USE op1);
15400 
15401   ins_cost(INSN_COST * 2);
15402   format %{ "cmp  $op1, $op2" %}
15403 
15404   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15405 
15406   ins_pipe(icmp_reg_imm);
15407 %}
15408 
15409 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15410 %{
15411   match(Set cr (CmpUL op1 op2));
15412 
15413   effect(DEF cr, USE op1, USE op2);
15414 
15415   ins_cost(INSN_COST);
15416   format %{ "cmp  $op1, $op2" %}
15417 
15418   ins_encode(aarch64_enc_cmp(op1, op2));
15419 
15420   ins_pipe(icmp_reg_reg);
15421 %}
15422 
15423 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15424 %{
15425   match(Set cr (CmpUL op1 zero));
15426 
15427   effect(DEF cr, USE op1);
15428 
15429   ins_cost(INSN_COST);
15430   format %{ "tst  $op1" %}
15431 
15432   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15433 
15434   ins_pipe(icmp_reg_imm);
15435 %}
15436 
15437 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15438 %{
15439   match(Set cr (CmpUL op1 op2));
15440 
15441   effect(DEF cr, USE op1);
15442 
15443   ins_cost(INSN_COST);
15444   format %{ "cmp  $op1, $op2" %}
15445 
15446   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15447 
15448   ins_pipe(icmp_reg_imm);
15449 %}
15450 
15451 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15452 %{
15453   match(Set cr (CmpUL op1 op2));
15454 
15455   effect(DEF cr, USE op1);
15456 
15457   ins_cost(INSN_COST * 2);
15458   format %{ "cmp  $op1, $op2" %}
15459 
15460   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15461 
15462   ins_pipe(icmp_reg_imm);
15463 %}
15464 
15465 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15466 %{
15467   match(Set cr (CmpP op1 op2));
15468 
15469   effect(DEF cr, USE op1, USE op2);
15470 
15471   ins_cost(INSN_COST);
15472   format %{ "cmp  $op1, $op2\t // ptr" %}
15473 
15474   ins_encode(aarch64_enc_cmpp(op1, op2));
15475 
15476   ins_pipe(icmp_reg_reg);
15477 %}
15478 
15479 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15480 %{
15481   match(Set cr (CmpN op1 op2));
15482 
15483   effect(DEF cr, USE op1, USE op2);
15484 
15485   ins_cost(INSN_COST);
15486   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15487 
15488   ins_encode(aarch64_enc_cmpn(op1, op2));
15489 
15490   ins_pipe(icmp_reg_reg);
15491 %}
15492 
15493 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15494 %{
15495   match(Set cr (CmpP op1 zero));
15496 
15497   effect(DEF cr, USE op1, USE zero);
15498 
15499   ins_cost(INSN_COST);
15500   format %{ "cmp  $op1, 0\t // ptr" %}
15501 
15502   ins_encode(aarch64_enc_testp(op1));
15503 
15504   ins_pipe(icmp_reg_imm);
15505 %}
15506 
15507 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15508 %{
15509   match(Set cr (CmpN op1 zero));
15510 
15511   effect(DEF cr, USE op1, USE zero);
15512 
15513   ins_cost(INSN_COST);
15514   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15515 
15516   ins_encode(aarch64_enc_testn(op1));
15517 
15518   ins_pipe(icmp_reg_imm);
15519 %}
15520 
15521 // FP comparisons
15522 //
15523 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15524 // using normal cmpOp. See declaration of rFlagsReg for details.
15525 
15526 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15527 %{
15528   match(Set cr (CmpF src1 src2));
15529 
15530   ins_cost(3 * INSN_COST);
15531   format %{ "fcmps $src1, $src2" %}
15532 
15533   ins_encode %{
15534     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15535   %}
15536 
15537   ins_pipe(pipe_class_compare);
15538 %}
15539 
15540 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15541 %{
15542   match(Set cr (CmpF src1 src2));
15543 
15544   ins_cost(3 * INSN_COST);
15545   format %{ "fcmps $src1, 0.0" %}
15546 
15547   ins_encode %{
15548     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15549   %}
15550 
15551   ins_pipe(pipe_class_compare);
15552 %}
15553 // FROM HERE
15554 
15555 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15556 %{
15557   match(Set cr (CmpD src1 src2));
15558 
15559   ins_cost(3 * INSN_COST);
15560   format %{ "fcmpd $src1, $src2" %}
15561 
15562   ins_encode %{
15563     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15564   %}
15565 
15566   ins_pipe(pipe_class_compare);
15567 %}
15568 
15569 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15570 %{
15571   match(Set cr (CmpD src1 src2));
15572 
15573   ins_cost(3 * INSN_COST);
15574   format %{ "fcmpd $src1, 0.0" %}
15575 
15576   ins_encode %{
15577     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15578   %}
15579 
15580   ins_pipe(pipe_class_compare);
15581 %}
15582 
15583 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15584 %{
15585   match(Set dst (CmpF3 src1 src2));
15586   effect(KILL cr);
15587 
15588   ins_cost(5 * INSN_COST);
15589   format %{ "fcmps $src1, $src2\n\t"
15590             "csinvw($dst, zr, zr, eq\n\t"
15591             "csnegw($dst, $dst, $dst, lt)"
15592   %}
15593 
15594   ins_encode %{
15595     Label done;
15596     FloatRegister s1 = as_FloatRegister($src1$$reg);
15597     FloatRegister s2 = as_FloatRegister($src2$$reg);
15598     Register d = as_Register($dst$$reg);
15599     __ fcmps(s1, s2);
15600     // installs 0 if EQ else -1
15601     __ csinvw(d, zr, zr, Assembler::EQ);
15602     // keeps -1 if less or unordered else installs 1
15603     __ csnegw(d, d, d, Assembler::LT);
15604     __ bind(done);
15605   %}
15606 
15607   ins_pipe(pipe_class_default);
15608 
15609 %}
15610 
15611 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15612 %{
15613   match(Set dst (CmpD3 src1 src2));
15614   effect(KILL cr);
15615 
15616   ins_cost(5 * INSN_COST);
15617   format %{ "fcmpd $src1, $src2\n\t"
15618             "csinvw($dst, zr, zr, eq\n\t"
15619             "csnegw($dst, $dst, $dst, lt)"
15620   %}
15621 
15622   ins_encode %{
15623     Label done;
15624     FloatRegister s1 = as_FloatRegister($src1$$reg);
15625     FloatRegister s2 = as_FloatRegister($src2$$reg);
15626     Register d = as_Register($dst$$reg);
15627     __ fcmpd(s1, s2);
15628     // installs 0 if EQ else -1
15629     __ csinvw(d, zr, zr, Assembler::EQ);
15630     // keeps -1 if less or unordered else installs 1
15631     __ csnegw(d, d, d, Assembler::LT);
15632     __ bind(done);
15633   %}
15634   ins_pipe(pipe_class_default);
15635 
15636 %}
15637 
15638 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15639 %{
15640   match(Set dst (CmpF3 src1 zero));
15641   effect(KILL cr);
15642 
15643   ins_cost(5 * INSN_COST);
15644   format %{ "fcmps $src1, 0.0\n\t"
15645             "csinvw($dst, zr, zr, eq\n\t"
15646             "csnegw($dst, $dst, $dst, lt)"
15647   %}
15648 
15649   ins_encode %{
15650     Label done;
15651     FloatRegister s1 = as_FloatRegister($src1$$reg);
15652     Register d = as_Register($dst$$reg);
15653     __ fcmps(s1, 0.0);
15654     // installs 0 if EQ else -1
15655     __ csinvw(d, zr, zr, Assembler::EQ);
15656     // keeps -1 if less or unordered else installs 1
15657     __ csnegw(d, d, d, Assembler::LT);
15658     __ bind(done);
15659   %}
15660 
15661   ins_pipe(pipe_class_default);
15662 
15663 %}
15664 
15665 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15666 %{
15667   match(Set dst (CmpD3 src1 zero));
15668   effect(KILL cr);
15669 
15670   ins_cost(5 * INSN_COST);
15671   format %{ "fcmpd $src1, 0.0\n\t"
15672             "csinvw($dst, zr, zr, eq\n\t"
15673             "csnegw($dst, $dst, $dst, lt)"
15674   %}
15675 
15676   ins_encode %{
15677     Label done;
15678     FloatRegister s1 = as_FloatRegister($src1$$reg);
15679     Register d = as_Register($dst$$reg);
15680     __ fcmpd(s1, 0.0);
15681     // installs 0 if EQ else -1
15682     __ csinvw(d, zr, zr, Assembler::EQ);
15683     // keeps -1 if less or unordered else installs 1
15684     __ csnegw(d, d, d, Assembler::LT);
15685     __ bind(done);
15686   %}
15687   ins_pipe(pipe_class_default);
15688 
15689 %}
15690 
15691 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15692 %{
15693   match(Set dst (CmpLTMask p q));
15694   effect(KILL cr);
15695 
15696   ins_cost(3 * INSN_COST);
15697 
15698   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15699             "csetw $dst, lt\n\t"
15700             "subw $dst, zr, $dst"
15701   %}
15702 
15703   ins_encode %{
15704     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15705     __ csetw(as_Register($dst$$reg), Assembler::LT);
15706     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15707   %}
15708 
15709   ins_pipe(ialu_reg_reg);
15710 %}
15711 
15712 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15713 %{
15714   match(Set dst (CmpLTMask src zero));
15715   effect(KILL cr);
15716 
15717   ins_cost(INSN_COST);
15718 
15719   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15720 
15721   ins_encode %{
15722     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15723   %}
15724 
15725   ins_pipe(ialu_reg_shift);
15726 %}
15727 
15728 // ============================================================================
15729 // Max and Min
15730 
15731 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15732 %{
15733   effect( DEF dst, USE src1, USE src2, USE cr );
15734 
15735   ins_cost(INSN_COST * 2);
15736   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
15737 
15738   ins_encode %{
15739     __ cselw(as_Register($dst$$reg),
15740              as_Register($src1$$reg),
15741              as_Register($src2$$reg),
15742              Assembler::LT);
15743   %}
15744 
15745   ins_pipe(icond_reg_reg);
15746 %}
15747 
15748 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15749 %{
15750   match(Set dst (MinI src1 src2));
15751   ins_cost(INSN_COST * 3);
15752 
15753   expand %{
15754     rFlagsReg cr;
15755     compI_reg_reg(cr, src1, src2);
15756     cmovI_reg_reg_lt(dst, src1, src2, cr);
15757   %}
15758 
15759 %}
15760 // FROM HERE
15761 
15762 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15763 %{
15764   effect( DEF dst, USE src1, USE src2, USE cr );
15765 
15766   ins_cost(INSN_COST * 2);
15767   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
15768 
15769   ins_encode %{
15770     __ cselw(as_Register($dst$$reg),
15771              as_Register($src1$$reg),
15772              as_Register($src2$$reg),
15773              Assembler::GT);
15774   %}
15775 
15776   ins_pipe(icond_reg_reg);
15777 %}
15778 
15779 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15780 %{
15781   match(Set dst (MaxI src1 src2));
15782   ins_cost(INSN_COST * 3);
15783   expand %{
15784     rFlagsReg cr;
15785     compI_reg_reg(cr, src1, src2);
15786     cmovI_reg_reg_gt(dst, src1, src2, cr);
15787   %}
15788 %}
15789 
15790 // ============================================================================
15791 // Branch Instructions
15792 
15793 // Direct Branch.
15794 instruct branch(label lbl)
15795 %{
15796   match(Goto);
15797 
15798   effect(USE lbl);
15799 
15800   ins_cost(BRANCH_COST);
15801   format %{ "b  $lbl" %}
15802 
15803   ins_encode(aarch64_enc_b(lbl));
15804 
15805   ins_pipe(pipe_branch);
15806 %}
15807 
15808 // Conditional Near Branch
15809 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15810 %{
15811   // Same match rule as `branchConFar'.
15812   match(If cmp cr);
15813 
15814   effect(USE lbl);
15815 
15816   ins_cost(BRANCH_COST);
15817   // If set to 1 this indicates that the current instruction is a
15818   // short variant of a long branch. This avoids using this
15819   // instruction in first-pass matching. It will then only be used in
15820   // the `Shorten_branches' pass.
15821   // ins_short_branch(1);
15822   format %{ "b$cmp  $lbl" %}
15823 
15824   ins_encode(aarch64_enc_br_con(cmp, lbl));
15825 
15826   ins_pipe(pipe_branch_cond);
15827 %}
15828 
15829 // Conditional Near Branch Unsigned
15830 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15831 %{
15832   // Same match rule as `branchConFar'.
15833   match(If cmp cr);
15834 
15835   effect(USE lbl);
15836 
15837   ins_cost(BRANCH_COST);
15838   // If set to 1 this indicates that the current instruction is a
15839   // short variant of a long branch. This avoids using this
15840   // instruction in first-pass matching. It will then only be used in
15841   // the `Shorten_branches' pass.
15842   // ins_short_branch(1);
15843   format %{ "b$cmp  $lbl\t# unsigned" %}
15844 
15845   ins_encode(aarch64_enc_br_conU(cmp, lbl));
15846 
15847   ins_pipe(pipe_branch_cond);
15848 %}
15849 
15850 // Make use of CBZ and CBNZ.  These instructions, as well as being
15851 // shorter than (cmp; branch), have the additional benefit of not
15852 // killing the flags.
15853 
15854 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15855   match(If cmp (CmpI op1 op2));
15856   effect(USE labl);
15857 
15858   ins_cost(BRANCH_COST);
15859   format %{ "cbw$cmp   $op1, $labl" %}
15860   ins_encode %{
15861     Label* L = $labl$$label;
15862     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15863     if (cond == Assembler::EQ)
15864       __ cbzw($op1$$Register, *L);
15865     else
15866       __ cbnzw($op1$$Register, *L);
15867   %}
15868   ins_pipe(pipe_cmp_branch);
15869 %}
15870 
15871 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15872   match(If cmp (CmpL op1 op2));
15873   effect(USE labl);
15874 
15875   ins_cost(BRANCH_COST);
15876   format %{ "cb$cmp   $op1, $labl" %}
15877   ins_encode %{
15878     Label* L = $labl$$label;
15879     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15880     if (cond == Assembler::EQ)
15881       __ cbz($op1$$Register, *L);
15882     else
15883       __ cbnz($op1$$Register, *L);
15884   %}
15885   ins_pipe(pipe_cmp_branch);
15886 %}
15887 
15888 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15889   match(If cmp (CmpP op1 op2));
15890   effect(USE labl);
15891 
15892   ins_cost(BRANCH_COST);
15893   format %{ "cb$cmp   $op1, $labl" %}
15894   ins_encode %{
15895     Label* L = $labl$$label;
15896     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15897     if (cond == Assembler::EQ)
15898       __ cbz($op1$$Register, *L);
15899     else
15900       __ cbnz($op1$$Register, *L);
15901   %}
15902   ins_pipe(pipe_cmp_branch);
15903 %}
15904 
15905 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15906   match(If cmp (CmpN op1 op2));
15907   effect(USE labl);
15908 
15909   ins_cost(BRANCH_COST);
15910   format %{ "cbw$cmp   $op1, $labl" %}
15911   ins_encode %{
15912     Label* L = $labl$$label;
15913     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15914     if (cond == Assembler::EQ)
15915       __ cbzw($op1$$Register, *L);
15916     else
15917       __ cbnzw($op1$$Register, *L);
15918   %}
15919   ins_pipe(pipe_cmp_branch);
15920 %}
15921 
15922 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15923   match(If cmp (CmpP (DecodeN oop) zero));
15924   effect(USE labl);
15925 
15926   ins_cost(BRANCH_COST);
15927   format %{ "cb$cmp   $oop, $labl" %}
15928   ins_encode %{
15929     Label* L = $labl$$label;
15930     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15931     if (cond == Assembler::EQ)
15932       __ cbzw($oop$$Register, *L);
15933     else
15934       __ cbnzw($oop$$Register, *L);
15935   %}
15936   ins_pipe(pipe_cmp_branch);
15937 %}
15938 
15939 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
15940   match(If cmp (CmpU op1 op2));
15941   effect(USE labl);
15942 
15943   ins_cost(BRANCH_COST);
15944   format %{ "cbw$cmp   $op1, $labl" %}
15945   ins_encode %{
15946     Label* L = $labl$$label;
15947     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15948     if (cond == Assembler::EQ || cond == Assembler::LS)
15949       __ cbzw($op1$$Register, *L);
15950     else
15951       __ cbnzw($op1$$Register, *L);
15952   %}
15953   ins_pipe(pipe_cmp_branch);
15954 %}
15955 
15956 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
15957   match(If cmp (CmpUL op1 op2));
15958   effect(USE labl);
15959 
15960   ins_cost(BRANCH_COST);
15961   format %{ "cb$cmp   $op1, $labl" %}
15962   ins_encode %{
15963     Label* L = $labl$$label;
15964     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15965     if (cond == Assembler::EQ || cond == Assembler::LS)
15966       __ cbz($op1$$Register, *L);
15967     else
15968       __ cbnz($op1$$Register, *L);
15969   %}
15970   ins_pipe(pipe_cmp_branch);
15971 %}
15972 
15973 // Test bit and Branch
15974 
15975 // Patterns for short (< 32KiB) variants
15976 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15977   match(If cmp (CmpL op1 op2));
15978   effect(USE labl);
15979 
15980   ins_cost(BRANCH_COST);
15981   format %{ "cb$cmp   $op1, $labl # long" %}
15982   ins_encode %{
15983     Label* L = $labl$$label;
15984     Assembler::Condition cond =
15985       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15986     __ tbr(cond, $op1$$Register, 63, *L);
15987   %}
15988   ins_pipe(pipe_cmp_branch);
15989   ins_short_branch(1);
15990 %}
15991 
15992 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15993   match(If cmp (CmpI op1 op2));
15994   effect(USE labl);
15995 
15996   ins_cost(BRANCH_COST);
15997   format %{ "cb$cmp   $op1, $labl # int" %}
15998   ins_encode %{
15999     Label* L = $labl$$label;
16000     Assembler::Condition cond =
16001       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16002     __ tbr(cond, $op1$$Register, 31, *L);
16003   %}
16004   ins_pipe(pipe_cmp_branch);
16005   ins_short_branch(1);
16006 %}
16007 
16008 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16009   match(If cmp (CmpL (AndL op1 op2) op3));
16010   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16011   effect(USE labl);
16012 
16013   ins_cost(BRANCH_COST);
16014   format %{ "tb$cmp   $op1, $op2, $labl" %}
16015   ins_encode %{
16016     Label* L = $labl$$label;
16017     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16018     int bit = exact_log2_long($op2$$constant);
16019     __ tbr(cond, $op1$$Register, bit, *L);
16020   %}
16021   ins_pipe(pipe_cmp_branch);
16022   ins_short_branch(1);
16023 %}
16024 
16025 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16026   match(If cmp (CmpI (AndI op1 op2) op3));
16027   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16028   effect(USE labl);
16029 
16030   ins_cost(BRANCH_COST);
16031   format %{ "tb$cmp   $op1, $op2, $labl" %}
16032   ins_encode %{
16033     Label* L = $labl$$label;
16034     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16035     int bit = exact_log2((juint)$op2$$constant);
16036     __ tbr(cond, $op1$$Register, bit, *L);
16037   %}
16038   ins_pipe(pipe_cmp_branch);
16039   ins_short_branch(1);
16040 %}
16041 
16042 // And far variants
16043 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16044   match(If cmp (CmpL op1 op2));
16045   effect(USE labl);
16046 
16047   ins_cost(BRANCH_COST);
16048   format %{ "cb$cmp   $op1, $labl # long" %}
16049   ins_encode %{
16050     Label* L = $labl$$label;
16051     Assembler::Condition cond =
16052       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16053     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16054   %}
16055   ins_pipe(pipe_cmp_branch);
16056 %}
16057 
16058 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16059   match(If cmp (CmpI op1 op2));
16060   effect(USE labl);
16061 
16062   ins_cost(BRANCH_COST);
16063   format %{ "cb$cmp   $op1, $labl # int" %}
16064   ins_encode %{
16065     Label* L = $labl$$label;
16066     Assembler::Condition cond =
16067       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16068     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16069   %}
16070   ins_pipe(pipe_cmp_branch);
16071 %}
16072 
16073 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16074   match(If cmp (CmpL (AndL op1 op2) op3));
16075   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16076   effect(USE labl);
16077 
16078   ins_cost(BRANCH_COST);
16079   format %{ "tb$cmp   $op1, $op2, $labl" %}
16080   ins_encode %{
16081     Label* L = $labl$$label;
16082     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16083     int bit = exact_log2_long($op2$$constant);
16084     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16085   %}
16086   ins_pipe(pipe_cmp_branch);
16087 %}
16088 
16089 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16090   match(If cmp (CmpI (AndI op1 op2) op3));
16091   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16092   effect(USE labl);
16093 
16094   ins_cost(BRANCH_COST);
16095   format %{ "tb$cmp   $op1, $op2, $labl" %}
16096   ins_encode %{
16097     Label* L = $labl$$label;
16098     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16099     int bit = exact_log2((juint)$op2$$constant);
16100     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16101   %}
16102   ins_pipe(pipe_cmp_branch);
16103 %}
16104 
16105 // Test bits
16106 
16107 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16108   match(Set cr (CmpL (AndL op1 op2) op3));
16109   predicate(Assembler::operand_valid_for_logical_immediate
16110             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16111 
16112   ins_cost(INSN_COST);
16113   format %{ "tst $op1, $op2 # long" %}
16114   ins_encode %{
16115     __ tst($op1$$Register, $op2$$constant);
16116   %}
16117   ins_pipe(ialu_reg_reg);
16118 %}
16119 
16120 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16121   match(Set cr (CmpI (AndI op1 op2) op3));
16122   predicate(Assembler::operand_valid_for_logical_immediate
16123             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16124 
16125   ins_cost(INSN_COST);
16126   format %{ "tst $op1, $op2 # int" %}
16127   ins_encode %{
16128     __ tstw($op1$$Register, $op2$$constant);
16129   %}
16130   ins_pipe(ialu_reg_reg);
16131 %}
16132 
16133 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16134   match(Set cr (CmpL (AndL op1 op2) op3));
16135 
16136   ins_cost(INSN_COST);
16137   format %{ "tst $op1, $op2 # long" %}
16138   ins_encode %{
16139     __ tst($op1$$Register, $op2$$Register);
16140   %}
16141   ins_pipe(ialu_reg_reg);
16142 %}
16143 
16144 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16145   match(Set cr (CmpI (AndI op1 op2) op3));
16146 
16147   ins_cost(INSN_COST);
16148   format %{ "tstw $op1, $op2 # int" %}
16149   ins_encode %{
16150     __ tstw($op1$$Register, $op2$$Register);
16151   %}
16152   ins_pipe(ialu_reg_reg);
16153 %}
16154 
16155 
16156 // Conditional Far Branch
16157 // Conditional Far Branch Unsigned
16158 // TODO: fixme
16159 
16160 // counted loop end branch near
16161 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16162 %{
16163   match(CountedLoopEnd cmp cr);
16164 
16165   effect(USE lbl);
16166 
16167   ins_cost(BRANCH_COST);
16168   // short variant.
16169   // ins_short_branch(1);
16170   format %{ "b$cmp $lbl \t// counted loop end" %}
16171 
16172   ins_encode(aarch64_enc_br_con(cmp, lbl));
16173 
16174   ins_pipe(pipe_branch);
16175 %}
16176 
16177 // counted loop end branch near Unsigned
16178 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16179 %{
16180   match(CountedLoopEnd cmp cr);
16181 
16182   effect(USE lbl);
16183 
16184   ins_cost(BRANCH_COST);
16185   // short variant.
16186   // ins_short_branch(1);
16187   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16188 
16189   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16190 
16191   ins_pipe(pipe_branch);
16192 %}
16193 
16194 // counted loop end branch far
16195 // counted loop end branch far unsigned
16196 // TODO: fixme
16197 
16198 // ============================================================================
16199 // inlined locking and unlocking
16200 
16201 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16202 %{
16203   match(Set cr (FastLock object box));
16204   effect(TEMP tmp, TEMP tmp2);
16205 
16206   // TODO
16207   // identify correct cost
16208   ins_cost(5 * INSN_COST);
16209   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16210 
16211   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16212 
16213   ins_pipe(pipe_serial);
16214 %}
16215 
16216 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16217 %{
16218   match(Set cr (FastUnlock object box));
16219   effect(TEMP tmp, TEMP tmp2);
16220 
16221   ins_cost(5 * INSN_COST);
16222   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16223 
16224   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16225 
16226   ins_pipe(pipe_serial);
16227 %}
16228 
16229 
16230 // ============================================================================
16231 // Safepoint Instructions
16232 
16233 // TODO
16234 // provide a near and far version of this code
16235 
16236 instruct safePoint(rFlagsReg cr, iRegP poll)
16237 %{
16238   match(SafePoint poll);
16239   effect(KILL cr);
16240 
16241   format %{
16242     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16243   %}
16244   ins_encode %{
16245     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16246   %}
16247   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16248 %}
16249 
16250 
16251 // ============================================================================
16252 // Procedure Call/Return Instructions
16253 
16254 // Call Java Static Instruction
16255 
16256 instruct CallStaticJavaDirect(method meth)
16257 %{
16258   match(CallStaticJava);
16259 
16260   effect(USE meth);
16261 
16262   ins_cost(CALL_COST);
16263 
16264   format %{ "call,static $meth \t// ==> " %}
16265 
16266   ins_encode(aarch64_enc_java_static_call(meth),
16267              aarch64_enc_call_epilog);
16268 
16269   ins_pipe(pipe_class_call);
16270 %}
16271 
16272 // TO HERE
16273 
16274 // Call Java Dynamic Instruction
16275 instruct CallDynamicJavaDirect(method meth)
16276 %{
16277   match(CallDynamicJava);
16278 
16279   effect(USE meth);
16280 
16281   ins_cost(CALL_COST);
16282 
16283   format %{ "CALL,dynamic $meth \t// ==> " %}
16284 
16285   ins_encode(aarch64_enc_java_dynamic_call(meth),
16286              aarch64_enc_call_epilog);
16287 
16288   ins_pipe(pipe_class_call);
16289 %}
16290 
16291 // Call Runtime Instruction
16292 
16293 instruct CallRuntimeDirect(method meth)
16294 %{
16295   match(CallRuntime);
16296 
16297   effect(USE meth);
16298 
16299   ins_cost(CALL_COST);
16300 
16301   format %{ "CALL, runtime $meth" %}
16302 
16303   ins_encode( aarch64_enc_java_to_runtime(meth) );
16304 
16305   ins_pipe(pipe_class_call);
16306 %}
16307 
16308 // Call Runtime Instruction
16309 
16310 instruct CallLeafDirect(method meth)
16311 %{
16312   match(CallLeaf);
16313 
16314   effect(USE meth);
16315 
16316   ins_cost(CALL_COST);
16317 
16318   format %{ "CALL, runtime leaf $meth" %}
16319 
16320   ins_encode( aarch64_enc_java_to_runtime(meth) );
16321 
16322   ins_pipe(pipe_class_call);
16323 %}
16324 
16325 // Call Runtime Instruction
16326 
16327 // entry point is null, target holds the address to call
16328 instruct CallLeafNoFPIndirect(iRegP target)
16329 %{
16330   predicate(n->as_Call()->entry_point() == NULL);
16331 
16332   match(CallLeafNoFP target);
16333 
16334   ins_cost(CALL_COST);
16335 
16336   format %{ "CALL, runtime leaf nofp indirect $target" %}
16337 
16338   ins_encode %{
16339     __ blr($target$$Register);
16340   %}
16341 
16342   ins_pipe(pipe_class_call);
16343 %}
16344 
16345 instruct CallLeafNoFPDirect(method meth)
16346 %{
16347   predicate(n->as_Call()->entry_point() != NULL);
16348 
16349   match(CallLeafNoFP);
16350 
16351   effect(USE meth);
16352 
16353   ins_cost(CALL_COST);
16354 
16355   format %{ "CALL, runtime leaf nofp $meth" %}
16356 
16357   ins_encode( aarch64_enc_java_to_runtime(meth) );
16358 
16359   ins_pipe(pipe_class_call);
16360 %}
16361 
16362 // Tail Call; Jump from runtime stub to Java code.
16363 // Also known as an 'interprocedural jump'.
16364 // Target of jump will eventually return to caller.
16365 // TailJump below removes the return address.
16366 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16367 %{
16368   match(TailCall jump_target method_ptr);
16369 
16370   ins_cost(CALL_COST);
16371 
16372   format %{ "br $jump_target\t# $method_ptr holds method" %}
16373 
16374   ins_encode(aarch64_enc_tail_call(jump_target));
16375 
16376   ins_pipe(pipe_class_call);
16377 %}
16378 
16379 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16380 %{
16381   match(TailJump jump_target ex_oop);
16382 
16383   ins_cost(CALL_COST);
16384 
16385   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16386 
16387   ins_encode(aarch64_enc_tail_jmp(jump_target));
16388 
16389   ins_pipe(pipe_class_call);
16390 %}
16391 
16392 // Create exception oop: created by stack-crawling runtime code.
16393 // Created exception is now available to this handler, and is setup
16394 // just prior to jumping to this handler. No code emitted.
16395 // TODO check
16396 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16397 instruct CreateException(iRegP_R0 ex_oop)
16398 %{
16399   match(Set ex_oop (CreateEx));
16400 
16401   format %{ " -- \t// exception oop; no code emitted" %}
16402 
16403   size(0);
16404 
16405   ins_encode( /*empty*/ );
16406 
16407   ins_pipe(pipe_class_empty);
16408 %}
16409 
16410 // Rethrow exception: The exception oop will come in the first
16411 // argument position. Then JUMP (not call) to the rethrow stub code.
16412 instruct RethrowException() %{
16413   match(Rethrow);
16414   ins_cost(CALL_COST);
16415 
16416   format %{ "b rethrow_stub" %}
16417 
16418   ins_encode( aarch64_enc_rethrow() );
16419 
16420   ins_pipe(pipe_class_call);
16421 %}
16422 
16423 
16424 // Return Instruction
16425 // epilog node loads ret address into lr as part of frame pop
16426 instruct Ret()
16427 %{
16428   match(Return);
16429 
16430   format %{ "ret\t// return register" %}
16431 
16432   ins_encode( aarch64_enc_ret() );
16433 
16434   ins_pipe(pipe_branch);
16435 %}
16436 
16437 // Die now.
16438 instruct ShouldNotReachHere() %{
16439   match(Halt);
16440 
16441   ins_cost(CALL_COST);
16442   format %{ "ShouldNotReachHere" %}
16443 
16444   ins_encode %{
16445     if (is_reachable()) {
16446       __ stop(_halt_reason);
16447     }
16448   %}
16449 
16450   ins_pipe(pipe_class_default);
16451 %}
16452 
16453 // ============================================================================
16454 // Partial Subtype Check
16455 //
16456 // superklass array for an instance of the superklass.  Set a hidden
16457 // internal cache on a hit (cache is checked with exposed code in
16458 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16459 // encoding ALSO sets flags.
16460 
16461 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16462 %{
16463   match(Set result (PartialSubtypeCheck sub super));
16464   effect(KILL cr, KILL temp);
16465 
16466   ins_cost(1100);  // slightly larger than the next version
16467   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16468 
16469   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16470 
16471   opcode(0x1); // Force zero of result reg on hit
16472 
16473   ins_pipe(pipe_class_memory);
16474 %}
16475 
16476 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16477 %{
16478   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16479   effect(KILL temp, KILL result);
16480 
16481   ins_cost(1100);  // slightly larger than the next version
16482   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16483 
16484   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16485 
16486   opcode(0x0); // Don't zero result reg on hit
16487 
16488   ins_pipe(pipe_class_memory);
16489 %}
16490 
16491 // Intrisics for String.compareTo()
16492 
16493 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16494                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16495 %{
16496   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16497   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16498   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16499 
16500   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16501   ins_encode %{
16502     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16503     __ string_compare($str1$$Register, $str2$$Register,
16504                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16505                       $tmp1$$Register, $tmp2$$Register,
16506                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16507   %}
16508   ins_pipe(pipe_class_memory);
16509 %}
16510 
16511 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16512                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16513 %{
16514   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16515   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16516   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16517 
16518   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16519   ins_encode %{
16520     __ string_compare($str1$$Register, $str2$$Register,
16521                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16522                       $tmp1$$Register, $tmp2$$Register,
16523                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16524   %}
16525   ins_pipe(pipe_class_memory);
16526 %}
16527 
16528 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16529                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16530                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16531 %{
16532   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16533   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16534   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16535          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16536 
16537   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16538   ins_encode %{
16539     __ string_compare($str1$$Register, $str2$$Register,
16540                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16541                       $tmp1$$Register, $tmp2$$Register,
16542                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16543                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16544   %}
16545   ins_pipe(pipe_class_memory);
16546 %}
16547 
16548 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16549                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16550                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16551 %{
16552   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16553   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16554   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16555          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16556 
16557   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16558   ins_encode %{
16559     __ string_compare($str1$$Register, $str2$$Register,
16560                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16561                       $tmp1$$Register, $tmp2$$Register,
16562                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16563                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16564   %}
16565   ins_pipe(pipe_class_memory);
16566 %}
16567 
16568 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16569 // these string_compare variants as NEON register type for convenience so that the prototype of
16570 // string_compare can be shared with all variants.
16571 
16572 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16573                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16574                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16575                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16576 %{
16577   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16578   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16579   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16580          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16581 
16582   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16583   ins_encode %{
16584     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16585     __ string_compare($str1$$Register, $str2$$Register,
16586                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16587                       $tmp1$$Register, $tmp2$$Register,
16588                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16589                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16590                       StrIntrinsicNode::LL);
16591   %}
16592   ins_pipe(pipe_class_memory);
16593 %}
16594 
16595 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16596                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16597                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16598                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16599 %{
16600   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16601   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16602   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16603          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16604 
16605   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16606   ins_encode %{
16607     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16608     __ string_compare($str1$$Register, $str2$$Register,
16609                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16610                       $tmp1$$Register, $tmp2$$Register,
16611                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16612                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16613                       StrIntrinsicNode::LU);
16614   %}
16615   ins_pipe(pipe_class_memory);
16616 %}
16617 
16618 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16619                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16620                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16621                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16622 %{
16623   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16624   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16625   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16626          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16627 
16628   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16629   ins_encode %{
16630     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16631     __ string_compare($str1$$Register, $str2$$Register,
16632                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16633                       $tmp1$$Register, $tmp2$$Register,
16634                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16635                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16636                       StrIntrinsicNode::UL);
16637   %}
16638   ins_pipe(pipe_class_memory);
16639 %}
16640 
16641 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16642                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16643                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16644                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16645 %{
16646   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16647   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16648   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16649          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16650 
16651   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16652   ins_encode %{
16653     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16654     __ string_compare($str1$$Register, $str2$$Register,
16655                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16656                       $tmp1$$Register, $tmp2$$Register,
16657                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16658                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16659                       StrIntrinsicNode::UU);
16660   %}
16661   ins_pipe(pipe_class_memory);
16662 %}
16663 
16664 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16665        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16666        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16667 %{
16668   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16669   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16670   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16671          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16672   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16673 
16674   ins_encode %{
16675     __ string_indexof($str1$$Register, $str2$$Register,
16676                       $cnt1$$Register, $cnt2$$Register,
16677                       $tmp1$$Register, $tmp2$$Register,
16678                       $tmp3$$Register, $tmp4$$Register,
16679                       $tmp5$$Register, $tmp6$$Register,
16680                       -1, $result$$Register, StrIntrinsicNode::UU);
16681   %}
16682   ins_pipe(pipe_class_memory);
16683 %}
16684 
16685 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16686        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16687        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16688 %{
16689   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16690   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16691   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16692          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16693   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16694 
16695   ins_encode %{
16696     __ string_indexof($str1$$Register, $str2$$Register,
16697                       $cnt1$$Register, $cnt2$$Register,
16698                       $tmp1$$Register, $tmp2$$Register,
16699                       $tmp3$$Register, $tmp4$$Register,
16700                       $tmp5$$Register, $tmp6$$Register,
16701                       -1, $result$$Register, StrIntrinsicNode::LL);
16702   %}
16703   ins_pipe(pipe_class_memory);
16704 %}
16705 
16706 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16707        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16708        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16709 %{
16710   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16711   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16712   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16713          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16714   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
16715 
16716   ins_encode %{
16717     __ string_indexof($str1$$Register, $str2$$Register,
16718                       $cnt1$$Register, $cnt2$$Register,
16719                       $tmp1$$Register, $tmp2$$Register,
16720                       $tmp3$$Register, $tmp4$$Register,
16721                       $tmp5$$Register, $tmp6$$Register,
16722                       -1, $result$$Register, StrIntrinsicNode::UL);
16723   %}
16724   ins_pipe(pipe_class_memory);
16725 %}
16726 
16727 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16728                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16729                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16730 %{
16731   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16732   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16733   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16734          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16735   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
16736 
16737   ins_encode %{
16738     int icnt2 = (int)$int_cnt2$$constant;
16739     __ string_indexof($str1$$Register, $str2$$Register,
16740                       $cnt1$$Register, zr,
16741                       $tmp1$$Register, $tmp2$$Register,
16742                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16743                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16744   %}
16745   ins_pipe(pipe_class_memory);
16746 %}
16747 
16748 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16749                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16750                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16751 %{
16752   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16753   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16754   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16755          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16756   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
16757 
16758   ins_encode %{
16759     int icnt2 = (int)$int_cnt2$$constant;
16760     __ string_indexof($str1$$Register, $str2$$Register,
16761                       $cnt1$$Register, zr,
16762                       $tmp1$$Register, $tmp2$$Register,
16763                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16764                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16765   %}
16766   ins_pipe(pipe_class_memory);
16767 %}
16768 
16769 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16770                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16771                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16772 %{
16773   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16774   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16775   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16776          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16777   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
16778 
16779   ins_encode %{
16780     int icnt2 = (int)$int_cnt2$$constant;
16781     __ string_indexof($str1$$Register, $str2$$Register,
16782                       $cnt1$$Register, zr,
16783                       $tmp1$$Register, $tmp2$$Register,
16784                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16785                       icnt2, $result$$Register, StrIntrinsicNode::UL);
16786   %}
16787   ins_pipe(pipe_class_memory);
16788 %}
16789 
16790 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16791                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16792                              iRegINoSp tmp3, rFlagsReg cr)
16793 %{
16794   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16795   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16796   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16797          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16798 
16799   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16800 
16801   ins_encode %{
16802     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16803                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16804                            $tmp3$$Register);
16805   %}
16806   ins_pipe(pipe_class_memory);
16807 %}
16808 
16809 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16810                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16811                               iRegINoSp tmp3, rFlagsReg cr)
16812 %{
16813   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16814   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16815   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16816          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16817 
16818   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16819 
16820   ins_encode %{
16821     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16822                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
16823                             $tmp3$$Register);
16824   %}
16825   ins_pipe(pipe_class_memory);
16826 %}
16827 
16828 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16829                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16830                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16831   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16832   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16833   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16834   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16835   ins_encode %{
16836     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16837                                $result$$Register, $ztmp1$$FloatRegister,
16838                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16839                                $ptmp$$PRegister, true /* isL */);
16840   %}
16841   ins_pipe(pipe_class_memory);
16842 %}
16843 
16844 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16845                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16846                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16847   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16848   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16849   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16850   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16851   ins_encode %{
16852     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16853                                $result$$Register, $ztmp1$$FloatRegister,
16854                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16855                                $ptmp$$PRegister, false /* isL */);
16856   %}
16857   ins_pipe(pipe_class_memory);
16858 %}
16859 
16860 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16861                         iRegI_R0 result, rFlagsReg cr)
16862 %{
16863   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16864   match(Set result (StrEquals (Binary str1 str2) cnt));
16865   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16866 
16867   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16868   ins_encode %{
16869     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16870     __ string_equals($str1$$Register, $str2$$Register,
16871                      $result$$Register, $cnt$$Register, 1);
16872   %}
16873   ins_pipe(pipe_class_memory);
16874 %}
16875 
16876 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16877                         iRegI_R0 result, rFlagsReg cr)
16878 %{
16879   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
16880   match(Set result (StrEquals (Binary str1 str2) cnt));
16881   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16882 
16883   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16884   ins_encode %{
16885     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16886     __ string_equals($str1$$Register, $str2$$Register,
16887                      $result$$Register, $cnt$$Register, 2);
16888   %}
16889   ins_pipe(pipe_class_memory);
16890 %}
16891 
16892 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16893                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16894                        iRegP_R10 tmp, rFlagsReg cr)
16895 %{
16896   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16897   match(Set result (AryEq ary1 ary2));
16898   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16899 
16900   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16901   ins_encode %{
16902     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16903                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16904                                    $result$$Register, $tmp$$Register, 1);
16905     if (tpc == NULL) {
16906       ciEnv::current()->record_failure("CodeCache is full");
16907       return;
16908     }
16909   %}
16910   ins_pipe(pipe_class_memory);
16911 %}
16912 
16913 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16914                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16915                        iRegP_R10 tmp, rFlagsReg cr)
16916 %{
16917   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16918   match(Set result (AryEq ary1 ary2));
16919   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16920 
16921   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16922   ins_encode %{
16923     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16924                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16925                                    $result$$Register, $tmp$$Register, 2);
16926     if (tpc == NULL) {
16927       ciEnv::current()->record_failure("CodeCache is full");
16928       return;
16929     }
16930   %}
16931   ins_pipe(pipe_class_memory);
16932 %}
16933 
16934 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16935 %{
16936   match(Set result (CountPositives ary1 len));
16937   effect(USE_KILL ary1, USE_KILL len, KILL cr);
16938   format %{ "count positives byte[] $ary1,$len -> $result" %}
16939   ins_encode %{
16940     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16941     if (tpc == NULL) {
16942       ciEnv::current()->record_failure("CodeCache is full");
16943       return;
16944     }
16945   %}
16946   ins_pipe( pipe_slow );
16947 %}
16948 
16949 // fast char[] to byte[] compression
16950 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16951                          vRegD_V0 tmp1, vRegD_V1 tmp2,
16952                          vRegD_V2 tmp3, vRegD_V3 tmp4,
16953                          iRegI_R0 result, rFlagsReg cr)
16954 %{
16955   match(Set result (StrCompressedCopy src (Binary dst len)));
16956   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4,
16957          USE_KILL src, USE_KILL dst, USE len, KILL cr);
16958 
16959   format %{ "String Compress $src,$dst,$len -> $result  // KILL $src,$dst" %}
16960   ins_encode %{
16961     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16962                            $result$$Register,
16963                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16964                            $tmp3$$FloatRegister, $tmp4$$FloatRegister);
16965   %}
16966   ins_pipe(pipe_slow);
16967 %}
16968 
16969 // fast byte[] to char[] inflation
16970 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
16971                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
16972 %{
16973   match(Set dummy (StrInflatedCopy src (Binary dst len)));
16974   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16975 
16976   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
16977   ins_encode %{
16978     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16979                                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16980                                         $tmp3$$FloatRegister, $tmp4$$Register);
16981     if (tpc == NULL) {
16982       ciEnv::current()->record_failure("CodeCache is full");
16983       return;
16984     }
16985   %}
16986   ins_pipe(pipe_class_memory);
16987 %}
16988 
16989 // encode char[] to byte[] in ISO_8859_1
16990 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16991                           vRegD_V0 vtmp0, vRegD_V1 vtmp1,
16992                           vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16993                           iRegI_R0 result, rFlagsReg cr)
16994 %{
16995   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16996   match(Set result (EncodeISOArray src (Binary dst len)));
16997   effect(USE_KILL src, USE_KILL dst, USE len,
16998          KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
16999 
17000   format %{ "Encode ISO array $src,$dst,$len -> $result" %}
17001   ins_encode %{
17002     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17003                         $result$$Register, false,
17004                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17005                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17006   %}
17007   ins_pipe(pipe_class_memory);
17008 %}
17009 
17010 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17011                             vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17012                             vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17013                             iRegI_R0 result, rFlagsReg cr)
17014 %{
17015   predicate(((EncodeISOArrayNode*)n)->is_ascii());
17016   match(Set result (EncodeISOArray src (Binary dst len)));
17017   effect(USE_KILL src, USE_KILL dst, USE len,
17018          KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17019 
17020   format %{ "Encode ASCII array $src,$dst,$len -> $result" %}
17021   ins_encode %{
17022     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17023                         $result$$Register, true,
17024                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17025                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17026   %}
17027   ins_pipe(pipe_class_memory);
17028 %}
17029 
17030 // ============================================================================
17031 // This name is KNOWN by the ADLC and cannot be changed.
17032 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17033 // for this guy.
17034 instruct tlsLoadP(thread_RegP dst)
17035 %{
17036   match(Set dst (ThreadLocal));
17037 
17038   ins_cost(0);
17039 
17040   format %{ " -- \t// $dst=Thread::current(), empty" %}
17041 
17042   size(0);
17043 
17044   ins_encode( /*empty*/ );
17045 
17046   ins_pipe(pipe_class_empty);
17047 %}
17048 
17049 //----------PEEPHOLE RULES-----------------------------------------------------
17050 // These must follow all instruction definitions as they use the names
17051 // defined in the instructions definitions.
17052 //
17053 // peepmatch ( root_instr_name [preceding_instruction]* );
17054 //
17055 // peepconstraint %{
17056 // (instruction_number.operand_name relational_op instruction_number.operand_name
17057 //  [, ...] );
17058 // // instruction numbers are zero-based using left to right order in peepmatch
17059 //
17060 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17061 // // provide an instruction_number.operand_name for each operand that appears
17062 // // in the replacement instruction's match rule
17063 //
17064 // ---------VM FLAGS---------------------------------------------------------
17065 //
17066 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17067 //
17068 // Each peephole rule is given an identifying number starting with zero and
17069 // increasing by one in the order seen by the parser.  An individual peephole
17070 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17071 // on the command-line.
17072 //
17073 // ---------CURRENT LIMITATIONS----------------------------------------------
17074 //
17075 // Only match adjacent instructions in same basic block
17076 // Only equality constraints
17077 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17078 // Only one replacement instruction
17079 //
17080 // ---------EXAMPLE----------------------------------------------------------
17081 //
17082 // // pertinent parts of existing instructions in architecture description
17083 // instruct movI(iRegINoSp dst, iRegI src)
17084 // %{
17085 //   match(Set dst (CopyI src));
17086 // %}
17087 //
17088 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17089 // %{
17090 //   match(Set dst (AddI dst src));
17091 //   effect(KILL cr);
17092 // %}
17093 //
17094 // // Change (inc mov) to lea
17095 // peephole %{
17096 //   // increment preceded by register-register move
17097 //   peepmatch ( incI_iReg movI );
17098 //   // require that the destination register of the increment
17099 //   // match the destination register of the move
17100 //   peepconstraint ( 0.dst == 1.dst );
17101 //   // construct a replacement instruction that sets
17102 //   // the destination to ( move's source register + one )
17103 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17104 // %}
17105 //
17106 
17107 // Implementation no longer uses movX instructions since
17108 // machine-independent system no longer uses CopyX nodes.
17109 //
17110 // peephole
17111 // %{
17112 //   peepmatch (incI_iReg movI);
17113 //   peepconstraint (0.dst == 1.dst);
17114 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17115 // %}
17116 
17117 // peephole
17118 // %{
17119 //   peepmatch (decI_iReg movI);
17120 //   peepconstraint (0.dst == 1.dst);
17121 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17122 // %}
17123 
17124 // peephole
17125 // %{
17126 //   peepmatch (addI_iReg_imm movI);
17127 //   peepconstraint (0.dst == 1.dst);
17128 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17129 // %}
17130 
17131 // peephole
17132 // %{
17133 //   peepmatch (incL_iReg movL);
17134 //   peepconstraint (0.dst == 1.dst);
17135 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17136 // %}
17137 
17138 // peephole
17139 // %{
17140 //   peepmatch (decL_iReg movL);
17141 //   peepconstraint (0.dst == 1.dst);
17142 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17143 // %}
17144 
17145 // peephole
17146 // %{
17147 //   peepmatch (addL_iReg_imm movL);
17148 //   peepconstraint (0.dst == 1.dst);
17149 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17150 // %}
17151 
17152 // peephole
17153 // %{
17154 //   peepmatch (addP_iReg_imm movP);
17155 //   peepconstraint (0.dst == 1.dst);
17156 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17157 // %}
17158 
17159 // // Change load of spilled value to only a spill
17160 // instruct storeI(memory mem, iRegI src)
17161 // %{
17162 //   match(Set mem (StoreI mem src));
17163 // %}
17164 //
17165 // instruct loadI(iRegINoSp dst, memory mem)
17166 // %{
17167 //   match(Set dst (LoadI mem));
17168 // %}
17169 //
17170 
17171 //----------SMARTSPILL RULES---------------------------------------------------
17172 // These must follow all instruction definitions as they use the names
17173 // defined in the instructions definitions.
17174 
17175 // Local Variables:
17176 // mode: c++
17177 // End: