1 //
    2 // Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // architecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182 
  183   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  184   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  185   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  186   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  187 
  188   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  189   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  190   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  191   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  192 
  193   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  194   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  195   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  196   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  197 
  198   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  199   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  200   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  201   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  202 
  203   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  204   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  205   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  206   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  207 
  208   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  209   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  210   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  211   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  212 
  213   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  214   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  215   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  216   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  217 
  218   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  219   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  220   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  221   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  222 
  223   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  224   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  225   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  226   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  227 
  228   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  229   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  230   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  231   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  232 
  233   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  234   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  235   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  236   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  237 
  238   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  239   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  240   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  241   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  242 
  243   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  244   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  245   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  246   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  247 
  248   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  249   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  250   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  251   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  252 
  253   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  254   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  255   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  256   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  257 
  258   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  259   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  260   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  261   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  262 
  263   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  264   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  265   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  266   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  267 
  268   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  269   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  270   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  271   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  272 
  273   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  274   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  275   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  276   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  277 
  278   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  279   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  280   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  281   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  282 
  283   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  284   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  285   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  286   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  287 
  288   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  289   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  290   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  291   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  292 
  293   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  294   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  295   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  296   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  297 
  298   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  299   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  300   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  301   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  302 
  303   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  304   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  305   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  306   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  307 
  308   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  309   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  310   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  311   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  312 
  313   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  314   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  315   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  316   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  317 
  318   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  319   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  320   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  321   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  322 
  323   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  324   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  325   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  326   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  327 
  328   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  329   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  330   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  331   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  332 
  333   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  334   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  335   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  336   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  337 
  338 // ----------------------------
  339 // SVE Predicate Registers
  340 // ----------------------------
  341   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  342   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  343   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  344   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  345   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  346   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  347   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  348   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  349   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  350   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  351   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  352   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  353   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  354   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  355   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  356   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  357 
  358 // ----------------------------
  359 // Special Registers
  360 // ----------------------------
  361 
  362 // the AArch64 CSPR status flag register is not directly accessible as
  363 // instruction operand. the FPSR status flag register is a system
  364 // register which can be written/read using MSR/MRS but again does not
  365 // appear as an operand (a code identifying the FSPR occurs as an
  366 // immediate value in the instruction).
  367 
  368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  369 
  370 // Specify priority of register selection within phases of register
  371 // allocation.  Highest priority is first.  A useful heuristic is to
  372 // give registers a low priority when they are required by machine
  373 // instructions, like EAX and EDX on I486, and choose no-save registers
  374 // before save-on-call, & save-on-call before save-on-entry.  Registers
  375 // which participate in fixed calling sequences should come last.
  376 // Registers which are used as pairs must fall on an even boundary.
  377 
  378 alloc_class chunk0(
  379     // volatiles
  380     R10, R10_H,
  381     R11, R11_H,
  382     R12, R12_H,
  383     R13, R13_H,
  384     R14, R14_H,
  385     R15, R15_H,
  386     R16, R16_H,
  387     R17, R17_H,
  388     R18, R18_H,
  389 
  390     // arg registers
  391     R0, R0_H,
  392     R1, R1_H,
  393     R2, R2_H,
  394     R3, R3_H,
  395     R4, R4_H,
  396     R5, R5_H,
  397     R6, R6_H,
  398     R7, R7_H,
  399 
  400     // non-volatiles
  401     R19, R19_H,
  402     R20, R20_H,
  403     R21, R21_H,
  404     R22, R22_H,
  405     R23, R23_H,
  406     R24, R24_H,
  407     R25, R25_H,
  408     R26, R26_H,
  409 
  410     // non-allocatable registers
  411 
  412     R27, R27_H, // heapbase
  413     R28, R28_H, // thread
  414     R29, R29_H, // fp
  415     R30, R30_H, // lr
  416     R31, R31_H, // sp
  417     R8, R8_H,   // rscratch1
  418     R9, R9_H,   // rscratch2
  419 );
  420 
  421 alloc_class chunk1(
  422 
  423     // no save
  424     V16, V16_H, V16_J, V16_K,
  425     V17, V17_H, V17_J, V17_K,
  426     V18, V18_H, V18_J, V18_K,
  427     V19, V19_H, V19_J, V19_K,
  428     V20, V20_H, V20_J, V20_K,
  429     V21, V21_H, V21_J, V21_K,
  430     V22, V22_H, V22_J, V22_K,
  431     V23, V23_H, V23_J, V23_K,
  432     V24, V24_H, V24_J, V24_K,
  433     V25, V25_H, V25_J, V25_K,
  434     V26, V26_H, V26_J, V26_K,
  435     V27, V27_H, V27_J, V27_K,
  436     V28, V28_H, V28_J, V28_K,
  437     V29, V29_H, V29_J, V29_K,
  438     V30, V30_H, V30_J, V30_K,
  439     V31, V31_H, V31_J, V31_K,
  440 
  441     // arg registers
  442     V0, V0_H, V0_J, V0_K,
  443     V1, V1_H, V1_J, V1_K,
  444     V2, V2_H, V2_J, V2_K,
  445     V3, V3_H, V3_J, V3_K,
  446     V4, V4_H, V4_J, V4_K,
  447     V5, V5_H, V5_J, V5_K,
  448     V6, V6_H, V6_J, V6_K,
  449     V7, V7_H, V7_J, V7_K,
  450 
  451     // non-volatiles
  452     V8, V8_H, V8_J, V8_K,
  453     V9, V9_H, V9_J, V9_K,
  454     V10, V10_H, V10_J, V10_K,
  455     V11, V11_H, V11_J, V11_K,
  456     V12, V12_H, V12_J, V12_K,
  457     V13, V13_H, V13_J, V13_K,
  458     V14, V14_H, V14_J, V14_K,
  459     V15, V15_H, V15_J, V15_K,
  460 );
  461 
  462 alloc_class chunk2 (
  463     // Governing predicates for load/store and arithmetic
  464     P0,
  465     P1,
  466     P2,
  467     P3,
  468     P4,
  469     P5,
  470     P6,
  471 
  472     // Extra predicates
  473     P8,
  474     P9,
  475     P10,
  476     P11,
  477     P12,
  478     P13,
  479     P14,
  480     P15,
  481 
  482     // Preserved for all-true predicate
  483     P7,
  484 );
  485 
  486 alloc_class chunk3(RFLAGS);
  487 
  488 //----------Architecture Description Register Classes--------------------------
  489 // Several register classes are automatically defined based upon information in
  490 // this architecture description.
  491 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  493 //
  494 
  495 // Class for all 32 bit general purpose registers
  496 reg_class all_reg32(
  497     R0,
  498     R1,
  499     R2,
  500     R3,
  501     R4,
  502     R5,
  503     R6,
  504     R7,
  505     R10,
  506     R11,
  507     R12,
  508     R13,
  509     R14,
  510     R15,
  511     R16,
  512     R17,
  513     R18,
  514     R19,
  515     R20,
  516     R21,
  517     R22,
  518     R23,
  519     R24,
  520     R25,
  521     R26,
  522     R27,
  523     R28,
  524     R29,
  525     R30,
  526     R31
  527 );
  528 
  529 
  530 // Class for all 32 bit integer registers (excluding SP which
  531 // will never be used as an integer register)
  532 reg_class any_reg32 %{
  533   return _ANY_REG32_mask;
  534 %}
  535 
  536 // Singleton class for R0 int register
  537 reg_class int_r0_reg(R0);
  538 
  539 // Singleton class for R2 int register
  540 reg_class int_r2_reg(R2);
  541 
  542 // Singleton class for R3 int register
  543 reg_class int_r3_reg(R3);
  544 
  545 // Singleton class for R4 int register
  546 reg_class int_r4_reg(R4);
  547 
  548 // Singleton class for R31 int register
  549 reg_class int_r31_reg(R31);
  550 
  551 // Class for all 64 bit general purpose registers
  552 reg_class all_reg(
  553     R0, R0_H,
  554     R1, R1_H,
  555     R2, R2_H,
  556     R3, R3_H,
  557     R4, R4_H,
  558     R5, R5_H,
  559     R6, R6_H,
  560     R7, R7_H,
  561     R10, R10_H,
  562     R11, R11_H,
  563     R12, R12_H,
  564     R13, R13_H,
  565     R14, R14_H,
  566     R15, R15_H,
  567     R16, R16_H,
  568     R17, R17_H,
  569     R18, R18_H,
  570     R19, R19_H,
  571     R20, R20_H,
  572     R21, R21_H,
  573     R22, R22_H,
  574     R23, R23_H,
  575     R24, R24_H,
  576     R25, R25_H,
  577     R26, R26_H,
  578     R27, R27_H,
  579     R28, R28_H,
  580     R29, R29_H,
  581     R30, R30_H,
  582     R31, R31_H
  583 );
  584 
  585 // Class for all long integer registers (including SP)
  586 reg_class any_reg %{
  587   return _ANY_REG_mask;
  588 %}
  589 
  590 // Class for non-allocatable 32 bit registers
  591 reg_class non_allocatable_reg32(
  592 #ifdef R18_RESERVED
  593     // See comment in register_aarch64.hpp
  594     R18,                        // tls on Windows
  595 #endif
  596     R28,                        // thread
  597     R30,                        // lr
  598     R31                         // sp
  599 );
  600 
  601 // Class for non-allocatable 64 bit registers
  602 reg_class non_allocatable_reg(
  603 #ifdef R18_RESERVED
  604     // See comment in register_aarch64.hpp
  605     R18, R18_H,                 // tls on Windows, platform register on macOS
  606 #endif
  607     R28, R28_H,                 // thread
  608     R30, R30_H,                 // lr
  609     R31, R31_H                  // sp
  610 );
  611 
  612 // Class for all non-special integer registers
  613 reg_class no_special_reg32 %{
  614   return _NO_SPECIAL_REG32_mask;
  615 %}
  616 
  617 // Class for all non-special long integer registers
  618 reg_class no_special_reg %{
  619   return _NO_SPECIAL_REG_mask;
  620 %}
  621 
  622 // Class for 64 bit register r0
  623 reg_class r0_reg(
  624     R0, R0_H
  625 );
  626 
  627 // Class for 64 bit register r1
  628 reg_class r1_reg(
  629     R1, R1_H
  630 );
  631 
  632 // Class for 64 bit register r2
  633 reg_class r2_reg(
  634     R2, R2_H
  635 );
  636 
  637 // Class for 64 bit register r3
  638 reg_class r3_reg(
  639     R3, R3_H
  640 );
  641 
  642 // Class for 64 bit register r4
  643 reg_class r4_reg(
  644     R4, R4_H
  645 );
  646 
  647 // Class for 64 bit register r5
  648 reg_class r5_reg(
  649     R5, R5_H
  650 );
  651 
  652 // Class for 64 bit register r10
  653 reg_class r10_reg(
  654     R10, R10_H
  655 );
  656 
  657 // Class for 64 bit register r11
  658 reg_class r11_reg(
  659     R11, R11_H
  660 );
  661 
  662 // Class for method register
  663 reg_class method_reg(
  664     R12, R12_H
  665 );
  666 
  667 // Class for heapbase register
  668 reg_class heapbase_reg(
  669     R27, R27_H
  670 );
  671 
  672 // Class for thread register
  673 reg_class thread_reg(
  674     R28, R28_H
  675 );
  676 
  677 // Class for frame pointer register
  678 reg_class fp_reg(
  679     R29, R29_H
  680 );
  681 
  682 // Class for link register
  683 reg_class lr_reg(
  684     R30, R30_H
  685 );
  686 
  687 // Class for long sp register
  688 reg_class sp_reg(
  689   R31, R31_H
  690 );
  691 
  692 // Class for all pointer registers
  693 reg_class ptr_reg %{
  694   return _PTR_REG_mask;
  695 %}
  696 
  697 // Class for all non_special pointer registers
  698 reg_class no_special_ptr_reg %{
  699   return _NO_SPECIAL_PTR_REG_mask;
  700 %}
  701 
  702 // Class for all float registers
  703 reg_class float_reg(
  704     V0,
  705     V1,
  706     V2,
  707     V3,
  708     V4,
  709     V5,
  710     V6,
  711     V7,
  712     V8,
  713     V9,
  714     V10,
  715     V11,
  716     V12,
  717     V13,
  718     V14,
  719     V15,
  720     V16,
  721     V17,
  722     V18,
  723     V19,
  724     V20,
  725     V21,
  726     V22,
  727     V23,
  728     V24,
  729     V25,
  730     V26,
  731     V27,
  732     V28,
  733     V29,
  734     V30,
  735     V31
  736 );
  737 
  738 // Double precision float registers have virtual `high halves' that
  739 // are needed by the allocator.
  740 // Class for all double registers
  741 reg_class double_reg(
  742     V0, V0_H,
  743     V1, V1_H,
  744     V2, V2_H,
  745     V3, V3_H,
  746     V4, V4_H,
  747     V5, V5_H,
  748     V6, V6_H,
  749     V7, V7_H,
  750     V8, V8_H,
  751     V9, V9_H,
  752     V10, V10_H,
  753     V11, V11_H,
  754     V12, V12_H,
  755     V13, V13_H,
  756     V14, V14_H,
  757     V15, V15_H,
  758     V16, V16_H,
  759     V17, V17_H,
  760     V18, V18_H,
  761     V19, V19_H,
  762     V20, V20_H,
  763     V21, V21_H,
  764     V22, V22_H,
  765     V23, V23_H,
  766     V24, V24_H,
  767     V25, V25_H,
  768     V26, V26_H,
  769     V27, V27_H,
  770     V28, V28_H,
  771     V29, V29_H,
  772     V30, V30_H,
  773     V31, V31_H
  774 );
  775 
  776 // Class for all SVE vector registers.
  777 reg_class vectora_reg (
  778     V0, V0_H, V0_J, V0_K,
  779     V1, V1_H, V1_J, V1_K,
  780     V2, V2_H, V2_J, V2_K,
  781     V3, V3_H, V3_J, V3_K,
  782     V4, V4_H, V4_J, V4_K,
  783     V5, V5_H, V5_J, V5_K,
  784     V6, V6_H, V6_J, V6_K,
  785     V7, V7_H, V7_J, V7_K,
  786     V8, V8_H, V8_J, V8_K,
  787     V9, V9_H, V9_J, V9_K,
  788     V10, V10_H, V10_J, V10_K,
  789     V11, V11_H, V11_J, V11_K,
  790     V12, V12_H, V12_J, V12_K,
  791     V13, V13_H, V13_J, V13_K,
  792     V14, V14_H, V14_J, V14_K,
  793     V15, V15_H, V15_J, V15_K,
  794     V16, V16_H, V16_J, V16_K,
  795     V17, V17_H, V17_J, V17_K,
  796     V18, V18_H, V18_J, V18_K,
  797     V19, V19_H, V19_J, V19_K,
  798     V20, V20_H, V20_J, V20_K,
  799     V21, V21_H, V21_J, V21_K,
  800     V22, V22_H, V22_J, V22_K,
  801     V23, V23_H, V23_J, V23_K,
  802     V24, V24_H, V24_J, V24_K,
  803     V25, V25_H, V25_J, V25_K,
  804     V26, V26_H, V26_J, V26_K,
  805     V27, V27_H, V27_J, V27_K,
  806     V28, V28_H, V28_J, V28_K,
  807     V29, V29_H, V29_J, V29_K,
  808     V30, V30_H, V30_J, V30_K,
  809     V31, V31_H, V31_J, V31_K,
  810 );
  811 
  812 // Class for all 64bit vector registers
  813 reg_class vectord_reg(
  814     V0, V0_H,
  815     V1, V1_H,
  816     V2, V2_H,
  817     V3, V3_H,
  818     V4, V4_H,
  819     V5, V5_H,
  820     V6, V6_H,
  821     V7, V7_H,
  822     V8, V8_H,
  823     V9, V9_H,
  824     V10, V10_H,
  825     V11, V11_H,
  826     V12, V12_H,
  827     V13, V13_H,
  828     V14, V14_H,
  829     V15, V15_H,
  830     V16, V16_H,
  831     V17, V17_H,
  832     V18, V18_H,
  833     V19, V19_H,
  834     V20, V20_H,
  835     V21, V21_H,
  836     V22, V22_H,
  837     V23, V23_H,
  838     V24, V24_H,
  839     V25, V25_H,
  840     V26, V26_H,
  841     V27, V27_H,
  842     V28, V28_H,
  843     V29, V29_H,
  844     V30, V30_H,
  845     V31, V31_H
  846 );
  847 
  848 // Class for all 128bit vector registers
  849 reg_class vectorx_reg(
  850     V0, V0_H, V0_J, V0_K,
  851     V1, V1_H, V1_J, V1_K,
  852     V2, V2_H, V2_J, V2_K,
  853     V3, V3_H, V3_J, V3_K,
  854     V4, V4_H, V4_J, V4_K,
  855     V5, V5_H, V5_J, V5_K,
  856     V6, V6_H, V6_J, V6_K,
  857     V7, V7_H, V7_J, V7_K,
  858     V8, V8_H, V8_J, V8_K,
  859     V9, V9_H, V9_J, V9_K,
  860     V10, V10_H, V10_J, V10_K,
  861     V11, V11_H, V11_J, V11_K,
  862     V12, V12_H, V12_J, V12_K,
  863     V13, V13_H, V13_J, V13_K,
  864     V14, V14_H, V14_J, V14_K,
  865     V15, V15_H, V15_J, V15_K,
  866     V16, V16_H, V16_J, V16_K,
  867     V17, V17_H, V17_J, V17_K,
  868     V18, V18_H, V18_J, V18_K,
  869     V19, V19_H, V19_J, V19_K,
  870     V20, V20_H, V20_J, V20_K,
  871     V21, V21_H, V21_J, V21_K,
  872     V22, V22_H, V22_J, V22_K,
  873     V23, V23_H, V23_J, V23_K,
  874     V24, V24_H, V24_J, V24_K,
  875     V25, V25_H, V25_J, V25_K,
  876     V26, V26_H, V26_J, V26_K,
  877     V27, V27_H, V27_J, V27_K,
  878     V28, V28_H, V28_J, V28_K,
  879     V29, V29_H, V29_J, V29_K,
  880     V30, V30_H, V30_J, V30_K,
  881     V31, V31_H, V31_J, V31_K
  882 );
  883 
  884 // Class for 128 bit register v0
  885 reg_class v0_reg(
  886     V0, V0_H
  887 );
  888 
  889 // Class for 128 bit register v1
  890 reg_class v1_reg(
  891     V1, V1_H
  892 );
  893 
  894 // Class for 128 bit register v2
  895 reg_class v2_reg(
  896     V2, V2_H
  897 );
  898 
  899 // Class for 128 bit register v3
  900 reg_class v3_reg(
  901     V3, V3_H
  902 );
  903 
  904 // Class for 128 bit register v4
  905 reg_class v4_reg(
  906     V4, V4_H
  907 );
  908 
  909 // Class for 128 bit register v5
  910 reg_class v5_reg(
  911     V5, V5_H
  912 );
  913 
  914 // Class for 128 bit register v6
  915 reg_class v6_reg(
  916     V6, V6_H
  917 );
  918 
  919 // Class for 128 bit register v7
  920 reg_class v7_reg(
  921     V7, V7_H
  922 );
  923 
  924 // Class for 128 bit register v8
  925 reg_class v8_reg(
  926     V8, V8_H
  927 );
  928 
  929 // Class for 128 bit register v9
  930 reg_class v9_reg(
  931     V9, V9_H
  932 );
  933 
  934 // Class for 128 bit register v10
  935 reg_class v10_reg(
  936     V10, V10_H
  937 );
  938 
  939 // Class for 128 bit register v11
  940 reg_class v11_reg(
  941     V11, V11_H
  942 );
  943 
  944 // Class for 128 bit register v12
  945 reg_class v12_reg(
  946     V12, V12_H
  947 );
  948 
  949 // Class for 128 bit register v13
  950 reg_class v13_reg(
  951     V13, V13_H
  952 );
  953 
  954 // Class for 128 bit register v14
  955 reg_class v14_reg(
  956     V14, V14_H
  957 );
  958 
  959 // Class for 128 bit register v15
  960 reg_class v15_reg(
  961     V15, V15_H
  962 );
  963 
  964 // Class for 128 bit register v16
  965 reg_class v16_reg(
  966     V16, V16_H
  967 );
  968 
  969 // Class for 128 bit register v17
  970 reg_class v17_reg(
  971     V17, V17_H
  972 );
  973 
  974 // Class for 128 bit register v18
  975 reg_class v18_reg(
  976     V18, V18_H
  977 );
  978 
  979 // Class for 128 bit register v19
  980 reg_class v19_reg(
  981     V19, V19_H
  982 );
  983 
  984 // Class for 128 bit register v20
  985 reg_class v20_reg(
  986     V20, V20_H
  987 );
  988 
  989 // Class for 128 bit register v21
  990 reg_class v21_reg(
  991     V21, V21_H
  992 );
  993 
  994 // Class for 128 bit register v22
  995 reg_class v22_reg(
  996     V22, V22_H
  997 );
  998 
  999 // Class for 128 bit register v23
 1000 reg_class v23_reg(
 1001     V23, V23_H
 1002 );
 1003 
 1004 // Class for 128 bit register v24
 1005 reg_class v24_reg(
 1006     V24, V24_H
 1007 );
 1008 
 1009 // Class for 128 bit register v25
 1010 reg_class v25_reg(
 1011     V25, V25_H
 1012 );
 1013 
 1014 // Class for 128 bit register v26
 1015 reg_class v26_reg(
 1016     V26, V26_H
 1017 );
 1018 
 1019 // Class for 128 bit register v27
 1020 reg_class v27_reg(
 1021     V27, V27_H
 1022 );
 1023 
 1024 // Class for 128 bit register v28
 1025 reg_class v28_reg(
 1026     V28, V28_H
 1027 );
 1028 
 1029 // Class for 128 bit register v29
 1030 reg_class v29_reg(
 1031     V29, V29_H
 1032 );
 1033 
 1034 // Class for 128 bit register v30
 1035 reg_class v30_reg(
 1036     V30, V30_H
 1037 );
 1038 
 1039 // Class for 128 bit register v31
 1040 reg_class v31_reg(
 1041     V31, V31_H
 1042 );
 1043 
 1044 // Class for all SVE predicate registers.
 1045 reg_class pr_reg (
 1046     P0,
 1047     P1,
 1048     P2,
 1049     P3,
 1050     P4,
 1051     P5,
 1052     P6,
 1053     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1054     P8,
 1055     P9,
 1056     P10,
 1057     P11,
 1058     P12,
 1059     P13,
 1060     P14,
 1061     P15
 1062 );
 1063 
 1064 // Class for SVE governing predicate registers, which are used
 1065 // to determine the active elements of a predicated instruction.
 1066 reg_class gov_pr (
 1067     P0,
 1068     P1,
 1069     P2,
 1070     P3,
 1071     P4,
 1072     P5,
 1073     P6,
 1074     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1075 );
 1076 
 1077 reg_class p0_reg(P0);
 1078 reg_class p1_reg(P1);
 1079 
 1080 // Singleton class for condition codes
 1081 reg_class int_flags(RFLAGS);
 1082 
 1083 %}
 1084 
 1085 //----------DEFINITION BLOCK---------------------------------------------------
 1086 // Define name --> value mappings to inform the ADLC of an integer valued name
 1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1088 // Format:
 1089 //        int_def  <name>         ( <int_value>, <expression>);
 1090 // Generated Code in ad_<arch>.hpp
 1091 //        #define  <name>   (<expression>)
 1092 //        // value == <int_value>
 1093 // Generated code in ad_<arch>.cpp adlc_verification()
 1094 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1095 //
 1096 
 1097 // we follow the ppc-aix port in using a simple cost model which ranks
 1098 // register operations as cheap, memory ops as more expensive and
 1099 // branches as most expensive. the first two have a low as well as a
 1100 // normal cost. huge cost appears to be a way of saying don't do
 1101 // something
 1102 
 1103 definitions %{
 1104   // The default cost (of a register move instruction).
 1105   int_def INSN_COST            (    100,     100);
 1106   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1107   int_def CALL_COST            (    200,     2 * INSN_COST);
 1108   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1109 %}
 1110 
 1111 
 1112 //----------SOURCE BLOCK-------------------------------------------------------
 1113 // This is a block of C++ code which provides values, functions, and
 1114 // definitions necessary in the rest of the architecture description
 1115 
 1116 source_hpp %{
 1117 
 1118 #include "asm/macroAssembler.hpp"
 1119 #include "gc/shared/barrierSetAssembler.hpp"
 1120 #include "gc/shared/cardTable.hpp"
 1121 #include "gc/shared/cardTableBarrierSet.hpp"
 1122 #include "gc/shared/collectedHeap.hpp"
 1123 #include "opto/addnode.hpp"
 1124 #include "opto/convertnode.hpp"
 1125 #include "runtime/objectMonitor.hpp"
 1126 
 1127 extern RegMask _ANY_REG32_mask;
 1128 extern RegMask _ANY_REG_mask;
 1129 extern RegMask _PTR_REG_mask;
 1130 extern RegMask _NO_SPECIAL_REG32_mask;
 1131 extern RegMask _NO_SPECIAL_REG_mask;
 1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1133 
 1134 class CallStubImpl {
 1135 
 1136   //--------------------------------------------------------------
 1137   //---<  Used for optimization in Compile::shorten_branches  >---
 1138   //--------------------------------------------------------------
 1139 
 1140  public:
 1141   // Size of call trampoline stub.
 1142   static uint size_call_trampoline() {
 1143     return 0; // no call trampolines on this platform
 1144   }
 1145 
 1146   // number of relocations needed by a call trampoline stub
 1147   static uint reloc_call_trampoline() {
 1148     return 0; // no call trampolines on this platform
 1149   }
 1150 };
 1151 
 1152 class HandlerImpl {
 1153 
 1154  public:
 1155 
 1156   static int emit_exception_handler(CodeBuffer &cbuf);
 1157   static int emit_deopt_handler(CodeBuffer& cbuf);
 1158 
 1159   static uint size_exception_handler() {
 1160     return MacroAssembler::far_codestub_branch_size();
 1161   }
 1162 
 1163   static uint size_deopt_handler() {
 1164     // count one adr and one far branch instruction
 1165     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1166   }
 1167 };
 1168 
 1169 class Node::PD {
 1170 public:
 1171   enum NodeFlags {
 1172     _last_flag = Node::_last_flag
 1173   };
 1174 };
 1175 
 1176   bool is_CAS(int opcode, bool maybe_volatile);
 1177 
 1178   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1179 
 1180   bool unnecessary_acquire(const Node *barrier);
 1181   bool needs_acquiring_load(const Node *load);
 1182 
 1183   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1184 
 1185   bool unnecessary_release(const Node *barrier);
 1186   bool unnecessary_volatile(const Node *barrier);
 1187   bool needs_releasing_store(const Node *store);
 1188 
 1189   // predicate controlling translation of CompareAndSwapX
 1190   bool needs_acquiring_load_exclusive(const Node *load);
 1191 
 1192   // predicate controlling addressing modes
 1193   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1194 %}
 1195 
 1196 source %{
 1197 
 1198   // Derived RegMask with conditionally allocatable registers
 1199 
 1200   void PhaseOutput::pd_perform_mach_node_analysis() {
 1201   }
 1202 
 1203   int MachNode::pd_alignment_required() const {
 1204     return 1;
 1205   }
 1206 
 1207   int MachNode::compute_padding(int current_offset) const {
 1208     return 0;
 1209   }
 1210 
 1211   RegMask _ANY_REG32_mask;
 1212   RegMask _ANY_REG_mask;
 1213   RegMask _PTR_REG_mask;
 1214   RegMask _NO_SPECIAL_REG32_mask;
 1215   RegMask _NO_SPECIAL_REG_mask;
 1216   RegMask _NO_SPECIAL_PTR_REG_mask;
 1217 
 1218   void reg_mask_init() {
 1219     // We derive below RegMask(s) from the ones which are auto-generated from
 1220     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1221     // registers conditionally reserved.
 1222 
 1223     _ANY_REG32_mask = _ALL_REG32_mask;
 1224     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1225 
 1226     _ANY_REG_mask = _ALL_REG_mask;
 1227 
 1228     _PTR_REG_mask = _ALL_REG_mask;
 1229 
 1230     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1231     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1232 
 1233     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1234     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1235 
 1236     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1237     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1238 
 1239     // r27 is not allocatable when compressed oops is on and heapbase is not
 1240     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1241     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1242       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1243       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1244       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1245     }
 1246 
 1247     // r29 is not allocatable when PreserveFramePointer is on
 1248     if (PreserveFramePointer) {
 1249       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1250       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1251       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1252     }
 1253   }
 1254 
 1255   // Optimizaton of volatile gets and puts
 1256   // -------------------------------------
 1257   //
 1258   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1259   // use to implement volatile reads and writes. For a volatile read
 1260   // we simply need
 1261   //
 1262   //   ldar<x>
 1263   //
 1264   // and for a volatile write we need
 1265   //
 1266   //   stlr<x>
 1267   //
 1268   // Alternatively, we can implement them by pairing a normal
 1269   // load/store with a memory barrier. For a volatile read we need
 1270   //
 1271   //   ldr<x>
 1272   //   dmb ishld
 1273   //
 1274   // for a volatile write
 1275   //
 1276   //   dmb ish
 1277   //   str<x>
 1278   //   dmb ish
 1279   //
 1280   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1281   // sequences. These are normally translated to an instruction
 1282   // sequence like the following
 1283   //
 1284   //   dmb      ish
 1285   // retry:
 1286   //   ldxr<x>   rval raddr
 1287   //   cmp       rval rold
 1288   //   b.ne done
 1289   //   stlxr<x>  rval, rnew, rold
 1290   //   cbnz      rval retry
 1291   // done:
 1292   //   cset      r0, eq
 1293   //   dmb ishld
 1294   //
 1295   // Note that the exclusive store is already using an stlxr
 1296   // instruction. That is required to ensure visibility to other
 1297   // threads of the exclusive write (assuming it succeeds) before that
 1298   // of any subsequent writes.
 1299   //
 1300   // The following instruction sequence is an improvement on the above
 1301   //
 1302   // retry:
 1303   //   ldaxr<x>  rval raddr
 1304   //   cmp       rval rold
 1305   //   b.ne done
 1306   //   stlxr<x>  rval, rnew, rold
 1307   //   cbnz      rval retry
 1308   // done:
 1309   //   cset      r0, eq
 1310   //
 1311   // We don't need the leading dmb ish since the stlxr guarantees
 1312   // visibility of prior writes in the case that the swap is
 1313   // successful. Crucially we don't have to worry about the case where
 1314   // the swap is not successful since no valid program should be
 1315   // relying on visibility of prior changes by the attempting thread
 1316   // in the case where the CAS fails.
 1317   //
 1318   // Similarly, we don't need the trailing dmb ishld if we substitute
 1319   // an ldaxr instruction since that will provide all the guarantees we
 1320   // require regarding observation of changes made by other threads
 1321   // before any change to the CAS address observed by the load.
 1322   //
 1323   // In order to generate the desired instruction sequence we need to
 1324   // be able to identify specific 'signature' ideal graph node
 1325   // sequences which i) occur as a translation of a volatile reads or
 1326   // writes or CAS operations and ii) do not occur through any other
 1327   // translation or graph transformation. We can then provide
 1328   // alternative aldc matching rules which translate these node
 1329   // sequences to the desired machine code sequences. Selection of the
 1330   // alternative rules can be implemented by predicates which identify
 1331   // the relevant node sequences.
 1332   //
 1333   // The ideal graph generator translates a volatile read to the node
 1334   // sequence
 1335   //
 1336   //   LoadX[mo_acquire]
 1337   //   MemBarAcquire
 1338   //
 1339   // As a special case when using the compressed oops optimization we
 1340   // may also see this variant
 1341   //
 1342   //   LoadN[mo_acquire]
 1343   //   DecodeN
 1344   //   MemBarAcquire
 1345   //
 1346   // A volatile write is translated to the node sequence
 1347   //
 1348   //   MemBarRelease
 1349   //   StoreX[mo_release] {CardMark}-optional
 1350   //   MemBarVolatile
 1351   //
 1352   // n.b. the above node patterns are generated with a strict
 1353   // 'signature' configuration of input and output dependencies (see
 1354   // the predicates below for exact details). The card mark may be as
 1355   // simple as a few extra nodes or, in a few GC configurations, may
 1356   // include more complex control flow between the leading and
 1357   // trailing memory barriers. However, whatever the card mark
 1358   // configuration these signatures are unique to translated volatile
 1359   // reads/stores -- they will not appear as a result of any other
 1360   // bytecode translation or inlining nor as a consequence of
 1361   // optimizing transforms.
 1362   //
 1363   // We also want to catch inlined unsafe volatile gets and puts and
 1364   // be able to implement them using either ldar<x>/stlr<x> or some
 1365   // combination of ldr<x>/stlr<x> and dmb instructions.
 1366   //
 1367   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1368   // normal volatile put node sequence containing an extra cpuorder
 1369   // membar
 1370   //
 1371   //   MemBarRelease
 1372   //   MemBarCPUOrder
 1373   //   StoreX[mo_release] {CardMark}-optional
 1374   //   MemBarCPUOrder
 1375   //   MemBarVolatile
 1376   //
 1377   // n.b. as an aside, a cpuorder membar is not itself subject to
 1378   // matching and translation by adlc rules.  However, the rule
 1379   // predicates need to detect its presence in order to correctly
 1380   // select the desired adlc rules.
 1381   //
 1382   // Inlined unsafe volatile gets manifest as a slightly different
 1383   // node sequence to a normal volatile get because of the
 1384   // introduction of some CPUOrder memory barriers to bracket the
 1385   // Load. However, but the same basic skeleton of a LoadX feeding a
 1386   // MemBarAcquire, possibly through an optional DecodeN, is still
 1387   // present
 1388   //
 1389   //   MemBarCPUOrder
 1390   //        ||       \\
 1391   //   MemBarCPUOrder LoadX[mo_acquire]
 1392   //        ||            |
 1393   //        ||       {DecodeN} optional
 1394   //        ||       /
 1395   //     MemBarAcquire
 1396   //
 1397   // In this case the acquire membar does not directly depend on the
 1398   // load. However, we can be sure that the load is generated from an
 1399   // inlined unsafe volatile get if we see it dependent on this unique
 1400   // sequence of membar nodes. Similarly, given an acquire membar we
 1401   // can know that it was added because of an inlined unsafe volatile
 1402   // get if it is fed and feeds a cpuorder membar and if its feed
 1403   // membar also feeds an acquiring load.
 1404   //
 1405   // Finally an inlined (Unsafe) CAS operation is translated to the
 1406   // following ideal graph
 1407   //
 1408   //   MemBarRelease
 1409   //   MemBarCPUOrder
 1410   //   CompareAndSwapX {CardMark}-optional
 1411   //   MemBarCPUOrder
 1412   //   MemBarAcquire
 1413   //
 1414   // So, where we can identify these volatile read and write
 1415   // signatures we can choose to plant either of the above two code
 1416   // sequences. For a volatile read we can simply plant a normal
 1417   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1418   // also choose to inhibit translation of the MemBarAcquire and
 1419   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1420   //
 1421   // When we recognise a volatile store signature we can choose to
 1422   // plant at a dmb ish as a translation for the MemBarRelease, a
 1423   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1424   // Alternatively, we can inhibit translation of the MemBarRelease
 1425   // and MemBarVolatile and instead plant a simple stlr<x>
 1426   // instruction.
 1427   //
 1428   // when we recognise a CAS signature we can choose to plant a dmb
 1429   // ish as a translation for the MemBarRelease, the conventional
 1430   // macro-instruction sequence for the CompareAndSwap node (which
 1431   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1432   // Alternatively, we can elide generation of the dmb instructions
 1433   // and plant the alternative CompareAndSwap macro-instruction
 1434   // sequence (which uses ldaxr<x>).
 1435   //
 1436   // Of course, the above only applies when we see these signature
 1437   // configurations. We still want to plant dmb instructions in any
 1438   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1439   // MemBarVolatile. For example, at the end of a constructor which
 1440   // writes final/volatile fields we will see a MemBarRelease
 1441   // instruction and this needs a 'dmb ish' lest we risk the
 1442   // constructed object being visible without making the
 1443   // final/volatile field writes visible.
 1444   //
 1445   // n.b. the translation rules below which rely on detection of the
 1446   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1447   // If we see anything other than the signature configurations we
 1448   // always just translate the loads and stores to ldr<x> and str<x>
 1449   // and translate acquire, release and volatile membars to the
 1450   // relevant dmb instructions.
 1451   //
 1452 
 1453   // is_CAS(int opcode, bool maybe_volatile)
 1454   //
 1455   // return true if opcode is one of the possible CompareAndSwapX
 1456   // values otherwise false.
 1457 
 1458   bool is_CAS(int opcode, bool maybe_volatile)
 1459   {
 1460     switch(opcode) {
 1461       // We handle these
 1462     case Op_CompareAndSwapI:
 1463     case Op_CompareAndSwapL:
 1464     case Op_CompareAndSwapP:
 1465     case Op_CompareAndSwapN:
 1466     case Op_ShenandoahCompareAndSwapP:
 1467     case Op_ShenandoahCompareAndSwapN:
 1468     case Op_CompareAndSwapB:
 1469     case Op_CompareAndSwapS:
 1470     case Op_GetAndSetI:
 1471     case Op_GetAndSetL:
 1472     case Op_GetAndSetP:
 1473     case Op_GetAndSetN:
 1474     case Op_GetAndAddI:
 1475     case Op_GetAndAddL:
 1476       return true;
 1477     case Op_CompareAndExchangeI:
 1478     case Op_CompareAndExchangeN:
 1479     case Op_CompareAndExchangeB:
 1480     case Op_CompareAndExchangeS:
 1481     case Op_CompareAndExchangeL:
 1482     case Op_CompareAndExchangeP:
 1483     case Op_WeakCompareAndSwapB:
 1484     case Op_WeakCompareAndSwapS:
 1485     case Op_WeakCompareAndSwapI:
 1486     case Op_WeakCompareAndSwapL:
 1487     case Op_WeakCompareAndSwapP:
 1488     case Op_WeakCompareAndSwapN:
 1489     case Op_ShenandoahWeakCompareAndSwapP:
 1490     case Op_ShenandoahWeakCompareAndSwapN:
 1491     case Op_ShenandoahCompareAndExchangeP:
 1492     case Op_ShenandoahCompareAndExchangeN:
 1493       return maybe_volatile;
 1494     default:
 1495       return false;
 1496     }
 1497   }
 1498 
 1499   // helper to determine the maximum number of Phi nodes we may need to
 1500   // traverse when searching from a card mark membar for the merge mem
 1501   // feeding a trailing membar or vice versa
 1502 
 1503 // predicates controlling emit of ldr<x>/ldar<x>
 1504 
 1505 bool unnecessary_acquire(const Node *barrier)
 1506 {
 1507   assert(barrier->is_MemBar(), "expecting a membar");
 1508 
 1509   MemBarNode* mb = barrier->as_MemBar();
 1510 
 1511   if (mb->trailing_load()) {
 1512     return true;
 1513   }
 1514 
 1515   if (mb->trailing_load_store()) {
 1516     Node* load_store = mb->in(MemBarNode::Precedent);
 1517     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1518     return is_CAS(load_store->Opcode(), true);
 1519   }
 1520 
 1521   return false;
 1522 }
 1523 
 1524 bool needs_acquiring_load(const Node *n)
 1525 {
 1526   assert(n->is_Load(), "expecting a load");
 1527   LoadNode *ld = n->as_Load();
 1528   return ld->is_acquire();
 1529 }
 1530 
 1531 bool unnecessary_release(const Node *n)
 1532 {
 1533   assert((n->is_MemBar() &&
 1534           n->Opcode() == Op_MemBarRelease),
 1535          "expecting a release membar");
 1536 
 1537   MemBarNode *barrier = n->as_MemBar();
 1538   if (!barrier->leading()) {
 1539     return false;
 1540   } else {
 1541     Node* trailing = barrier->trailing_membar();
 1542     MemBarNode* trailing_mb = trailing->as_MemBar();
 1543     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1544     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1545 
 1546     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1547     if (mem->is_Store()) {
 1548       assert(mem->as_Store()->is_release(), "");
 1549       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1550       return true;
 1551     } else {
 1552       assert(mem->is_LoadStore(), "");
 1553       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1554       return is_CAS(mem->Opcode(), true);
 1555     }
 1556   }
 1557   return false;
 1558 }
 1559 
 1560 bool unnecessary_volatile(const Node *n)
 1561 {
 1562   // assert n->is_MemBar();
 1563   MemBarNode *mbvol = n->as_MemBar();
 1564 
 1565   bool release = mbvol->trailing_store();
 1566   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1567 #ifdef ASSERT
 1568   if (release) {
 1569     Node* leading = mbvol->leading_membar();
 1570     assert(leading->Opcode() == Op_MemBarRelease, "");
 1571     assert(leading->as_MemBar()->leading_store(), "");
 1572     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1573   }
 1574 #endif
 1575 
 1576   return release;
 1577 }
 1578 
 1579 // predicates controlling emit of str<x>/stlr<x>
 1580 
 1581 bool needs_releasing_store(const Node *n)
 1582 {
 1583   // assert n->is_Store();
 1584   StoreNode *st = n->as_Store();
 1585   return st->trailing_membar() != NULL;
 1586 }
 1587 
 1588 // predicate controlling translation of CAS
 1589 //
 1590 // returns true if CAS needs to use an acquiring load otherwise false
 1591 
 1592 bool needs_acquiring_load_exclusive(const Node *n)
 1593 {
 1594   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1595   LoadStoreNode* ldst = n->as_LoadStore();
 1596   if (is_CAS(n->Opcode(), false)) {
 1597     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1598   } else {
 1599     return ldst->trailing_membar() != NULL;
 1600   }
 1601 
 1602   // so we can just return true here
 1603   return true;
 1604 }
 1605 
 1606 #define __ _masm.
 1607 
 1608 // advance declarations for helper functions to convert register
 1609 // indices to register objects
 1610 
 1611 // the ad file has to provide implementations of certain methods
 1612 // expected by the generic code
 1613 //
 1614 // REQUIRED FUNCTIONALITY
 1615 
 1616 //=============================================================================
 1617 
 1618 // !!!!! Special hack to get all types of calls to specify the byte offset
 1619 //       from the start of the call to the point where the return address
 1620 //       will point.
 1621 
 1622 int MachCallStaticJavaNode::ret_addr_offset()
 1623 {
 1624   // call should be a simple bl
 1625   int off = 4;
 1626   return off;
 1627 }
 1628 
 1629 int MachCallDynamicJavaNode::ret_addr_offset()
 1630 {
 1631   return 16; // movz, movk, movk, bl
 1632 }
 1633 
 1634 int MachCallRuntimeNode::ret_addr_offset() {
 1635   // for generated stubs the call will be
 1636   //   bl(addr)
 1637   // or with far branches
 1638   //   bl(trampoline_stub)
 1639   // for real runtime callouts it will be six instructions
 1640   // see aarch64_enc_java_to_runtime
 1641   //   adr(rscratch2, retaddr)
 1642   //   lea(rscratch1, RuntimeAddress(addr)
 1643   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1644   //   blr(rscratch1)
 1645   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1646   if (cb) {
 1647     return 1 * NativeInstruction::instruction_size;
 1648   } else {
 1649     return 6 * NativeInstruction::instruction_size;
 1650   }
 1651 }
 1652 
 1653 //=============================================================================
 1654 
 1655 #ifndef PRODUCT
 1656 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1657   st->print("BREAKPOINT");
 1658 }
 1659 #endif
 1660 
 1661 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1662   C2_MacroAssembler _masm(&cbuf);
 1663   __ brk(0);
 1664 }
 1665 
 1666 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1667   return MachNode::size(ra_);
 1668 }
 1669 
 1670 //=============================================================================
 1671 
 1672 #ifndef PRODUCT
 1673   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1674     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1675   }
 1676 #endif
 1677 
 1678   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1679     C2_MacroAssembler _masm(&cbuf);
 1680     for (int i = 0; i < _count; i++) {
 1681       __ nop();
 1682     }
 1683   }
 1684 
 1685   uint MachNopNode::size(PhaseRegAlloc*) const {
 1686     return _count * NativeInstruction::instruction_size;
 1687   }
 1688 
 1689 //=============================================================================
 1690 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1691 
 1692 int ConstantTable::calculate_table_base_offset() const {
 1693   return 0;  // absolute addressing, no offset
 1694 }
 1695 
 1696 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1697 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1698   ShouldNotReachHere();
 1699 }
 1700 
 1701 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1702   // Empty encoding
 1703 }
 1704 
 1705 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1706   return 0;
 1707 }
 1708 
 1709 #ifndef PRODUCT
 1710 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1711   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1712 }
 1713 #endif
 1714 
 1715 #ifndef PRODUCT
 1716 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1717   Compile* C = ra_->C;
 1718 
 1719   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1720 
 1721   if (C->output()->need_stack_bang(framesize))
 1722     st->print("# stack bang size=%d\n\t", framesize);
 1723 
 1724   if (VM_Version::use_rop_protection()) {
 1725     st->print("ldr zr, [lr]\n\t");
 1726     st->print("pacia  lr, rfp\n\t");
 1727   }
 1728   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1729     st->print("sub  sp, sp, #%d\n\t", framesize);
 1730     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1731     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1732   } else {
 1733     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1734     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1735     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1736     st->print("sub  sp, sp, rscratch1");
 1737   }
 1738   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1739     st->print("\n\t");
 1740     st->print("ldr  rscratch1, [guard]\n\t");
 1741     st->print("dmb ishld\n\t");
 1742     st->print("ldr  rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
 1743     st->print("cmp  rscratch1, rscratch2\n\t");
 1744     st->print("b.eq skip");
 1745     st->print("\n\t");
 1746     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1747     st->print("b skip\n\t");
 1748     st->print("guard: int\n\t");
 1749     st->print("\n\t");
 1750     st->print("skip:\n\t");
 1751   }
 1752 }
 1753 #endif
 1754 
 1755 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1756   Compile* C = ra_->C;
 1757   C2_MacroAssembler _masm(&cbuf);
 1758 
 1759   // n.b. frame size includes space for return pc and rfp
 1760   const int framesize = C->output()->frame_size_in_bytes();
 1761 
 1762   // insert a nop at the start of the prolog so we can patch in a
 1763   // branch if we need to invalidate the method later
 1764   __ nop();
 1765 
 1766   if (C->clinit_barrier_on_entry()) {
 1767     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1768 
 1769     Label L_skip_barrier;
 1770 
 1771     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1772     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1773     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1774     __ bind(L_skip_barrier);
 1775   }
 1776 
 1777   if (C->max_vector_size() > 0) {
 1778     __ reinitialize_ptrue();
 1779   }
 1780 
 1781   int bangsize = C->output()->bang_size_in_bytes();
 1782   if (C->output()->need_stack_bang(bangsize))
 1783     __ generate_stack_overflow_check(bangsize);
 1784 
 1785   __ build_frame(framesize);
 1786 
 1787   int max_monitors = C->method() != NULL ? C->max_monitors() : 0;
 1788   if (UseFastLocking && max_monitors > 0) {
 1789     C2CheckLockStackStub* stub = new (C->comp_arena()) C2CheckLockStackStub();
 1790     C->output()->add_stub(stub);
 1791     __ ldr(r9, Address(rthread, JavaThread::lock_stack_current_offset()));
 1792     __ ldr(r10, Address(rthread, JavaThread::lock_stack_limit_offset()));
 1793     __ add(r9, r9, max_monitors * oopSize);
 1794     __ cmp(r9, r10);
 1795     __ br(Assembler::GE, stub->entry());
 1796     __ bind(stub->continuation());
 1797   }
 1798 
 1799   if (C->stub_function() == NULL) {
 1800     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1801     if (BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1802       // Dummy labels for just measuring the code size
 1803       Label dummy_slow_path;
 1804       Label dummy_continuation;
 1805       Label dummy_guard;
 1806       Label* slow_path = &dummy_slow_path;
 1807       Label* continuation = &dummy_continuation;
 1808       Label* guard = &dummy_guard;
 1809       if (!Compile::current()->output()->in_scratch_emit_size()) {
 1810         // Use real labels from actual stub when not emitting code for the purpose of measuring its size
 1811         C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
 1812         Compile::current()->output()->add_stub(stub);
 1813         slow_path = &stub->entry();
 1814         continuation = &stub->continuation();
 1815         guard = &stub->guard();
 1816       }
 1817       // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
 1818       bs->nmethod_entry_barrier(&_masm, slow_path, continuation, guard);
 1819     }
 1820   }
 1821 
 1822   if (VerifyStackAtCalls) {
 1823     Unimplemented();
 1824   }
 1825 
 1826   C->output()->set_frame_complete(cbuf.insts_size());
 1827 
 1828   if (C->has_mach_constant_base_node()) {
 1829     // NOTE: We set the table base offset here because users might be
 1830     // emitted before MachConstantBaseNode.
 1831     ConstantTable& constant_table = C->output()->constant_table();
 1832     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1833   }
 1834 }
 1835 
 1836 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1837 {
 1838   return MachNode::size(ra_); // too many variables; just compute it
 1839                               // the hard way
 1840 }
 1841 
 1842 int MachPrologNode::reloc() const
 1843 {
 1844   return 0;
 1845 }
 1846 
 1847 //=============================================================================
 1848 
 1849 #ifndef PRODUCT
 1850 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1851   Compile* C = ra_->C;
 1852   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1853 
 1854   st->print("# pop frame %d\n\t",framesize);
 1855 
 1856   if (framesize == 0) {
 1857     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1858   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1859     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1860     st->print("add  sp, sp, #%d\n\t", framesize);
 1861   } else {
 1862     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1863     st->print("add  sp, sp, rscratch1\n\t");
 1864     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1865   }
 1866   if (VM_Version::use_rop_protection()) {
 1867     st->print("autia lr, rfp\n\t");
 1868     st->print("ldr zr, [lr]\n\t");
 1869   }
 1870 
 1871   if (do_polling() && C->is_method_compilation()) {
 1872     st->print("# test polling word\n\t");
 1873     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1874     st->print("cmp  sp, rscratch1\n\t");
 1875     st->print("bhi #slow_path");
 1876   }
 1877 }
 1878 #endif
 1879 
 1880 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1881   Compile* C = ra_->C;
 1882   C2_MacroAssembler _masm(&cbuf);
 1883   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1884 
 1885   __ remove_frame(framesize);
 1886 
 1887   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1888     __ reserved_stack_check();
 1889   }
 1890 
 1891   if (do_polling() && C->is_method_compilation()) {
 1892     Label dummy_label;
 1893     Label* code_stub = &dummy_label;
 1894     if (!C->output()->in_scratch_emit_size()) {
 1895       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
 1896       C->output()->add_stub(stub);
 1897       code_stub = &stub->entry();
 1898     }
 1899     __ relocate(relocInfo::poll_return_type);
 1900     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1901   }
 1902 }
 1903 
 1904 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 1905   // Variable size. Determine dynamically.
 1906   return MachNode::size(ra_);
 1907 }
 1908 
 1909 int MachEpilogNode::reloc() const {
 1910   // Return number of relocatable values contained in this instruction.
 1911   return 1; // 1 for polling page.
 1912 }
 1913 
 1914 const Pipeline * MachEpilogNode::pipeline() const {
 1915   return MachNode::pipeline_class();
 1916 }
 1917 
 1918 //=============================================================================
 1919 
 1920 // Figure out which register class each belongs in: rc_int, rc_float or
 1921 // rc_stack.
 1922 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 1923 
 1924 static enum RC rc_class(OptoReg::Name reg) {
 1925 
 1926   if (reg == OptoReg::Bad) {
 1927     return rc_bad;
 1928   }
 1929 
 1930   // we have 32 int registers * 2 halves
 1931   int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
 1932 
 1933   if (reg < slots_of_int_registers) {
 1934     return rc_int;
 1935   }
 1936 
 1937   // we have 32 float register * 8 halves
 1938   int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
 1939   if (reg < slots_of_int_registers + slots_of_float_registers) {
 1940     return rc_float;
 1941   }
 1942 
 1943   int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
 1944   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 1945     return rc_predicate;
 1946   }
 1947 
 1948   // Between predicate regs & stack is the flags.
 1949   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 1950 
 1951   return rc_stack;
 1952 }
 1953 
 1954 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 1955   Compile* C = ra_->C;
 1956 
 1957   // Get registers to move.
 1958   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 1959   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 1960   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 1961   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 1962 
 1963   enum RC src_hi_rc = rc_class(src_hi);
 1964   enum RC src_lo_rc = rc_class(src_lo);
 1965   enum RC dst_hi_rc = rc_class(dst_hi);
 1966   enum RC dst_lo_rc = rc_class(dst_lo);
 1967 
 1968   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 1969 
 1970   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 1971     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 1972            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 1973            "expected aligned-adjacent pairs");
 1974   }
 1975 
 1976   if (src_lo == dst_lo && src_hi == dst_hi) {
 1977     return 0;            // Self copy, no move.
 1978   }
 1979 
 1980   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 1981               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 1982   int src_offset = ra_->reg2offset(src_lo);
 1983   int dst_offset = ra_->reg2offset(dst_lo);
 1984 
 1985   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 1986     uint ireg = ideal_reg();
 1987     if (ireg == Op_VecA && cbuf) {
 1988       C2_MacroAssembler _masm(cbuf);
 1989       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 1990       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1991         // stack->stack
 1992         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 1993                                                 sve_vector_reg_size_in_bytes);
 1994       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 1995         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 1996                             sve_vector_reg_size_in_bytes);
 1997       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 1998         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 1999                               sve_vector_reg_size_in_bytes);
 2000       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2001         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2002                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 2003                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 2004       } else {
 2005         ShouldNotReachHere();
 2006       }
 2007     } else if (cbuf) {
 2008       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 2009       C2_MacroAssembler _masm(cbuf);
 2010       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 2011       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2012         // stack->stack
 2013         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2014         if (ireg == Op_VecD) {
 2015           __ unspill(rscratch1, true, src_offset);
 2016           __ spill(rscratch1, true, dst_offset);
 2017         } else {
 2018           __ spill_copy128(src_offset, dst_offset);
 2019         }
 2020       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2021         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2022                ireg == Op_VecD ? __ T8B : __ T16B,
 2023                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2024       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2025         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2026                  ireg == Op_VecD ? __ D : __ Q,
 2027                  ra_->reg2offset(dst_lo));
 2028       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2029         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2030                    ireg == Op_VecD ? __ D : __ Q,
 2031                    ra_->reg2offset(src_lo));
 2032       } else {
 2033         ShouldNotReachHere();
 2034       }
 2035     }
 2036   } else if (cbuf) {
 2037     C2_MacroAssembler _masm(cbuf);
 2038     switch (src_lo_rc) {
 2039     case rc_int:
 2040       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2041         if (is64) {
 2042             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2043                    as_Register(Matcher::_regEncode[src_lo]));
 2044         } else {
 2045             C2_MacroAssembler _masm(cbuf);
 2046             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2047                     as_Register(Matcher::_regEncode[src_lo]));
 2048         }
 2049       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2050         if (is64) {
 2051             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2052                      as_Register(Matcher::_regEncode[src_lo]));
 2053         } else {
 2054             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2055                      as_Register(Matcher::_regEncode[src_lo]));
 2056         }
 2057       } else {                    // gpr --> stack spill
 2058         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2059         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2060       }
 2061       break;
 2062     case rc_float:
 2063       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2064         if (is64) {
 2065             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2066                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2067         } else {
 2068             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2069                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2070         }
 2071       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2072         if (is64) {
 2073             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2074                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2075         } else {
 2076             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2077                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2078         }
 2079       } else {                    // fpr --> stack spill
 2080         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2081         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2082                  is64 ? __ D : __ S, dst_offset);
 2083       }
 2084       break;
 2085     case rc_stack:
 2086       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2087         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2088       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2089         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2090                    is64 ? __ D : __ S, src_offset);
 2091       } else if (dst_lo_rc == rc_predicate) {
 2092         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2093                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2094       } else {                    // stack --> stack copy
 2095         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2096         if (ideal_reg() == Op_RegVectMask) {
 2097           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2098                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2099         } else {
 2100           __ unspill(rscratch1, is64, src_offset);
 2101           __ spill(rscratch1, is64, dst_offset);
 2102         }
 2103       }
 2104       break;
 2105     case rc_predicate:
 2106       if (dst_lo_rc == rc_predicate) {
 2107         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2108       } else if (dst_lo_rc == rc_stack) {
 2109         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2110                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2111       } else {
 2112         assert(false, "bad src and dst rc_class combination.");
 2113         ShouldNotReachHere();
 2114       }
 2115       break;
 2116     default:
 2117       assert(false, "bad rc_class for spill");
 2118       ShouldNotReachHere();
 2119     }
 2120   }
 2121 
 2122   if (st) {
 2123     st->print("spill ");
 2124     if (src_lo_rc == rc_stack) {
 2125       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2126     } else {
 2127       st->print("%s -> ", Matcher::regName[src_lo]);
 2128     }
 2129     if (dst_lo_rc == rc_stack) {
 2130       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2131     } else {
 2132       st->print("%s", Matcher::regName[dst_lo]);
 2133     }
 2134     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2135       int vsize = 0;
 2136       switch (ideal_reg()) {
 2137       case Op_VecD:
 2138         vsize = 64;
 2139         break;
 2140       case Op_VecX:
 2141         vsize = 128;
 2142         break;
 2143       case Op_VecA:
 2144         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2145         break;
 2146       default:
 2147         assert(false, "bad register type for spill");
 2148         ShouldNotReachHere();
 2149       }
 2150       st->print("\t# vector spill size = %d", vsize);
 2151     } else if (ideal_reg() == Op_RegVectMask) {
 2152       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2153       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2154       st->print("\t# predicate spill size = %d", vsize);
 2155     } else {
 2156       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2157     }
 2158   }
 2159 
 2160   return 0;
 2161 
 2162 }
 2163 
 2164 #ifndef PRODUCT
 2165 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2166   if (!ra_)
 2167     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2168   else
 2169     implementation(NULL, ra_, false, st);
 2170 }
 2171 #endif
 2172 
 2173 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2174   implementation(&cbuf, ra_, false, NULL);
 2175 }
 2176 
 2177 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2178   return MachNode::size(ra_);
 2179 }
 2180 
 2181 //=============================================================================
 2182 
 2183 #ifndef PRODUCT
 2184 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2185   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2186   int reg = ra_->get_reg_first(this);
 2187   st->print("add %s, rsp, #%d]\t# box lock",
 2188             Matcher::regName[reg], offset);
 2189 }
 2190 #endif
 2191 
 2192 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2193   C2_MacroAssembler _masm(&cbuf);
 2194 
 2195   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2196   int reg    = ra_->get_encode(this);
 2197 
 2198   // This add will handle any 24-bit signed offset. 24 bits allows an
 2199   // 8 megabyte stack frame.
 2200   __ add(as_Register(reg), sp, offset);
 2201 }
 2202 
 2203 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2204   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2205   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2206 
 2207   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2208     return NativeInstruction::instruction_size;
 2209   } else {
 2210     return 2 * NativeInstruction::instruction_size;
 2211   }
 2212 }
 2213 
 2214 //=============================================================================
 2215 
 2216 #ifndef PRODUCT
 2217 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2218 {
 2219   st->print_cr("# MachUEPNode");
 2220   if (UseCompressedClassPointers) {
 2221     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2222     if (CompressedKlassPointers::shift() != 0) {
 2223       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2224     }
 2225   } else {
 2226    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2227   }
 2228   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2229   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2230 }
 2231 #endif
 2232 
 2233 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2234 {
 2235   // This is the unverified entry point.
 2236   C2_MacroAssembler _masm(&cbuf);
 2237 
 2238   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2239   Label skip;
 2240   // TODO
 2241   // can we avoid this skip and still use a reloc?
 2242   __ br(Assembler::EQ, skip);
 2243   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2244   __ bind(skip);
 2245 }
 2246 
 2247 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2248 {
 2249   return MachNode::size(ra_);
 2250 }
 2251 
 2252 // REQUIRED EMIT CODE
 2253 
 2254 //=============================================================================
 2255 
 2256 // Emit exception handler code.
 2257 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2258 {
 2259   // mov rscratch1 #exception_blob_entry_point
 2260   // br rscratch1
 2261   // Note that the code buffer's insts_mark is always relative to insts.
 2262   // That's why we must use the macroassembler to generate a handler.
 2263   C2_MacroAssembler _masm(&cbuf);
 2264   address base = __ start_a_stub(size_exception_handler());
 2265   if (base == NULL) {
 2266     ciEnv::current()->record_failure("CodeCache is full");
 2267     return 0;  // CodeBuffer::expand failed
 2268   }
 2269   int offset = __ offset();
 2270   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2271   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2272   __ end_a_stub();
 2273   return offset;
 2274 }
 2275 
 2276 // Emit deopt handler code.
 2277 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2278 {
 2279   // Note that the code buffer's insts_mark is always relative to insts.
 2280   // That's why we must use the macroassembler to generate a handler.
 2281   C2_MacroAssembler _masm(&cbuf);
 2282   address base = __ start_a_stub(size_deopt_handler());
 2283   if (base == NULL) {
 2284     ciEnv::current()->record_failure("CodeCache is full");
 2285     return 0;  // CodeBuffer::expand failed
 2286   }
 2287   int offset = __ offset();
 2288 
 2289   __ adr(lr, __ pc());
 2290   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2291 
 2292   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2293   __ end_a_stub();
 2294   return offset;
 2295 }
 2296 
 2297 // REQUIRED MATCHER CODE
 2298 
 2299 //=============================================================================
 2300 
 2301 const bool Matcher::match_rule_supported(int opcode) {
 2302   if (!has_match_rule(opcode))
 2303     return false;
 2304 
 2305   bool ret_value = true;
 2306   switch (opcode) {
 2307     case Op_OnSpinWait:
 2308       return VM_Version::supports_on_spin_wait();
 2309     case Op_CacheWB:
 2310     case Op_CacheWBPreSync:
 2311     case Op_CacheWBPostSync:
 2312       if (!VM_Version::supports_data_cache_line_flush()) {
 2313         ret_value = false;
 2314       }
 2315       break;
 2316     case Op_ExpandBits:
 2317     case Op_CompressBits:
 2318       if (!(UseSVE > 1 && VM_Version::supports_svebitperm())) {
 2319         ret_value = false;
 2320       }
 2321       break;
 2322   }
 2323 
 2324   return ret_value; // Per default match rules are supported.
 2325 }
 2326 
 2327 const RegMask* Matcher::predicate_reg_mask(void) {
 2328   return &_PR_REG_mask;
 2329 }
 2330 
 2331 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2332   return new TypeVectMask(elemTy, length);
 2333 }
 2334 
 2335 // Vector calling convention not yet implemented.
 2336 const bool Matcher::supports_vector_calling_convention(void) {
 2337   return false;
 2338 }
 2339 
 2340 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2341   Unimplemented();
 2342   return OptoRegPair(0, 0);
 2343 }
 2344 
 2345 // Is this branch offset short enough that a short branch can be used?
 2346 //
 2347 // NOTE: If the platform does not provide any short branch variants, then
 2348 //       this method should return false for offset 0.
 2349 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2350   // The passed offset is relative to address of the branch.
 2351 
 2352   return (-32768 <= offset && offset < 32768);
 2353 }
 2354 
 2355 // Vector width in bytes.
 2356 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2357   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2358   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2359   // Minimum 2 values in vector
 2360   if (size < 2*type2aelembytes(bt)) size = 0;
 2361   // But never < 4
 2362   if (size < 4) size = 0;
 2363   return size;
 2364 }
 2365 
 2366 // Limits on vector size (number of elements) loaded into vector.
 2367 const int Matcher::max_vector_size(const BasicType bt) {
 2368   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2369 }
 2370 
 2371 const int Matcher::min_vector_size(const BasicType bt) {
 2372   int max_size = max_vector_size(bt);
 2373   // Limit the min vector size to 8 bytes.
 2374   int size = 8 / type2aelembytes(bt);
 2375   if (bt == T_BYTE) {
 2376     // To support vector api shuffle/rearrange.
 2377     size = 4;
 2378   } else if (bt == T_BOOLEAN) {
 2379     // To support vector api load/store mask.
 2380     size = 2;
 2381   }
 2382   if (size < 2) size = 2;
 2383   return MIN2(size, max_size);
 2384 }
 2385 
 2386 // Actual max scalable vector register length.
 2387 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2388   return Matcher::max_vector_size(bt);
 2389 }
 2390 
 2391 // Vector ideal reg.
 2392 const uint Matcher::vector_ideal_reg(int len) {
 2393   if (UseSVE > 0 && 16 < len && len <= 256) {
 2394     return Op_VecA;
 2395   }
 2396   switch(len) {
 2397     // For 16-bit/32-bit mask vector, reuse VecD.
 2398     case  2:
 2399     case  4:
 2400     case  8: return Op_VecD;
 2401     case 16: return Op_VecX;
 2402   }
 2403   ShouldNotReachHere();
 2404   return 0;
 2405 }
 2406 
 2407 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
 2408   assert(Matcher::is_generic_vector(generic_opnd), "not generic");
 2409   switch (ideal_reg) {
 2410     case Op_VecA: return new vecAOper();
 2411     case Op_VecD: return new vecDOper();
 2412     case Op_VecX: return new vecXOper();
 2413   }
 2414   ShouldNotReachHere();
 2415   return NULL;
 2416 }
 2417 
 2418 bool Matcher::is_reg2reg_move(MachNode* m) {
 2419   return false;
 2420 }
 2421 
 2422 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2423   return opnd->opcode() == VREG;
 2424 }
 2425 
 2426 // Return whether or not this register is ever used as an argument.
 2427 // This function is used on startup to build the trampoline stubs in
 2428 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2429 // call in the trampoline, and arguments in those registers not be
 2430 // available to the callee.
 2431 bool Matcher::can_be_java_arg(int reg)
 2432 {
 2433   return
 2434     reg ==  R0_num || reg == R0_H_num ||
 2435     reg ==  R1_num || reg == R1_H_num ||
 2436     reg ==  R2_num || reg == R2_H_num ||
 2437     reg ==  R3_num || reg == R3_H_num ||
 2438     reg ==  R4_num || reg == R4_H_num ||
 2439     reg ==  R5_num || reg == R5_H_num ||
 2440     reg ==  R6_num || reg == R6_H_num ||
 2441     reg ==  R7_num || reg == R7_H_num ||
 2442     reg ==  V0_num || reg == V0_H_num ||
 2443     reg ==  V1_num || reg == V1_H_num ||
 2444     reg ==  V2_num || reg == V2_H_num ||
 2445     reg ==  V3_num || reg == V3_H_num ||
 2446     reg ==  V4_num || reg == V4_H_num ||
 2447     reg ==  V5_num || reg == V5_H_num ||
 2448     reg ==  V6_num || reg == V6_H_num ||
 2449     reg ==  V7_num || reg == V7_H_num;
 2450 }
 2451 
 2452 bool Matcher::is_spillable_arg(int reg)
 2453 {
 2454   return can_be_java_arg(reg);
 2455 }
 2456 
 2457 uint Matcher::int_pressure_limit()
 2458 {
 2459   // JDK-8183543: When taking the number of available registers as int
 2460   // register pressure threshold, the jtreg test:
 2461   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2462   // failed due to C2 compilation failure with
 2463   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2464   //
 2465   // A derived pointer is live at CallNode and then is flagged by RA
 2466   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2467   // derived pointers and lastly fail to spill after reaching maximum
 2468   // number of iterations. Lowering the default pressure threshold to
 2469   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2470   // a high register pressure area of the code so that split_DEF can
 2471   // generate DefinitionSpillCopy for the derived pointer.
 2472   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2473   if (!PreserveFramePointer) {
 2474     // When PreserveFramePointer is off, frame pointer is allocatable,
 2475     // but different from other SOC registers, it is excluded from
 2476     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2477     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2478     // See check_pressure_at_fatproj().
 2479     default_int_pressure_threshold--;
 2480   }
 2481   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2482 }
 2483 
 2484 uint Matcher::float_pressure_limit()
 2485 {
 2486   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2487   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2488 }
 2489 
 2490 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2491   return false;
 2492 }
 2493 
 2494 RegMask Matcher::divI_proj_mask() {
 2495   ShouldNotReachHere();
 2496   return RegMask();
 2497 }
 2498 
 2499 // Register for MODI projection of divmodI.
 2500 RegMask Matcher::modI_proj_mask() {
 2501   ShouldNotReachHere();
 2502   return RegMask();
 2503 }
 2504 
 2505 // Register for DIVL projection of divmodL.
 2506 RegMask Matcher::divL_proj_mask() {
 2507   ShouldNotReachHere();
 2508   return RegMask();
 2509 }
 2510 
 2511 // Register for MODL projection of divmodL.
 2512 RegMask Matcher::modL_proj_mask() {
 2513   ShouldNotReachHere();
 2514   return RegMask();
 2515 }
 2516 
 2517 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2518   return FP_REG_mask();
 2519 }
 2520 
 2521 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2522   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2523     Node* u = addp->fast_out(i);
 2524     if (u->is_LoadStore()) {
 2525       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2526       // instructions) only take register indirect as an operand, so
 2527       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2528       // must fail.
 2529       return false;
 2530     }
 2531     if (u->is_Mem()) {
 2532       int opsize = u->as_Mem()->memory_size();
 2533       assert(opsize > 0, "unexpected memory operand size");
 2534       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2535         return false;
 2536       }
 2537     }
 2538   }
 2539   return true;
 2540 }
 2541 
 2542 // Binary src (Replicate con)
 2543 bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
 2544   if (n == NULL || m == NULL) {
 2545     return false;
 2546   }
 2547 
 2548   if (UseSVE == 0 || !VectorNode::is_invariant_vector(m)) {
 2549     return false;
 2550   }
 2551 
 2552   Node* imm_node = m->in(1);
 2553   if (!imm_node->is_Con()) {
 2554     return false;
 2555   }
 2556 
 2557   const Type* t = imm_node->bottom_type();
 2558   if (!(t->isa_int() || t->isa_long())) {
 2559     return false;
 2560   }
 2561 
 2562   switch (n->Opcode()) {
 2563   case Op_AndV:
 2564   case Op_OrV:
 2565   case Op_XorV: {
 2566     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
 2567     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2568     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2569   }
 2570   case Op_AddVB:
 2571     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2572   case Op_AddVS:
 2573   case Op_AddVI:
 2574     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2575   case Op_AddVL:
 2576     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2577   default:
 2578     return false;
 2579   }
 2580 }
 2581 
 2582 // (XorV src (Replicate m1))
 2583 // (XorVMask src (MaskAll m1))
 2584 bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
 2585   if (n != NULL && m != NULL) {
 2586     return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
 2587            VectorNode::is_all_ones_vector(m);
 2588   }
 2589   return false;
 2590 }
 2591 
 2592 // Should the matcher clone input 'm' of node 'n'?
 2593 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2594   if (is_vshift_con_pattern(n, m) ||
 2595       is_vector_bitwise_not_pattern(n, m) ||
 2596       is_valid_sve_arith_imm_pattern(n, m)) {
 2597     mstack.push(m, Visit);
 2598     return true;
 2599   }
 2600   return false;
 2601 }
 2602 
 2603 // Should the Matcher clone shifts on addressing modes, expecting them
 2604 // to be subsumed into complex addressing expressions or compute them
 2605 // into registers?
 2606 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2607   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2608     return true;
 2609   }
 2610 
 2611   Node *off = m->in(AddPNode::Offset);
 2612   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2613       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2614       // Are there other uses besides address expressions?
 2615       !is_visited(off)) {
 2616     address_visited.set(off->_idx); // Flag as address_visited
 2617     mstack.push(off->in(2), Visit);
 2618     Node *conv = off->in(1);
 2619     if (conv->Opcode() == Op_ConvI2L &&
 2620         // Are there other uses besides address expressions?
 2621         !is_visited(conv)) {
 2622       address_visited.set(conv->_idx); // Flag as address_visited
 2623       mstack.push(conv->in(1), Pre_Visit);
 2624     } else {
 2625       mstack.push(conv, Pre_Visit);
 2626     }
 2627     address_visited.test_set(m->_idx); // Flag as address_visited
 2628     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2629     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2630     return true;
 2631   } else if (off->Opcode() == Op_ConvI2L &&
 2632              // Are there other uses besides address expressions?
 2633              !is_visited(off)) {
 2634     address_visited.test_set(m->_idx); // Flag as address_visited
 2635     address_visited.set(off->_idx); // Flag as address_visited
 2636     mstack.push(off->in(1), Pre_Visit);
 2637     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2638     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2639     return true;
 2640   }
 2641   return false;
 2642 }
 2643 
 2644 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2645   C2_MacroAssembler _masm(&cbuf);                                       \
 2646   {                                                                     \
 2647     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2648     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2649     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2650     __ INSN(REG, as_Register(BASE));                                    \
 2651   }
 2652 
 2653 
 2654 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2655   {
 2656     Address::extend scale;
 2657 
 2658     // Hooboy, this is fugly.  We need a way to communicate to the
 2659     // encoder that the index needs to be sign extended, so we have to
 2660     // enumerate all the cases.
 2661     switch (opcode) {
 2662     case INDINDEXSCALEDI2L:
 2663     case INDINDEXSCALEDI2LN:
 2664     case INDINDEXI2L:
 2665     case INDINDEXI2LN:
 2666       scale = Address::sxtw(size);
 2667       break;
 2668     default:
 2669       scale = Address::lsl(size);
 2670     }
 2671 
 2672     if (index == -1) {
 2673       return Address(base, disp);
 2674     } else {
 2675       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2676       return Address(base, as_Register(index), scale);
 2677     }
 2678   }
 2679 
 2680 
 2681 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2682 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2683 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2684 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2685                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2686 
 2687   // Used for all non-volatile memory accesses.  The use of
 2688   // $mem->opcode() to discover whether this pattern uses sign-extended
 2689   // offsets is something of a kludge.
 2690   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2691                         Register reg, int opcode,
 2692                         Register base, int index, int scale, int disp,
 2693                         int size_in_memory)
 2694   {
 2695     Address addr = mem2address(opcode, base, index, scale, disp);
 2696     if (addr.getMode() == Address::base_plus_offset) {
 2697       /* If we get an out-of-range offset it is a bug in the compiler,
 2698          so we assert here. */
 2699       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2700              "c2 compiler bug");
 2701       /* Fix up any out-of-range offsets. */
 2702       assert_different_registers(rscratch1, base);
 2703       assert_different_registers(rscratch1, reg);
 2704       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2705     }
 2706     (masm.*insn)(reg, addr);
 2707   }
 2708 
 2709   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2710                         FloatRegister reg, int opcode,
 2711                         Register base, int index, int size, int disp,
 2712                         int size_in_memory)
 2713   {
 2714     Address::extend scale;
 2715 
 2716     switch (opcode) {
 2717     case INDINDEXSCALEDI2L:
 2718     case INDINDEXSCALEDI2LN:
 2719       scale = Address::sxtw(size);
 2720       break;
 2721     default:
 2722       scale = Address::lsl(size);
 2723     }
 2724 
 2725     if (index == -1) {
 2726       /* If we get an out-of-range offset it is a bug in the compiler,
 2727          so we assert here. */
 2728       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2729       /* Fix up any out-of-range offsets. */
 2730       assert_different_registers(rscratch1, base);
 2731       Address addr = Address(base, disp);
 2732       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2733       (masm.*insn)(reg, addr);
 2734     } else {
 2735       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2736       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2737     }
 2738   }
 2739 
 2740   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2741                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2742                         int opcode, Register base, int index, int size, int disp)
 2743   {
 2744     if (index == -1) {
 2745       (masm.*insn)(reg, T, Address(base, disp));
 2746     } else {
 2747       assert(disp == 0, "unsupported address mode");
 2748       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2749     }
 2750   }
 2751 
 2752 %}
 2753 
 2754 
 2755 
 2756 //----------ENCODING BLOCK-----------------------------------------------------
 2757 // This block specifies the encoding classes used by the compiler to
 2758 // output byte streams.  Encoding classes are parameterized macros
 2759 // used by Machine Instruction Nodes in order to generate the bit
 2760 // encoding of the instruction.  Operands specify their base encoding
 2761 // interface with the interface keyword.  There are currently
 2762 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2763 // COND_INTER.  REG_INTER causes an operand to generate a function
 2764 // which returns its register number when queried.  CONST_INTER causes
 2765 // an operand to generate a function which returns the value of the
 2766 // constant when queried.  MEMORY_INTER causes an operand to generate
 2767 // four functions which return the Base Register, the Index Register,
 2768 // the Scale Value, and the Offset Value of the operand when queried.
 2769 // COND_INTER causes an operand to generate six functions which return
 2770 // the encoding code (ie - encoding bits for the instruction)
 2771 // associated with each basic boolean condition for a conditional
 2772 // instruction.
 2773 //
 2774 // Instructions specify two basic values for encoding.  Again, a
 2775 // function is available to check if the constant displacement is an
 2776 // oop. They use the ins_encode keyword to specify their encoding
 2777 // classes (which must be a sequence of enc_class names, and their
 2778 // parameters, specified in the encoding block), and they use the
 2779 // opcode keyword to specify, in order, their primary, secondary, and
 2780 // tertiary opcode.  Only the opcode sections which a particular
 2781 // instruction needs for encoding need to be specified.
 2782 encode %{
 2783   // Build emit functions for each basic byte or larger field in the
 2784   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2785   // from C++ code in the enc_class source block.  Emit functions will
 2786   // live in the main source block for now.  In future, we can
 2787   // generalize this by adding a syntax that specifies the sizes of
 2788   // fields in an order, so that the adlc can build the emit functions
 2789   // automagically
 2790 
 2791   // catch all for unimplemented encodings
 2792   enc_class enc_unimplemented %{
 2793     C2_MacroAssembler _masm(&cbuf);
 2794     __ unimplemented("C2 catch all");
 2795   %}
 2796 
 2797   // BEGIN Non-volatile memory access
 2798 
 2799   // This encoding class is generated automatically from ad_encode.m4.
 2800   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2801   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2802     Register dst_reg = as_Register($dst$$reg);
 2803     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2804                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2805   %}
 2806 
 2807   // This encoding class is generated automatically from ad_encode.m4.
 2808   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2809   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2810     Register dst_reg = as_Register($dst$$reg);
 2811     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2812                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2813   %}
 2814 
 2815   // This encoding class is generated automatically from ad_encode.m4.
 2816   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2817   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2818     Register dst_reg = as_Register($dst$$reg);
 2819     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2820                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2821   %}
 2822 
 2823   // This encoding class is generated automatically from ad_encode.m4.
 2824   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2825   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2826     Register dst_reg = as_Register($dst$$reg);
 2827     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2828                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2829   %}
 2830 
 2831   // This encoding class is generated automatically from ad_encode.m4.
 2832   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2833   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2834     Register dst_reg = as_Register($dst$$reg);
 2835     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2836                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2837   %}
 2838 
 2839   // This encoding class is generated automatically from ad_encode.m4.
 2840   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2841   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2842     Register dst_reg = as_Register($dst$$reg);
 2843     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2844                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2845   %}
 2846 
 2847   // This encoding class is generated automatically from ad_encode.m4.
 2848   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2849   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2850     Register dst_reg = as_Register($dst$$reg);
 2851     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2852                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2853   %}
 2854 
 2855   // This encoding class is generated automatically from ad_encode.m4.
 2856   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2857   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2858     Register dst_reg = as_Register($dst$$reg);
 2859     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2860                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2861   %}
 2862 
 2863   // This encoding class is generated automatically from ad_encode.m4.
 2864   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2865   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2866     Register dst_reg = as_Register($dst$$reg);
 2867     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2868                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2869   %}
 2870 
 2871   // This encoding class is generated automatically from ad_encode.m4.
 2872   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2873   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2874     Register dst_reg = as_Register($dst$$reg);
 2875     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2876                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2877   %}
 2878 
 2879   // This encoding class is generated automatically from ad_encode.m4.
 2880   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2881   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2882     Register dst_reg = as_Register($dst$$reg);
 2883     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2884                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2885   %}
 2886 
 2887   // This encoding class is generated automatically from ad_encode.m4.
 2888   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2889   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2890     Register dst_reg = as_Register($dst$$reg);
 2891     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2892                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2893   %}
 2894 
 2895   // This encoding class is generated automatically from ad_encode.m4.
 2896   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2897   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2898     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2899     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2900                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2901   %}
 2902 
 2903   // This encoding class is generated automatically from ad_encode.m4.
 2904   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2905   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 2906     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2907     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2908                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2909   %}
 2910 
 2911   // This encoding class is generated automatically from ad_encode.m4.
 2912   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2913   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 2914     Register src_reg = as_Register($src$$reg);
 2915     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 2916                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2917   %}
 2918 
 2919   // This encoding class is generated automatically from ad_encode.m4.
 2920   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2921   enc_class aarch64_enc_strb0(memory1 mem) %{
 2922     C2_MacroAssembler _masm(&cbuf);
 2923     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2924                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2925   %}
 2926 
 2927   // This encoding class is generated automatically from ad_encode.m4.
 2928   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2929   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 2930     Register src_reg = as_Register($src$$reg);
 2931     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 2932                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2933   %}
 2934 
 2935   // This encoding class is generated automatically from ad_encode.m4.
 2936   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2937   enc_class aarch64_enc_strh0(memory2 mem) %{
 2938     C2_MacroAssembler _masm(&cbuf);
 2939     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 2940                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2941   %}
 2942 
 2943   // This encoding class is generated automatically from ad_encode.m4.
 2944   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2945   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 2946     Register src_reg = as_Register($src$$reg);
 2947     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 2948                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2949   %}
 2950 
 2951   // This encoding class is generated automatically from ad_encode.m4.
 2952   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2953   enc_class aarch64_enc_strw0(memory4 mem) %{
 2954     C2_MacroAssembler _masm(&cbuf);
 2955     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 2956                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2957   %}
 2958 
 2959   // This encoding class is generated automatically from ad_encode.m4.
 2960   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2961   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 2962     Register src_reg = as_Register($src$$reg);
 2963     // we sometimes get asked to store the stack pointer into the
 2964     // current thread -- we cannot do that directly on AArch64
 2965     if (src_reg == r31_sp) {
 2966       C2_MacroAssembler _masm(&cbuf);
 2967       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 2968       __ mov(rscratch2, sp);
 2969       src_reg = rscratch2;
 2970     }
 2971     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 2972                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2973   %}
 2974 
 2975   // This encoding class is generated automatically from ad_encode.m4.
 2976   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2977   enc_class aarch64_enc_str0(memory8 mem) %{
 2978     C2_MacroAssembler _masm(&cbuf);
 2979     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 2980                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2981   %}
 2982 
 2983   // This encoding class is generated automatically from ad_encode.m4.
 2984   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2985   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 2986     FloatRegister src_reg = as_FloatRegister($src$$reg);
 2987     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 2988                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2989   %}
 2990 
 2991   // This encoding class is generated automatically from ad_encode.m4.
 2992   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2993   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 2994     FloatRegister src_reg = as_FloatRegister($src$$reg);
 2995     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 2996                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2997   %}
 2998 
 2999   // This encoding class is generated automatically from ad_encode.m4.
 3000   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3001   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3002       C2_MacroAssembler _masm(&cbuf);
 3003       __ membar(Assembler::StoreStore);
 3004       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3005                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3006   %}
 3007 
 3008   // END Non-volatile memory access
 3009 
 3010   // Vector loads and stores
 3011   enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
 3012     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3013     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3014        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3015   %}
 3016 
 3017   enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
 3018     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3019     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3020        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3021   %}
 3022 
 3023   enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
 3024     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3025     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3026        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3027   %}
 3028 
 3029   enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
 3030     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3031     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3032        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3033   %}
 3034 
 3035   enc_class aarch64_enc_strvH(vReg src, memory mem) %{
 3036     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3037     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3038        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3039   %}
 3040 
 3041   enc_class aarch64_enc_strvS(vReg src, memory mem) %{
 3042     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3043     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3044        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3045   %}
 3046 
 3047   enc_class aarch64_enc_strvD(vReg src, memory mem) %{
 3048     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3049     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3050        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3051   %}
 3052 
 3053   enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
 3054     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3055     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3056        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3057   %}
 3058 
 3059   // volatile loads and stores
 3060 
 3061   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3062     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3063                  rscratch1, stlrb);
 3064   %}
 3065 
 3066   enc_class aarch64_enc_stlrb0(memory mem) %{
 3067     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3068                  rscratch1, stlrb);
 3069   %}
 3070 
 3071   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3072     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3073                  rscratch1, stlrh);
 3074   %}
 3075 
 3076   enc_class aarch64_enc_stlrh0(memory mem) %{
 3077     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3078                  rscratch1, stlrh);
 3079   %}
 3080 
 3081   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3082     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3083                  rscratch1, stlrw);
 3084   %}
 3085 
 3086   enc_class aarch64_enc_stlrw0(memory mem) %{
 3087     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3088                  rscratch1, stlrw);
 3089   %}
 3090 
 3091   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3092     Register dst_reg = as_Register($dst$$reg);
 3093     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3094              rscratch1, ldarb);
 3095     __ sxtbw(dst_reg, dst_reg);
 3096   %}
 3097 
 3098   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3099     Register dst_reg = as_Register($dst$$reg);
 3100     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3101              rscratch1, ldarb);
 3102     __ sxtb(dst_reg, dst_reg);
 3103   %}
 3104 
 3105   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3106     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3107              rscratch1, ldarb);
 3108   %}
 3109 
 3110   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3111     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3112              rscratch1, ldarb);
 3113   %}
 3114 
 3115   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3116     Register dst_reg = as_Register($dst$$reg);
 3117     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3118              rscratch1, ldarh);
 3119     __ sxthw(dst_reg, dst_reg);
 3120   %}
 3121 
 3122   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3123     Register dst_reg = as_Register($dst$$reg);
 3124     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3125              rscratch1, ldarh);
 3126     __ sxth(dst_reg, dst_reg);
 3127   %}
 3128 
 3129   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3130     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3131              rscratch1, ldarh);
 3132   %}
 3133 
 3134   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3135     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3136              rscratch1, ldarh);
 3137   %}
 3138 
 3139   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3140     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3141              rscratch1, ldarw);
 3142   %}
 3143 
 3144   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3145     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3146              rscratch1, ldarw);
 3147   %}
 3148 
 3149   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3150     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3151              rscratch1, ldar);
 3152   %}
 3153 
 3154   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3155     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3156              rscratch1, ldarw);
 3157     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3158   %}
 3159 
 3160   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3161     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3162              rscratch1, ldar);
 3163     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3164   %}
 3165 
 3166   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3167     Register src_reg = as_Register($src$$reg);
 3168     // we sometimes get asked to store the stack pointer into the
 3169     // current thread -- we cannot do that directly on AArch64
 3170     if (src_reg == r31_sp) {
 3171       C2_MacroAssembler _masm(&cbuf);
 3172       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3173       __ mov(rscratch2, sp);
 3174       src_reg = rscratch2;
 3175     }
 3176     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3177                  rscratch1, stlr);
 3178   %}
 3179 
 3180   enc_class aarch64_enc_stlr0(memory mem) %{
 3181     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3182                  rscratch1, stlr);
 3183   %}
 3184 
 3185   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3186     {
 3187       C2_MacroAssembler _masm(&cbuf);
 3188       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3189       __ fmovs(rscratch2, src_reg);
 3190     }
 3191     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3192                  rscratch1, stlrw);
 3193   %}
 3194 
 3195   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3196     {
 3197       C2_MacroAssembler _masm(&cbuf);
 3198       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3199       __ fmovd(rscratch2, src_reg);
 3200     }
 3201     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3202                  rscratch1, stlr);
 3203   %}
 3204 
 3205   // synchronized read/update encodings
 3206 
 3207   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3208     C2_MacroAssembler _masm(&cbuf);
 3209     Register dst_reg = as_Register($dst$$reg);
 3210     Register base = as_Register($mem$$base);
 3211     int index = $mem$$index;
 3212     int scale = $mem$$scale;
 3213     int disp = $mem$$disp;
 3214     if (index == -1) {
 3215        if (disp != 0) {
 3216         __ lea(rscratch1, Address(base, disp));
 3217         __ ldaxr(dst_reg, rscratch1);
 3218       } else {
 3219         // TODO
 3220         // should we ever get anything other than this case?
 3221         __ ldaxr(dst_reg, base);
 3222       }
 3223     } else {
 3224       Register index_reg = as_Register(index);
 3225       if (disp == 0) {
 3226         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3227         __ ldaxr(dst_reg, rscratch1);
 3228       } else {
 3229         __ lea(rscratch1, Address(base, disp));
 3230         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3231         __ ldaxr(dst_reg, rscratch1);
 3232       }
 3233     }
 3234   %}
 3235 
 3236   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3237     C2_MacroAssembler _masm(&cbuf);
 3238     Register src_reg = as_Register($src$$reg);
 3239     Register base = as_Register($mem$$base);
 3240     int index = $mem$$index;
 3241     int scale = $mem$$scale;
 3242     int disp = $mem$$disp;
 3243     if (index == -1) {
 3244        if (disp != 0) {
 3245         __ lea(rscratch2, Address(base, disp));
 3246         __ stlxr(rscratch1, src_reg, rscratch2);
 3247       } else {
 3248         // TODO
 3249         // should we ever get anything other than this case?
 3250         __ stlxr(rscratch1, src_reg, base);
 3251       }
 3252     } else {
 3253       Register index_reg = as_Register(index);
 3254       if (disp == 0) {
 3255         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3256         __ stlxr(rscratch1, src_reg, rscratch2);
 3257       } else {
 3258         __ lea(rscratch2, Address(base, disp));
 3259         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3260         __ stlxr(rscratch1, src_reg, rscratch2);
 3261       }
 3262     }
 3263     __ cmpw(rscratch1, zr);
 3264   %}
 3265 
 3266   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3267     C2_MacroAssembler _masm(&cbuf);
 3268     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3269     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3270                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3271                /*weak*/ false, noreg);
 3272   %}
 3273 
 3274   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3275     C2_MacroAssembler _masm(&cbuf);
 3276     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3277     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3278                Assembler::word, /*acquire*/ false, /*release*/ true,
 3279                /*weak*/ false, noreg);
 3280   %}
 3281 
 3282   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3283     C2_MacroAssembler _masm(&cbuf);
 3284     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3285     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3286                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3287                /*weak*/ false, noreg);
 3288   %}
 3289 
 3290   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3291     C2_MacroAssembler _masm(&cbuf);
 3292     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3293     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3294                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3295                /*weak*/ false, noreg);
 3296   %}
 3297 
 3298 
 3299   // The only difference between aarch64_enc_cmpxchg and
 3300   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3301   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3302   // lock.
 3303   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3304     C2_MacroAssembler _masm(&cbuf);
 3305     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3306     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3307                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3308                /*weak*/ false, noreg);
 3309   %}
 3310 
 3311   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3312     C2_MacroAssembler _masm(&cbuf);
 3313     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3314     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3315                Assembler::word, /*acquire*/ true, /*release*/ true,
 3316                /*weak*/ false, noreg);
 3317   %}
 3318 
 3319   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3320     C2_MacroAssembler _masm(&cbuf);
 3321     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3322     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3323                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3324                /*weak*/ false, noreg);
 3325   %}
 3326 
 3327   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3328     C2_MacroAssembler _masm(&cbuf);
 3329     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3330     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3331                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3332                /*weak*/ false, noreg);
 3333   %}
 3334 
 3335   // auxiliary used for CompareAndSwapX to set result register
 3336   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3337     C2_MacroAssembler _masm(&cbuf);
 3338     Register res_reg = as_Register($res$$reg);
 3339     __ cset(res_reg, Assembler::EQ);
 3340   %}
 3341 
 3342   // prefetch encodings
 3343 
 3344   enc_class aarch64_enc_prefetchw(memory mem) %{
 3345     C2_MacroAssembler _masm(&cbuf);
 3346     Register base = as_Register($mem$$base);
 3347     int index = $mem$$index;
 3348     int scale = $mem$$scale;
 3349     int disp = $mem$$disp;
 3350     if (index == -1) {
 3351       __ prfm(Address(base, disp), PSTL1KEEP);
 3352     } else {
 3353       Register index_reg = as_Register(index);
 3354       if (disp == 0) {
 3355         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3356       } else {
 3357         __ lea(rscratch1, Address(base, disp));
 3358 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3359       }
 3360     }
 3361   %}
 3362 
 3363   /// mov envcodings
 3364 
 3365   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3366     C2_MacroAssembler _masm(&cbuf);
 3367     uint32_t con = (uint32_t)$src$$constant;
 3368     Register dst_reg = as_Register($dst$$reg);
 3369     if (con == 0) {
 3370       __ movw(dst_reg, zr);
 3371     } else {
 3372       __ movw(dst_reg, con);
 3373     }
 3374   %}
 3375 
 3376   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3377     C2_MacroAssembler _masm(&cbuf);
 3378     Register dst_reg = as_Register($dst$$reg);
 3379     uint64_t con = (uint64_t)$src$$constant;
 3380     if (con == 0) {
 3381       __ mov(dst_reg, zr);
 3382     } else {
 3383       __ mov(dst_reg, con);
 3384     }
 3385   %}
 3386 
 3387   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3388     C2_MacroAssembler _masm(&cbuf);
 3389     Register dst_reg = as_Register($dst$$reg);
 3390     address con = (address)$src$$constant;
 3391     if (con == NULL || con == (address)1) {
 3392       ShouldNotReachHere();
 3393     } else {
 3394       relocInfo::relocType rtype = $src->constant_reloc();
 3395       if (rtype == relocInfo::oop_type) {
 3396         __ movoop(dst_reg, (jobject)con);
 3397       } else if (rtype == relocInfo::metadata_type) {
 3398         __ mov_metadata(dst_reg, (Metadata*)con);
 3399       } else {
 3400         assert(rtype == relocInfo::none, "unexpected reloc type");
 3401         if (! __ is_valid_AArch64_address(con) ||
 3402             con < (address)(uintptr_t)os::vm_page_size()) {
 3403           __ mov(dst_reg, con);
 3404         } else {
 3405           uint64_t offset;
 3406           __ adrp(dst_reg, con, offset);
 3407           __ add(dst_reg, dst_reg, offset);
 3408         }
 3409       }
 3410     }
 3411   %}
 3412 
 3413   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3414     C2_MacroAssembler _masm(&cbuf);
 3415     Register dst_reg = as_Register($dst$$reg);
 3416     __ mov(dst_reg, zr);
 3417   %}
 3418 
 3419   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3420     C2_MacroAssembler _masm(&cbuf);
 3421     Register dst_reg = as_Register($dst$$reg);
 3422     __ mov(dst_reg, (uint64_t)1);
 3423   %}
 3424 
 3425   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3426     C2_MacroAssembler _masm(&cbuf);
 3427     __ load_byte_map_base($dst$$Register);
 3428   %}
 3429 
 3430   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3431     C2_MacroAssembler _masm(&cbuf);
 3432     Register dst_reg = as_Register($dst$$reg);
 3433     address con = (address)$src$$constant;
 3434     if (con == NULL) {
 3435       ShouldNotReachHere();
 3436     } else {
 3437       relocInfo::relocType rtype = $src->constant_reloc();
 3438       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3439       __ set_narrow_oop(dst_reg, (jobject)con);
 3440     }
 3441   %}
 3442 
 3443   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3444     C2_MacroAssembler _masm(&cbuf);
 3445     Register dst_reg = as_Register($dst$$reg);
 3446     __ mov(dst_reg, zr);
 3447   %}
 3448 
 3449   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3450     C2_MacroAssembler _masm(&cbuf);
 3451     Register dst_reg = as_Register($dst$$reg);
 3452     address con = (address)$src$$constant;
 3453     if (con == NULL) {
 3454       ShouldNotReachHere();
 3455     } else {
 3456       relocInfo::relocType rtype = $src->constant_reloc();
 3457       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3458       __ set_narrow_klass(dst_reg, (Klass *)con);
 3459     }
 3460   %}
 3461 
 3462   // arithmetic encodings
 3463 
 3464   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3465     C2_MacroAssembler _masm(&cbuf);
 3466     Register dst_reg = as_Register($dst$$reg);
 3467     Register src_reg = as_Register($src1$$reg);
 3468     int32_t con = (int32_t)$src2$$constant;
 3469     // add has primary == 0, subtract has primary == 1
 3470     if ($primary) { con = -con; }
 3471     if (con < 0) {
 3472       __ subw(dst_reg, src_reg, -con);
 3473     } else {
 3474       __ addw(dst_reg, src_reg, con);
 3475     }
 3476   %}
 3477 
 3478   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3479     C2_MacroAssembler _masm(&cbuf);
 3480     Register dst_reg = as_Register($dst$$reg);
 3481     Register src_reg = as_Register($src1$$reg);
 3482     int32_t con = (int32_t)$src2$$constant;
 3483     // add has primary == 0, subtract has primary == 1
 3484     if ($primary) { con = -con; }
 3485     if (con < 0) {
 3486       __ sub(dst_reg, src_reg, -con);
 3487     } else {
 3488       __ add(dst_reg, src_reg, con);
 3489     }
 3490   %}
 3491 
 3492   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3493     C2_MacroAssembler _masm(&cbuf);
 3494    Register dst_reg = as_Register($dst$$reg);
 3495    Register src1_reg = as_Register($src1$$reg);
 3496    Register src2_reg = as_Register($src2$$reg);
 3497     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3498   %}
 3499 
 3500   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3501     C2_MacroAssembler _masm(&cbuf);
 3502    Register dst_reg = as_Register($dst$$reg);
 3503    Register src1_reg = as_Register($src1$$reg);
 3504    Register src2_reg = as_Register($src2$$reg);
 3505     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3506   %}
 3507 
 3508   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3509     C2_MacroAssembler _masm(&cbuf);
 3510    Register dst_reg = as_Register($dst$$reg);
 3511    Register src1_reg = as_Register($src1$$reg);
 3512    Register src2_reg = as_Register($src2$$reg);
 3513     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3514   %}
 3515 
 3516   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3517     C2_MacroAssembler _masm(&cbuf);
 3518    Register dst_reg = as_Register($dst$$reg);
 3519    Register src1_reg = as_Register($src1$$reg);
 3520    Register src2_reg = as_Register($src2$$reg);
 3521     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3522   %}
 3523 
 3524   // compare instruction encodings
 3525 
 3526   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3527     C2_MacroAssembler _masm(&cbuf);
 3528     Register reg1 = as_Register($src1$$reg);
 3529     Register reg2 = as_Register($src2$$reg);
 3530     __ cmpw(reg1, reg2);
 3531   %}
 3532 
 3533   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3534     C2_MacroAssembler _masm(&cbuf);
 3535     Register reg = as_Register($src1$$reg);
 3536     int32_t val = $src2$$constant;
 3537     if (val >= 0) {
 3538       __ subsw(zr, reg, val);
 3539     } else {
 3540       __ addsw(zr, reg, -val);
 3541     }
 3542   %}
 3543 
 3544   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3545     C2_MacroAssembler _masm(&cbuf);
 3546     Register reg1 = as_Register($src1$$reg);
 3547     uint32_t val = (uint32_t)$src2$$constant;
 3548     __ movw(rscratch1, val);
 3549     __ cmpw(reg1, rscratch1);
 3550   %}
 3551 
 3552   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3553     C2_MacroAssembler _masm(&cbuf);
 3554     Register reg1 = as_Register($src1$$reg);
 3555     Register reg2 = as_Register($src2$$reg);
 3556     __ cmp(reg1, reg2);
 3557   %}
 3558 
 3559   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3560     C2_MacroAssembler _masm(&cbuf);
 3561     Register reg = as_Register($src1$$reg);
 3562     int64_t val = $src2$$constant;
 3563     if (val >= 0) {
 3564       __ subs(zr, reg, val);
 3565     } else if (val != -val) {
 3566       __ adds(zr, reg, -val);
 3567     } else {
 3568     // aargh, Long.MIN_VALUE is a special case
 3569       __ orr(rscratch1, zr, (uint64_t)val);
 3570       __ subs(zr, reg, rscratch1);
 3571     }
 3572   %}
 3573 
 3574   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3575     C2_MacroAssembler _masm(&cbuf);
 3576     Register reg1 = as_Register($src1$$reg);
 3577     uint64_t val = (uint64_t)$src2$$constant;
 3578     __ mov(rscratch1, val);
 3579     __ cmp(reg1, rscratch1);
 3580   %}
 3581 
 3582   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3583     C2_MacroAssembler _masm(&cbuf);
 3584     Register reg1 = as_Register($src1$$reg);
 3585     Register reg2 = as_Register($src2$$reg);
 3586     __ cmp(reg1, reg2);
 3587   %}
 3588 
 3589   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3590     C2_MacroAssembler _masm(&cbuf);
 3591     Register reg1 = as_Register($src1$$reg);
 3592     Register reg2 = as_Register($src2$$reg);
 3593     __ cmpw(reg1, reg2);
 3594   %}
 3595 
 3596   enc_class aarch64_enc_testp(iRegP src) %{
 3597     C2_MacroAssembler _masm(&cbuf);
 3598     Register reg = as_Register($src$$reg);
 3599     __ cmp(reg, zr);
 3600   %}
 3601 
 3602   enc_class aarch64_enc_testn(iRegN src) %{
 3603     C2_MacroAssembler _masm(&cbuf);
 3604     Register reg = as_Register($src$$reg);
 3605     __ cmpw(reg, zr);
 3606   %}
 3607 
 3608   enc_class aarch64_enc_b(label lbl) %{
 3609     C2_MacroAssembler _masm(&cbuf);
 3610     Label *L = $lbl$$label;
 3611     __ b(*L);
 3612   %}
 3613 
 3614   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3615     C2_MacroAssembler _masm(&cbuf);
 3616     Label *L = $lbl$$label;
 3617     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3618   %}
 3619 
 3620   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3621     C2_MacroAssembler _masm(&cbuf);
 3622     Label *L = $lbl$$label;
 3623     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3624   %}
 3625 
 3626   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3627   %{
 3628      Register sub_reg = as_Register($sub$$reg);
 3629      Register super_reg = as_Register($super$$reg);
 3630      Register temp_reg = as_Register($temp$$reg);
 3631      Register result_reg = as_Register($result$$reg);
 3632 
 3633      Label miss;
 3634      C2_MacroAssembler _masm(&cbuf);
 3635      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3636                                      NULL, &miss,
 3637                                      /*set_cond_codes:*/ true);
 3638      if ($primary) {
 3639        __ mov(result_reg, zr);
 3640      }
 3641      __ bind(miss);
 3642   %}
 3643 
 3644   enc_class aarch64_enc_java_static_call(method meth) %{
 3645     C2_MacroAssembler _masm(&cbuf);
 3646 
 3647     address addr = (address)$meth$$method;
 3648     address call;
 3649     if (!_method) {
 3650       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3651       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
 3652       if (call == NULL) {
 3653         ciEnv::current()->record_failure("CodeCache is full");
 3654         return;
 3655       }
 3656     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 3657       // The NOP here is purely to ensure that eliding a call to
 3658       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 3659       __ nop();
 3660       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 3661     } else {
 3662       int method_index = resolved_method_index(cbuf);
 3663       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3664                                                   : static_call_Relocation::spec(method_index);
 3665       call = __ trampoline_call(Address(addr, rspec));
 3666       if (call == NULL) {
 3667         ciEnv::current()->record_failure("CodeCache is full");
 3668         return;
 3669       }
 3670       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 3671         // Calls of the same statically bound method can share
 3672         // a stub to the interpreter.
 3673         cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
 3674       } else {
 3675         // Emit stub for static call
 3676         address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
 3677         if (stub == NULL) {
 3678           ciEnv::current()->record_failure("CodeCache is full");
 3679           return;
 3680         }
 3681       }
 3682     }
 3683 
 3684     __ post_call_nop();
 3685 
 3686     // Only non uncommon_trap calls need to reinitialize ptrue.
 3687     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3688       __ reinitialize_ptrue();
 3689     }
 3690   %}
 3691 
 3692   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3693     C2_MacroAssembler _masm(&cbuf);
 3694     int method_index = resolved_method_index(cbuf);
 3695     address call = __ ic_call((address)$meth$$method, method_index);
 3696     if (call == NULL) {
 3697       ciEnv::current()->record_failure("CodeCache is full");
 3698       return;
 3699     }
 3700     __ post_call_nop();
 3701     if (Compile::current()->max_vector_size() > 0) {
 3702       __ reinitialize_ptrue();
 3703     }
 3704   %}
 3705 
 3706   enc_class aarch64_enc_call_epilog() %{
 3707     C2_MacroAssembler _masm(&cbuf);
 3708     if (VerifyStackAtCalls) {
 3709       // Check that stack depth is unchanged: find majik cookie on stack
 3710       __ call_Unimplemented();
 3711     }
 3712   %}
 3713 
 3714   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3715     C2_MacroAssembler _masm(&cbuf);
 3716 
 3717     // some calls to generated routines (arraycopy code) are scheduled
 3718     // by C2 as runtime calls. if so we can call them using a br (they
 3719     // will be in a reachable segment) otherwise we have to use a blr
 3720     // which loads the absolute address into a register.
 3721     address entry = (address)$meth$$method;
 3722     CodeBlob *cb = CodeCache::find_blob(entry);
 3723     if (cb) {
 3724       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3725       if (call == NULL) {
 3726         ciEnv::current()->record_failure("CodeCache is full");
 3727         return;
 3728       }
 3729       __ post_call_nop();
 3730     } else {
 3731       Label retaddr;
 3732       __ adr(rscratch2, retaddr);
 3733       __ lea(rscratch1, RuntimeAddress(entry));
 3734       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3735       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3736       __ blr(rscratch1);
 3737       __ bind(retaddr);
 3738       __ post_call_nop();
 3739       __ add(sp, sp, 2 * wordSize);
 3740     }
 3741     if (Compile::current()->max_vector_size() > 0) {
 3742       __ reinitialize_ptrue();
 3743     }
 3744   %}
 3745 
 3746   enc_class aarch64_enc_rethrow() %{
 3747     C2_MacroAssembler _masm(&cbuf);
 3748     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3749   %}
 3750 
 3751   enc_class aarch64_enc_ret() %{
 3752     C2_MacroAssembler _masm(&cbuf);
 3753 #ifdef ASSERT
 3754     if (Compile::current()->max_vector_size() > 0) {
 3755       __ verify_ptrue();
 3756     }
 3757 #endif
 3758     __ ret(lr);
 3759   %}
 3760 
 3761   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3762     C2_MacroAssembler _masm(&cbuf);
 3763     Register target_reg = as_Register($jump_target$$reg);
 3764     __ br(target_reg);
 3765   %}
 3766 
 3767   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3768     C2_MacroAssembler _masm(&cbuf);
 3769     Register target_reg = as_Register($jump_target$$reg);
 3770     // exception oop should be in r0
 3771     // ret addr has been popped into lr
 3772     // callee expects it in r3
 3773     __ mov(r3, lr);
 3774     __ br(target_reg);
 3775   %}
 3776 
 3777   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3778     C2_MacroAssembler _masm(&cbuf);
 3779     Register oop = as_Register($object$$reg);
 3780     Register box = as_Register($box$$reg);
 3781     Register disp_hdr = as_Register($tmp$$reg);
 3782     Register tmp = as_Register($tmp2$$reg);
 3783     Label cont;
 3784     Label object_has_monitor;
 3785     Label count, no_count;
 3786 
 3787     assert_different_registers(oop, box, tmp, disp_hdr);
 3788 
 3789     // Load markWord from object into displaced_header.
 3790     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3791 
 3792     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3793       __ load_klass(tmp, oop);
 3794       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3795       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3796       __ br(Assembler::NE, cont);
 3797     }
 3798 
 3799     // Check for existing monitor
 3800     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3801 
 3802     if (!UseHeavyMonitors) {
 3803       if (UseFastLocking) {
 3804         __ fast_lock(oop, disp_hdr, tmp, rscratch1, no_count, false);
 3805 
 3806         // Indicate success at cont.
 3807         __ cmp(oop, oop);
 3808         __ b(count);
 3809       } else {
 3810         // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3811         __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3812 
 3813         // Initialize the box. (Must happen before we update the object mark!)
 3814         __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3815 
 3816         // Compare object markWord with an unlocked value (tmp) and if
 3817         // equal exchange the stack address of our box with object markWord.
 3818         // On failure disp_hdr contains the possibly locked markWord.
 3819         __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3820                    /*release*/ true, /*weak*/ false, disp_hdr);
 3821         __ br(Assembler::EQ, cont);
 3822 
 3823         assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3824 
 3825         // If the compare-and-exchange succeeded, then we found an unlocked
 3826         // object, will have now locked it will continue at label cont
 3827 
 3828         // Check if the owner is self by comparing the value in the
 3829         // markWord of object (disp_hdr) with the stack pointer.
 3830         __ mov(rscratch1, sp);
 3831         __ sub(disp_hdr, disp_hdr, rscratch1);
 3832         __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3833         // If condition is true we are cont and hence we can store 0 as the
 3834         // displaced header in the box, which indicates that it is a recursive lock.
 3835         __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3836         __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3837         __ b(cont);
 3838       }
 3839     } else {
 3840       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3841       __ b(cont);
 3842     }
 3843 
 3844     // Handle existing monitor.
 3845     __ bind(object_has_monitor);
 3846 
 3847     // The object's monitor m is unlocked iff m->owner == NULL,
 3848     // otherwise m->owner may contain a thread or a stack address.
 3849     //
 3850     // Try to CAS m->owner from NULL to current thread.
 3851     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3852     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3853                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 3854 
 3855     if (!UseFastLocking) {
 3856       // Store a non-null value into the box to avoid looking like a re-entrant
 3857       // lock. The fast-path monitor unlock code checks for
 3858       // markWord::monitor_value so use markWord::unused_mark which has the
 3859       // relevant bit set, and also matches ObjectSynchronizer::enter.
 3860       __ mov(tmp, (address)markWord::unused_mark().value());
 3861       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3862     }
 3863     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 3864 
 3865     __ cmp(rscratch1, rthread);
 3866     __ br(Assembler::NE, cont); // Check for recursive locking
 3867 
 3868     // Recursive lock case
 3869     __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
 3870     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 3871 
 3872     __ bind(cont);
 3873     // flag == EQ indicates success
 3874     // flag == NE indicates failure
 3875     __ br(Assembler::NE, no_count);
 3876 
 3877     __ bind(count);
 3878     __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
 3879 
 3880     __ bind(no_count);
 3881   %}
 3882 
 3883   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3884     C2_MacroAssembler _masm(&cbuf);
 3885     Register oop = as_Register($object$$reg);
 3886     Register box = as_Register($box$$reg);
 3887     Register disp_hdr = as_Register($tmp$$reg);
 3888     Register tmp = as_Register($tmp2$$reg);
 3889     Label cont;
 3890     Label object_has_monitor;
 3891     Label count, no_count;
 3892 
 3893     assert_different_registers(oop, box, tmp, disp_hdr);
 3894 
 3895     if (!UseHeavyMonitors && !UseFastLocking) {
 3896       // Find the lock address and load the displaced header from the stack.
 3897       __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3898 
 3899       // If the displaced header is 0, we have a recursive unlock.
 3900       __ cmp(disp_hdr, zr);
 3901       __ br(Assembler::EQ, cont);
 3902     }
 3903 
 3904     // Handle existing monitor.
 3905     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 3906     __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
 3907 
 3908     if (!UseHeavyMonitors) {
 3909       if (UseFastLocking) {
 3910         __ fast_unlock(oop, tmp, box, disp_hdr, no_count);
 3911 
 3912         // Indicate success at cont.
 3913         __ cmp(oop, oop);
 3914         __ b(count);
 3915       } else {
 3916         // Check if it is still a light weight lock, this is is true if we
 3917         // see the stack address of the basicLock in the markWord of the
 3918         // object.
 3919 
 3920         __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 3921                    /*release*/ true, /*weak*/ false, tmp);
 3922         __ b(cont);
 3923       }
 3924     } else {
 3925       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3926       __ b(cont);
 3927     }
 3928 
 3929     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3930 
 3931     // Handle existing monitor.
 3932     __ bind(object_has_monitor);
 3933     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 3934     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 3935 
 3936     if (UseFastLocking) {
 3937       // If the owner is anonymous, we need to fix it -- in an outline stub.
 3938       Register tmp2 = disp_hdr;
 3939       __ ldr(tmp2, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3940       // We cannot use tbnz here, the target might be too far away and cannot
 3941       // be encoded.
 3942       __ tst(tmp2, (uint64_t)(intptr_t) ANONYMOUS_OWNER);
 3943       C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
 3944       Compile::current()->output()->add_stub(stub);
 3945       __ br(Assembler::NE, stub->entry());
 3946       __ bind(stub->continuation());
 3947     }
 3948 
 3949     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3950 
 3951     Label notRecursive;
 3952     __ cbz(disp_hdr, notRecursive);
 3953 
 3954     // Recursive lock
 3955     __ sub(disp_hdr, disp_hdr, 1u);
 3956     __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3957     __ cmp(disp_hdr, disp_hdr); // Sets flags for result
 3958     __ b(cont);
 3959 
 3960     __ bind(notRecursive);
 3961     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 3962     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 3963     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 3964     __ cmp(rscratch1, zr); // Sets flags for result
 3965     __ cbnz(rscratch1, cont);
 3966     // need a release store here
 3967     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3968     __ stlr(zr, tmp); // set unowned
 3969 
 3970     __ bind(cont);
 3971     // flag == EQ indicates success
 3972     // flag == NE indicates failure
 3973     __ br(Assembler::NE, no_count);
 3974 
 3975     __ bind(count);
 3976     __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
 3977 
 3978     __ bind(no_count);
 3979   %}
 3980 
 3981 %}
 3982 
 3983 //----------FRAME--------------------------------------------------------------
 3984 // Definition of frame structure and management information.
 3985 //
 3986 //  S T A C K   L A Y O U T    Allocators stack-slot number
 3987 //                             |   (to get allocators register number
 3988 //  G  Owned by    |        |  v    add OptoReg::stack0())
 3989 //  r   CALLER     |        |
 3990 //  o     |        +--------+      pad to even-align allocators stack-slot
 3991 //  w     V        |  pad0  |        numbers; owned by CALLER
 3992 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 3993 //  h     ^        |   in   |  5
 3994 //        |        |  args  |  4   Holes in incoming args owned by SELF
 3995 //  |     |        |        |  3
 3996 //  |     |        +--------+
 3997 //  V     |        | old out|      Empty on Intel, window on Sparc
 3998 //        |    old |preserve|      Must be even aligned.
 3999 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 4000 //        |        |   in   |  3   area for Intel ret address
 4001 //     Owned by    |preserve|      Empty on Sparc.
 4002 //       SELF      +--------+
 4003 //        |        |  pad2  |  2   pad to align old SP
 4004 //        |        +--------+  1
 4005 //        |        | locks  |  0
 4006 //        |        +--------+----> OptoReg::stack0(), even aligned
 4007 //        |        |  pad1  | 11   pad to align new SP
 4008 //        |        +--------+
 4009 //        |        |        | 10
 4010 //        |        | spills |  9   spills
 4011 //        V        |        |  8   (pad0 slot for callee)
 4012 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 4013 //        ^        |  out   |  7
 4014 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 4015 //     Owned by    +--------+
 4016 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 4017 //        |    new |preserve|      Must be even-aligned.
 4018 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 4019 //        |        |        |
 4020 //
 4021 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 4022 //         known from SELF's arguments and the Java calling convention.
 4023 //         Region 6-7 is determined per call site.
 4024 // Note 2: If the calling convention leaves holes in the incoming argument
 4025 //         area, those holes are owned by SELF.  Holes in the outgoing area
 4026 //         are owned by the CALLEE.  Holes should not be necessary in the
 4027 //         incoming area, as the Java calling convention is completely under
 4028 //         the control of the AD file.  Doubles can be sorted and packed to
 4029 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 4030 //         varargs C calling conventions.
 4031 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 4032 //         even aligned with pad0 as needed.
 4033 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 4034 //           (the latter is true on Intel but is it false on AArch64?)
 4035 //         region 6-11 is even aligned; it may be padded out more so that
 4036 //         the region from SP to FP meets the minimum stack alignment.
 4037 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 4038 //         alignment.  Region 11, pad1, may be dynamically extended so that
 4039 //         SP meets the minimum alignment.
 4040 
 4041 frame %{
 4042   // These three registers define part of the calling convention
 4043   // between compiled code and the interpreter.
 4044 
 4045   // Inline Cache Register or Method for I2C.
 4046   inline_cache_reg(R12);
 4047 
 4048   // Number of stack slots consumed by locking an object
 4049   sync_stack_slots(2);
 4050 
 4051   // Compiled code's Frame Pointer
 4052   frame_pointer(R31);
 4053 
 4054   // Interpreter stores its frame pointer in a register which is
 4055   // stored to the stack by I2CAdaptors.
 4056   // I2CAdaptors convert from interpreted java to compiled java.
 4057   interpreter_frame_pointer(R29);
 4058 
 4059   // Stack alignment requirement
 4060   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4061 
 4062   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4063   // for calls to C.  Supports the var-args backing area for register parms.
 4064   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4065 
 4066   // The after-PROLOG location of the return address.  Location of
 4067   // return address specifies a type (REG or STACK) and a number
 4068   // representing the register number (i.e. - use a register name) or
 4069   // stack slot.
 4070   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4071   // Otherwise, it is above the locks and verification slot and alignment word
 4072   // TODO this may well be correct but need to check why that - 2 is there
 4073   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4074   // which folds in the space used for monitors
 4075   return_addr(STACK - 2 +
 4076               align_up((Compile::current()->in_preserve_stack_slots() +
 4077                         Compile::current()->fixed_slots()),
 4078                        stack_alignment_in_slots()));
 4079 
 4080   // Location of compiled Java return values.  Same as C for now.
 4081   return_value
 4082   %{
 4083     // TODO do we allow ideal_reg == Op_RegN???
 4084     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4085            "only return normal values");
 4086 
 4087     static const int lo[Op_RegL + 1] = { // enum name
 4088       0,                                 // Op_Node
 4089       0,                                 // Op_Set
 4090       R0_num,                            // Op_RegN
 4091       R0_num,                            // Op_RegI
 4092       R0_num,                            // Op_RegP
 4093       V0_num,                            // Op_RegF
 4094       V0_num,                            // Op_RegD
 4095       R0_num                             // Op_RegL
 4096     };
 4097 
 4098     static const int hi[Op_RegL + 1] = { // enum name
 4099       0,                                 // Op_Node
 4100       0,                                 // Op_Set
 4101       OptoReg::Bad,                      // Op_RegN
 4102       OptoReg::Bad,                      // Op_RegI
 4103       R0_H_num,                          // Op_RegP
 4104       OptoReg::Bad,                      // Op_RegF
 4105       V0_H_num,                          // Op_RegD
 4106       R0_H_num                           // Op_RegL
 4107     };
 4108 
 4109     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4110   %}
 4111 %}
 4112 
 4113 //----------ATTRIBUTES---------------------------------------------------------
 4114 //----------Operand Attributes-------------------------------------------------
 4115 op_attrib op_cost(1);        // Required cost attribute
 4116 
 4117 //----------Instruction Attributes---------------------------------------------
 4118 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4119 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4120 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4121                                 // a non-matching short branch variant
 4122                                 // of some long branch?
 4123 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4124                                 // be a power of 2) specifies the
 4125                                 // alignment that some part of the
 4126                                 // instruction (not necessarily the
 4127                                 // start) requires.  If > 1, a
 4128                                 // compute_padding() function must be
 4129                                 // provided for the instruction
 4130 
 4131 //----------OPERANDS-----------------------------------------------------------
 4132 // Operand definitions must precede instruction definitions for correct parsing
 4133 // in the ADLC because operands constitute user defined types which are used in
 4134 // instruction definitions.
 4135 
 4136 //----------Simple Operands----------------------------------------------------
 4137 
 4138 // Integer operands 32 bit
 4139 // 32 bit immediate
 4140 operand immI()
 4141 %{
 4142   match(ConI);
 4143 
 4144   op_cost(0);
 4145   format %{ %}
 4146   interface(CONST_INTER);
 4147 %}
 4148 
 4149 // 32 bit zero
 4150 operand immI0()
 4151 %{
 4152   predicate(n->get_int() == 0);
 4153   match(ConI);
 4154 
 4155   op_cost(0);
 4156   format %{ %}
 4157   interface(CONST_INTER);
 4158 %}
 4159 
 4160 // 32 bit unit increment
 4161 operand immI_1()
 4162 %{
 4163   predicate(n->get_int() == 1);
 4164   match(ConI);
 4165 
 4166   op_cost(0);
 4167   format %{ %}
 4168   interface(CONST_INTER);
 4169 %}
 4170 
 4171 // 32 bit unit decrement
 4172 operand immI_M1()
 4173 %{
 4174   predicate(n->get_int() == -1);
 4175   match(ConI);
 4176 
 4177   op_cost(0);
 4178   format %{ %}
 4179   interface(CONST_INTER);
 4180 %}
 4181 
 4182 // Shift values for add/sub extension shift
 4183 operand immIExt()
 4184 %{
 4185   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4186   match(ConI);
 4187 
 4188   op_cost(0);
 4189   format %{ %}
 4190   interface(CONST_INTER);
 4191 %}
 4192 
 4193 operand immI_gt_1()
 4194 %{
 4195   predicate(n->get_int() > 1);
 4196   match(ConI);
 4197 
 4198   op_cost(0);
 4199   format %{ %}
 4200   interface(CONST_INTER);
 4201 %}
 4202 
 4203 operand immI_le_4()
 4204 %{
 4205   predicate(n->get_int() <= 4);
 4206   match(ConI);
 4207 
 4208   op_cost(0);
 4209   format %{ %}
 4210   interface(CONST_INTER);
 4211 %}
 4212 
 4213 operand immI_16()
 4214 %{
 4215   predicate(n->get_int() == 16);
 4216   match(ConI);
 4217 
 4218   op_cost(0);
 4219   format %{ %}
 4220   interface(CONST_INTER);
 4221 %}
 4222 
 4223 operand immI_24()
 4224 %{
 4225   predicate(n->get_int() == 24);
 4226   match(ConI);
 4227 
 4228   op_cost(0);
 4229   format %{ %}
 4230   interface(CONST_INTER);
 4231 %}
 4232 
 4233 operand immI_32()
 4234 %{
 4235   predicate(n->get_int() == 32);
 4236   match(ConI);
 4237 
 4238   op_cost(0);
 4239   format %{ %}
 4240   interface(CONST_INTER);
 4241 %}
 4242 
 4243 operand immI_48()
 4244 %{
 4245   predicate(n->get_int() == 48);
 4246   match(ConI);
 4247 
 4248   op_cost(0);
 4249   format %{ %}
 4250   interface(CONST_INTER);
 4251 %}
 4252 
 4253 operand immI_56()
 4254 %{
 4255   predicate(n->get_int() == 56);
 4256   match(ConI);
 4257 
 4258   op_cost(0);
 4259   format %{ %}
 4260   interface(CONST_INTER);
 4261 %}
 4262 
 4263 operand immI_63()
 4264 %{
 4265   predicate(n->get_int() == 63);
 4266   match(ConI);
 4267 
 4268   op_cost(0);
 4269   format %{ %}
 4270   interface(CONST_INTER);
 4271 %}
 4272 
 4273 operand immI_64()
 4274 %{
 4275   predicate(n->get_int() == 64);
 4276   match(ConI);
 4277 
 4278   op_cost(0);
 4279   format %{ %}
 4280   interface(CONST_INTER);
 4281 %}
 4282 
 4283 operand immI_255()
 4284 %{
 4285   predicate(n->get_int() == 255);
 4286   match(ConI);
 4287 
 4288   op_cost(0);
 4289   format %{ %}
 4290   interface(CONST_INTER);
 4291 %}
 4292 
 4293 operand immI_65535()
 4294 %{
 4295   predicate(n->get_int() == 65535);
 4296   match(ConI);
 4297 
 4298   op_cost(0);
 4299   format %{ %}
 4300   interface(CONST_INTER);
 4301 %}
 4302 
 4303 operand immI_positive()
 4304 %{
 4305   predicate(n->get_int() > 0);
 4306   match(ConI);
 4307 
 4308   op_cost(0);
 4309   format %{ %}
 4310   interface(CONST_INTER);
 4311 %}
 4312 
 4313 operand immL_255()
 4314 %{
 4315   predicate(n->get_long() == 255L);
 4316   match(ConL);
 4317 
 4318   op_cost(0);
 4319   format %{ %}
 4320   interface(CONST_INTER);
 4321 %}
 4322 
 4323 operand immL_65535()
 4324 %{
 4325   predicate(n->get_long() == 65535L);
 4326   match(ConL);
 4327 
 4328   op_cost(0);
 4329   format %{ %}
 4330   interface(CONST_INTER);
 4331 %}
 4332 
 4333 operand immL_4294967295()
 4334 %{
 4335   predicate(n->get_long() == 4294967295L);
 4336   match(ConL);
 4337 
 4338   op_cost(0);
 4339   format %{ %}
 4340   interface(CONST_INTER);
 4341 %}
 4342 
 4343 operand immL_bitmask()
 4344 %{
 4345   predicate((n->get_long() != 0)
 4346             && ((n->get_long() & 0xc000000000000000l) == 0)
 4347             && is_power_of_2(n->get_long() + 1));
 4348   match(ConL);
 4349 
 4350   op_cost(0);
 4351   format %{ %}
 4352   interface(CONST_INTER);
 4353 %}
 4354 
 4355 operand immI_bitmask()
 4356 %{
 4357   predicate((n->get_int() != 0)
 4358             && ((n->get_int() & 0xc0000000) == 0)
 4359             && is_power_of_2(n->get_int() + 1));
 4360   match(ConI);
 4361 
 4362   op_cost(0);
 4363   format %{ %}
 4364   interface(CONST_INTER);
 4365 %}
 4366 
 4367 operand immL_positive_bitmaskI()
 4368 %{
 4369   predicate((n->get_long() != 0)
 4370             && ((julong)n->get_long() < 0x80000000ULL)
 4371             && is_power_of_2(n->get_long() + 1));
 4372   match(ConL);
 4373 
 4374   op_cost(0);
 4375   format %{ %}
 4376   interface(CONST_INTER);
 4377 %}
 4378 
 4379 // Scale values for scaled offset addressing modes (up to long but not quad)
 4380 operand immIScale()
 4381 %{
 4382   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4383   match(ConI);
 4384 
 4385   op_cost(0);
 4386   format %{ %}
 4387   interface(CONST_INTER);
 4388 %}
 4389 
 4390 // 26 bit signed offset -- for pc-relative branches
 4391 operand immI26()
 4392 %{
 4393   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4394   match(ConI);
 4395 
 4396   op_cost(0);
 4397   format %{ %}
 4398   interface(CONST_INTER);
 4399 %}
 4400 
 4401 // 19 bit signed offset -- for pc-relative loads
 4402 operand immI19()
 4403 %{
 4404   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4405   match(ConI);
 4406 
 4407   op_cost(0);
 4408   format %{ %}
 4409   interface(CONST_INTER);
 4410 %}
 4411 
 4412 // 12 bit unsigned offset -- for base plus immediate loads
 4413 operand immIU12()
 4414 %{
 4415   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4416   match(ConI);
 4417 
 4418   op_cost(0);
 4419   format %{ %}
 4420   interface(CONST_INTER);
 4421 %}
 4422 
 4423 operand immLU12()
 4424 %{
 4425   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4426   match(ConL);
 4427 
 4428   op_cost(0);
 4429   format %{ %}
 4430   interface(CONST_INTER);
 4431 %}
 4432 
 4433 // Offset for scaled or unscaled immediate loads and stores
 4434 operand immIOffset()
 4435 %{
 4436   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4437   match(ConI);
 4438 
 4439   op_cost(0);
 4440   format %{ %}
 4441   interface(CONST_INTER);
 4442 %}
 4443 
 4444 operand immIOffset1()
 4445 %{
 4446   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4447   match(ConI);
 4448 
 4449   op_cost(0);
 4450   format %{ %}
 4451   interface(CONST_INTER);
 4452 %}
 4453 
 4454 operand immIOffset2()
 4455 %{
 4456   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4457   match(ConI);
 4458 
 4459   op_cost(0);
 4460   format %{ %}
 4461   interface(CONST_INTER);
 4462 %}
 4463 
 4464 operand immIOffset4()
 4465 %{
 4466   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4467   match(ConI);
 4468 
 4469   op_cost(0);
 4470   format %{ %}
 4471   interface(CONST_INTER);
 4472 %}
 4473 
 4474 operand immIOffset8()
 4475 %{
 4476   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4477   match(ConI);
 4478 
 4479   op_cost(0);
 4480   format %{ %}
 4481   interface(CONST_INTER);
 4482 %}
 4483 
 4484 operand immIOffset16()
 4485 %{
 4486   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4487   match(ConI);
 4488 
 4489   op_cost(0);
 4490   format %{ %}
 4491   interface(CONST_INTER);
 4492 %}
 4493 
 4494 operand immLoffset()
 4495 %{
 4496   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4497   match(ConL);
 4498 
 4499   op_cost(0);
 4500   format %{ %}
 4501   interface(CONST_INTER);
 4502 %}
 4503 
 4504 operand immLoffset1()
 4505 %{
 4506   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4507   match(ConL);
 4508 
 4509   op_cost(0);
 4510   format %{ %}
 4511   interface(CONST_INTER);
 4512 %}
 4513 
 4514 operand immLoffset2()
 4515 %{
 4516   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4517   match(ConL);
 4518 
 4519   op_cost(0);
 4520   format %{ %}
 4521   interface(CONST_INTER);
 4522 %}
 4523 
 4524 operand immLoffset4()
 4525 %{
 4526   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4527   match(ConL);
 4528 
 4529   op_cost(0);
 4530   format %{ %}
 4531   interface(CONST_INTER);
 4532 %}
 4533 
 4534 operand immLoffset8()
 4535 %{
 4536   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4537   match(ConL);
 4538 
 4539   op_cost(0);
 4540   format %{ %}
 4541   interface(CONST_INTER);
 4542 %}
 4543 
 4544 operand immLoffset16()
 4545 %{
 4546   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4547   match(ConL);
 4548 
 4549   op_cost(0);
 4550   format %{ %}
 4551   interface(CONST_INTER);
 4552 %}
 4553 
 4554 // 8 bit signed value.
 4555 operand immI8()
 4556 %{
 4557   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4558   match(ConI);
 4559 
 4560   op_cost(0);
 4561   format %{ %}
 4562   interface(CONST_INTER);
 4563 %}
 4564 
 4565 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4566 operand immI8_shift8()
 4567 %{
 4568   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4569             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4570   match(ConI);
 4571 
 4572   op_cost(0);
 4573   format %{ %}
 4574   interface(CONST_INTER);
 4575 %}
 4576 
 4577 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4578 operand immL8_shift8()
 4579 %{
 4580   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4581             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4582   match(ConL);
 4583 
 4584   op_cost(0);
 4585   format %{ %}
 4586   interface(CONST_INTER);
 4587 %}
 4588 
 4589 // 8 bit integer valid for vector add sub immediate
 4590 operand immBAddSubV()
 4591 %{
 4592   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4593   match(ConI);
 4594 
 4595   op_cost(0);
 4596   format %{ %}
 4597   interface(CONST_INTER);
 4598 %}
 4599 
 4600 // 32 bit integer valid for add sub immediate
 4601 operand immIAddSub()
 4602 %{
 4603   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4604   match(ConI);
 4605   op_cost(0);
 4606   format %{ %}
 4607   interface(CONST_INTER);
 4608 %}
 4609 
 4610 // 32 bit integer valid for vector add sub immediate
 4611 operand immIAddSubV()
 4612 %{
 4613   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4614   match(ConI);
 4615 
 4616   op_cost(0);
 4617   format %{ %}
 4618   interface(CONST_INTER);
 4619 %}
 4620 
 4621 // 32 bit unsigned integer valid for logical immediate
 4622 
 4623 operand immBLog()
 4624 %{
 4625   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4626   match(ConI);
 4627 
 4628   op_cost(0);
 4629   format %{ %}
 4630   interface(CONST_INTER);
 4631 %}
 4632 
 4633 operand immSLog()
 4634 %{
 4635   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4636   match(ConI);
 4637 
 4638   op_cost(0);
 4639   format %{ %}
 4640   interface(CONST_INTER);
 4641 %}
 4642 
 4643 operand immILog()
 4644 %{
 4645   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4646   match(ConI);
 4647 
 4648   op_cost(0);
 4649   format %{ %}
 4650   interface(CONST_INTER);
 4651 %}
 4652 
 4653 // Integer operands 64 bit
 4654 // 64 bit immediate
 4655 operand immL()
 4656 %{
 4657   match(ConL);
 4658 
 4659   op_cost(0);
 4660   format %{ %}
 4661   interface(CONST_INTER);
 4662 %}
 4663 
 4664 // 64 bit zero
 4665 operand immL0()
 4666 %{
 4667   predicate(n->get_long() == 0);
 4668   match(ConL);
 4669 
 4670   op_cost(0);
 4671   format %{ %}
 4672   interface(CONST_INTER);
 4673 %}
 4674 
 4675 // 64 bit unit increment
 4676 operand immL_1()
 4677 %{
 4678   predicate(n->get_long() == 1);
 4679   match(ConL);
 4680 
 4681   op_cost(0);
 4682   format %{ %}
 4683   interface(CONST_INTER);
 4684 %}
 4685 
 4686 // 64 bit unit decrement
 4687 operand immL_M1()
 4688 %{
 4689   predicate(n->get_long() == -1);
 4690   match(ConL);
 4691 
 4692   op_cost(0);
 4693   format %{ %}
 4694   interface(CONST_INTER);
 4695 %}
 4696 
 4697 // 32 bit offset of pc in thread anchor
 4698 
 4699 operand immL_pc_off()
 4700 %{
 4701   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4702                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4703   match(ConL);
 4704 
 4705   op_cost(0);
 4706   format %{ %}
 4707   interface(CONST_INTER);
 4708 %}
 4709 
 4710 // 64 bit integer valid for add sub immediate
 4711 operand immLAddSub()
 4712 %{
 4713   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4714   match(ConL);
 4715   op_cost(0);
 4716   format %{ %}
 4717   interface(CONST_INTER);
 4718 %}
 4719 
 4720 // 64 bit integer valid for addv subv immediate
 4721 operand immLAddSubV()
 4722 %{
 4723   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4724   match(ConL);
 4725 
 4726   op_cost(0);
 4727   format %{ %}
 4728   interface(CONST_INTER);
 4729 %}
 4730 
 4731 // 64 bit integer valid for logical immediate
 4732 operand immLLog()
 4733 %{
 4734   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4735   match(ConL);
 4736   op_cost(0);
 4737   format %{ %}
 4738   interface(CONST_INTER);
 4739 %}
 4740 
 4741 // Long Immediate: low 32-bit mask
 4742 operand immL_32bits()
 4743 %{
 4744   predicate(n->get_long() == 0xFFFFFFFFL);
 4745   match(ConL);
 4746   op_cost(0);
 4747   format %{ %}
 4748   interface(CONST_INTER);
 4749 %}
 4750 
 4751 // Pointer operands
 4752 // Pointer Immediate
 4753 operand immP()
 4754 %{
 4755   match(ConP);
 4756 
 4757   op_cost(0);
 4758   format %{ %}
 4759   interface(CONST_INTER);
 4760 %}
 4761 
 4762 // NULL Pointer Immediate
 4763 operand immP0()
 4764 %{
 4765   predicate(n->get_ptr() == 0);
 4766   match(ConP);
 4767 
 4768   op_cost(0);
 4769   format %{ %}
 4770   interface(CONST_INTER);
 4771 %}
 4772 
 4773 // Pointer Immediate One
 4774 // this is used in object initialization (initial object header)
 4775 operand immP_1()
 4776 %{
 4777   predicate(n->get_ptr() == 1);
 4778   match(ConP);
 4779 
 4780   op_cost(0);
 4781   format %{ %}
 4782   interface(CONST_INTER);
 4783 %}
 4784 
 4785 // Card Table Byte Map Base
 4786 operand immByteMapBase()
 4787 %{
 4788   // Get base of card map
 4789   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4790             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4791   match(ConP);
 4792 
 4793   op_cost(0);
 4794   format %{ %}
 4795   interface(CONST_INTER);
 4796 %}
 4797 
 4798 // Pointer Immediate Minus One
 4799 // this is used when we want to write the current PC to the thread anchor
 4800 operand immP_M1()
 4801 %{
 4802   predicate(n->get_ptr() == -1);
 4803   match(ConP);
 4804 
 4805   op_cost(0);
 4806   format %{ %}
 4807   interface(CONST_INTER);
 4808 %}
 4809 
 4810 // Pointer Immediate Minus Two
 4811 // this is used when we want to write the current PC to the thread anchor
 4812 operand immP_M2()
 4813 %{
 4814   predicate(n->get_ptr() == -2);
 4815   match(ConP);
 4816 
 4817   op_cost(0);
 4818   format %{ %}
 4819   interface(CONST_INTER);
 4820 %}
 4821 
 4822 // Float and Double operands
 4823 // Double Immediate
 4824 operand immD()
 4825 %{
 4826   match(ConD);
 4827   op_cost(0);
 4828   format %{ %}
 4829   interface(CONST_INTER);
 4830 %}
 4831 
 4832 // Double Immediate: +0.0d
 4833 operand immD0()
 4834 %{
 4835   predicate(jlong_cast(n->getd()) == 0);
 4836   match(ConD);
 4837 
 4838   op_cost(0);
 4839   format %{ %}
 4840   interface(CONST_INTER);
 4841 %}
 4842 
 4843 // constant 'double +0.0'.
 4844 operand immDPacked()
 4845 %{
 4846   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4847   match(ConD);
 4848   op_cost(0);
 4849   format %{ %}
 4850   interface(CONST_INTER);
 4851 %}
 4852 
 4853 // Float Immediate
 4854 operand immF()
 4855 %{
 4856   match(ConF);
 4857   op_cost(0);
 4858   format %{ %}
 4859   interface(CONST_INTER);
 4860 %}
 4861 
 4862 // Float Immediate: +0.0f.
 4863 operand immF0()
 4864 %{
 4865   predicate(jint_cast(n->getf()) == 0);
 4866   match(ConF);
 4867 
 4868   op_cost(0);
 4869   format %{ %}
 4870   interface(CONST_INTER);
 4871 %}
 4872 
 4873 //
 4874 operand immFPacked()
 4875 %{
 4876   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4877   match(ConF);
 4878   op_cost(0);
 4879   format %{ %}
 4880   interface(CONST_INTER);
 4881 %}
 4882 
 4883 // Narrow pointer operands
 4884 // Narrow Pointer Immediate
 4885 operand immN()
 4886 %{
 4887   match(ConN);
 4888 
 4889   op_cost(0);
 4890   format %{ %}
 4891   interface(CONST_INTER);
 4892 %}
 4893 
 4894 // Narrow NULL Pointer Immediate
 4895 operand immN0()
 4896 %{
 4897   predicate(n->get_narrowcon() == 0);
 4898   match(ConN);
 4899 
 4900   op_cost(0);
 4901   format %{ %}
 4902   interface(CONST_INTER);
 4903 %}
 4904 
 4905 operand immNKlass()
 4906 %{
 4907   match(ConNKlass);
 4908 
 4909   op_cost(0);
 4910   format %{ %}
 4911   interface(CONST_INTER);
 4912 %}
 4913 
 4914 // Integer 32 bit Register Operands
 4915 // Integer 32 bitRegister (excludes SP)
 4916 operand iRegI()
 4917 %{
 4918   constraint(ALLOC_IN_RC(any_reg32));
 4919   match(RegI);
 4920   match(iRegINoSp);
 4921   op_cost(0);
 4922   format %{ %}
 4923   interface(REG_INTER);
 4924 %}
 4925 
 4926 // Integer 32 bit Register not Special
 4927 operand iRegINoSp()
 4928 %{
 4929   constraint(ALLOC_IN_RC(no_special_reg32));
 4930   match(RegI);
 4931   op_cost(0);
 4932   format %{ %}
 4933   interface(REG_INTER);
 4934 %}
 4935 
 4936 // Integer 64 bit Register Operands
 4937 // Integer 64 bit Register (includes SP)
 4938 operand iRegL()
 4939 %{
 4940   constraint(ALLOC_IN_RC(any_reg));
 4941   match(RegL);
 4942   match(iRegLNoSp);
 4943   op_cost(0);
 4944   format %{ %}
 4945   interface(REG_INTER);
 4946 %}
 4947 
 4948 // Integer 64 bit Register not Special
 4949 operand iRegLNoSp()
 4950 %{
 4951   constraint(ALLOC_IN_RC(no_special_reg));
 4952   match(RegL);
 4953   match(iRegL_R0);
 4954   format %{ %}
 4955   interface(REG_INTER);
 4956 %}
 4957 
 4958 // Pointer Register Operands
 4959 // Pointer Register
 4960 operand iRegP()
 4961 %{
 4962   constraint(ALLOC_IN_RC(ptr_reg));
 4963   match(RegP);
 4964   match(iRegPNoSp);
 4965   match(iRegP_R0);
 4966   //match(iRegP_R2);
 4967   //match(iRegP_R4);
 4968   match(iRegP_R5);
 4969   match(thread_RegP);
 4970   op_cost(0);
 4971   format %{ %}
 4972   interface(REG_INTER);
 4973 %}
 4974 
 4975 // Pointer 64 bit Register not Special
 4976 operand iRegPNoSp()
 4977 %{
 4978   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4979   match(RegP);
 4980   // match(iRegP);
 4981   // match(iRegP_R0);
 4982   // match(iRegP_R2);
 4983   // match(iRegP_R4);
 4984   // match(iRegP_R5);
 4985   // match(thread_RegP);
 4986   op_cost(0);
 4987   format %{ %}
 4988   interface(REG_INTER);
 4989 %}
 4990 
 4991 // Pointer 64 bit Register R0 only
 4992 operand iRegP_R0()
 4993 %{
 4994   constraint(ALLOC_IN_RC(r0_reg));
 4995   match(RegP);
 4996   // match(iRegP);
 4997   match(iRegPNoSp);
 4998   op_cost(0);
 4999   format %{ %}
 5000   interface(REG_INTER);
 5001 %}
 5002 
 5003 // Pointer 64 bit Register R1 only
 5004 operand iRegP_R1()
 5005 %{
 5006   constraint(ALLOC_IN_RC(r1_reg));
 5007   match(RegP);
 5008   // match(iRegP);
 5009   match(iRegPNoSp);
 5010   op_cost(0);
 5011   format %{ %}
 5012   interface(REG_INTER);
 5013 %}
 5014 
 5015 // Pointer 64 bit Register R2 only
 5016 operand iRegP_R2()
 5017 %{
 5018   constraint(ALLOC_IN_RC(r2_reg));
 5019   match(RegP);
 5020   // match(iRegP);
 5021   match(iRegPNoSp);
 5022   op_cost(0);
 5023   format %{ %}
 5024   interface(REG_INTER);
 5025 %}
 5026 
 5027 // Pointer 64 bit Register R3 only
 5028 operand iRegP_R3()
 5029 %{
 5030   constraint(ALLOC_IN_RC(r3_reg));
 5031   match(RegP);
 5032   // match(iRegP);
 5033   match(iRegPNoSp);
 5034   op_cost(0);
 5035   format %{ %}
 5036   interface(REG_INTER);
 5037 %}
 5038 
 5039 // Pointer 64 bit Register R4 only
 5040 operand iRegP_R4()
 5041 %{
 5042   constraint(ALLOC_IN_RC(r4_reg));
 5043   match(RegP);
 5044   // match(iRegP);
 5045   match(iRegPNoSp);
 5046   op_cost(0);
 5047   format %{ %}
 5048   interface(REG_INTER);
 5049 %}
 5050 
 5051 // Pointer 64 bit Register R5 only
 5052 operand iRegP_R5()
 5053 %{
 5054   constraint(ALLOC_IN_RC(r5_reg));
 5055   match(RegP);
 5056   // match(iRegP);
 5057   match(iRegPNoSp);
 5058   op_cost(0);
 5059   format %{ %}
 5060   interface(REG_INTER);
 5061 %}
 5062 
 5063 // Pointer 64 bit Register R10 only
 5064 operand iRegP_R10()
 5065 %{
 5066   constraint(ALLOC_IN_RC(r10_reg));
 5067   match(RegP);
 5068   // match(iRegP);
 5069   match(iRegPNoSp);
 5070   op_cost(0);
 5071   format %{ %}
 5072   interface(REG_INTER);
 5073 %}
 5074 
 5075 // Long 64 bit Register R0 only
 5076 operand iRegL_R0()
 5077 %{
 5078   constraint(ALLOC_IN_RC(r0_reg));
 5079   match(RegL);
 5080   match(iRegLNoSp);
 5081   op_cost(0);
 5082   format %{ %}
 5083   interface(REG_INTER);
 5084 %}
 5085 
 5086 // Long 64 bit Register R2 only
 5087 operand iRegL_R2()
 5088 %{
 5089   constraint(ALLOC_IN_RC(r2_reg));
 5090   match(RegL);
 5091   match(iRegLNoSp);
 5092   op_cost(0);
 5093   format %{ %}
 5094   interface(REG_INTER);
 5095 %}
 5096 
 5097 // Long 64 bit Register R3 only
 5098 operand iRegL_R3()
 5099 %{
 5100   constraint(ALLOC_IN_RC(r3_reg));
 5101   match(RegL);
 5102   match(iRegLNoSp);
 5103   op_cost(0);
 5104   format %{ %}
 5105   interface(REG_INTER);
 5106 %}
 5107 
 5108 // Long 64 bit Register R11 only
 5109 operand iRegL_R11()
 5110 %{
 5111   constraint(ALLOC_IN_RC(r11_reg));
 5112   match(RegL);
 5113   match(iRegLNoSp);
 5114   op_cost(0);
 5115   format %{ %}
 5116   interface(REG_INTER);
 5117 %}
 5118 
 5119 // Pointer 64 bit Register FP only
 5120 operand iRegP_FP()
 5121 %{
 5122   constraint(ALLOC_IN_RC(fp_reg));
 5123   match(RegP);
 5124   // match(iRegP);
 5125   op_cost(0);
 5126   format %{ %}
 5127   interface(REG_INTER);
 5128 %}
 5129 
 5130 // Register R0 only
 5131 operand iRegI_R0()
 5132 %{
 5133   constraint(ALLOC_IN_RC(int_r0_reg));
 5134   match(RegI);
 5135   match(iRegINoSp);
 5136   op_cost(0);
 5137   format %{ %}
 5138   interface(REG_INTER);
 5139 %}
 5140 
 5141 // Register R2 only
 5142 operand iRegI_R2()
 5143 %{
 5144   constraint(ALLOC_IN_RC(int_r2_reg));
 5145   match(RegI);
 5146   match(iRegINoSp);
 5147   op_cost(0);
 5148   format %{ %}
 5149   interface(REG_INTER);
 5150 %}
 5151 
 5152 // Register R3 only
 5153 operand iRegI_R3()
 5154 %{
 5155   constraint(ALLOC_IN_RC(int_r3_reg));
 5156   match(RegI);
 5157   match(iRegINoSp);
 5158   op_cost(0);
 5159   format %{ %}
 5160   interface(REG_INTER);
 5161 %}
 5162 
 5163 
 5164 // Register R4 only
 5165 operand iRegI_R4()
 5166 %{
 5167   constraint(ALLOC_IN_RC(int_r4_reg));
 5168   match(RegI);
 5169   match(iRegINoSp);
 5170   op_cost(0);
 5171   format %{ %}
 5172   interface(REG_INTER);
 5173 %}
 5174 
 5175 
 5176 // Pointer Register Operands
 5177 // Narrow Pointer Register
 5178 operand iRegN()
 5179 %{
 5180   constraint(ALLOC_IN_RC(any_reg32));
 5181   match(RegN);
 5182   match(iRegNNoSp);
 5183   op_cost(0);
 5184   format %{ %}
 5185   interface(REG_INTER);
 5186 %}
 5187 
 5188 operand iRegN_R0()
 5189 %{
 5190   constraint(ALLOC_IN_RC(r0_reg));
 5191   match(iRegN);
 5192   op_cost(0);
 5193   format %{ %}
 5194   interface(REG_INTER);
 5195 %}
 5196 
 5197 operand iRegN_R2()
 5198 %{
 5199   constraint(ALLOC_IN_RC(r2_reg));
 5200   match(iRegN);
 5201   op_cost(0);
 5202   format %{ %}
 5203   interface(REG_INTER);
 5204 %}
 5205 
 5206 operand iRegN_R3()
 5207 %{
 5208   constraint(ALLOC_IN_RC(r3_reg));
 5209   match(iRegN);
 5210   op_cost(0);
 5211   format %{ %}
 5212   interface(REG_INTER);
 5213 %}
 5214 
 5215 // Integer 64 bit Register not Special
 5216 operand iRegNNoSp()
 5217 %{
 5218   constraint(ALLOC_IN_RC(no_special_reg32));
 5219   match(RegN);
 5220   op_cost(0);
 5221   format %{ %}
 5222   interface(REG_INTER);
 5223 %}
 5224 
 5225 // heap base register -- used for encoding immN0
 5226 
 5227 operand iRegIHeapbase()
 5228 %{
 5229   constraint(ALLOC_IN_RC(heapbase_reg));
 5230   match(RegI);
 5231   op_cost(0);
 5232   format %{ %}
 5233   interface(REG_INTER);
 5234 %}
 5235 
 5236 // Float Register
 5237 // Float register operands
 5238 operand vRegF()
 5239 %{
 5240   constraint(ALLOC_IN_RC(float_reg));
 5241   match(RegF);
 5242 
 5243   op_cost(0);
 5244   format %{ %}
 5245   interface(REG_INTER);
 5246 %}
 5247 
 5248 // Double Register
 5249 // Double register operands
 5250 operand vRegD()
 5251 %{
 5252   constraint(ALLOC_IN_RC(double_reg));
 5253   match(RegD);
 5254 
 5255   op_cost(0);
 5256   format %{ %}
 5257   interface(REG_INTER);
 5258 %}
 5259 
 5260 // Generic vector class. This will be used for
 5261 // all vector operands, including NEON and SVE.
 5262 operand vReg()
 5263 %{
 5264   constraint(ALLOC_IN_RC(dynamic));
 5265   match(VecA);
 5266   match(VecD);
 5267   match(VecX);
 5268 
 5269   op_cost(0);
 5270   format %{ %}
 5271   interface(REG_INTER);
 5272 %}
 5273 
 5274 operand vecA()
 5275 %{
 5276   constraint(ALLOC_IN_RC(vectora_reg));
 5277   match(VecA);
 5278 
 5279   op_cost(0);
 5280   format %{ %}
 5281   interface(REG_INTER);
 5282 %}
 5283 
 5284 operand vecD()
 5285 %{
 5286   constraint(ALLOC_IN_RC(vectord_reg));
 5287   match(VecD);
 5288 
 5289   op_cost(0);
 5290   format %{ %}
 5291   interface(REG_INTER);
 5292 %}
 5293 
 5294 operand vecX()
 5295 %{
 5296   constraint(ALLOC_IN_RC(vectorx_reg));
 5297   match(VecX);
 5298 
 5299   op_cost(0);
 5300   format %{ %}
 5301   interface(REG_INTER);
 5302 %}
 5303 
 5304 operand vRegD_V0()
 5305 %{
 5306   constraint(ALLOC_IN_RC(v0_reg));
 5307   match(RegD);
 5308   op_cost(0);
 5309   format %{ %}
 5310   interface(REG_INTER);
 5311 %}
 5312 
 5313 operand vRegD_V1()
 5314 %{
 5315   constraint(ALLOC_IN_RC(v1_reg));
 5316   match(RegD);
 5317   op_cost(0);
 5318   format %{ %}
 5319   interface(REG_INTER);
 5320 %}
 5321 
 5322 operand vRegD_V2()
 5323 %{
 5324   constraint(ALLOC_IN_RC(v2_reg));
 5325   match(RegD);
 5326   op_cost(0);
 5327   format %{ %}
 5328   interface(REG_INTER);
 5329 %}
 5330 
 5331 operand vRegD_V3()
 5332 %{
 5333   constraint(ALLOC_IN_RC(v3_reg));
 5334   match(RegD);
 5335   op_cost(0);
 5336   format %{ %}
 5337   interface(REG_INTER);
 5338 %}
 5339 
 5340 operand vRegD_V4()
 5341 %{
 5342   constraint(ALLOC_IN_RC(v4_reg));
 5343   match(RegD);
 5344   op_cost(0);
 5345   format %{ %}
 5346   interface(REG_INTER);
 5347 %}
 5348 
 5349 operand vRegD_V5()
 5350 %{
 5351   constraint(ALLOC_IN_RC(v5_reg));
 5352   match(RegD);
 5353   op_cost(0);
 5354   format %{ %}
 5355   interface(REG_INTER);
 5356 %}
 5357 
 5358 operand vRegD_V6()
 5359 %{
 5360   constraint(ALLOC_IN_RC(v6_reg));
 5361   match(RegD);
 5362   op_cost(0);
 5363   format %{ %}
 5364   interface(REG_INTER);
 5365 %}
 5366 
 5367 operand vRegD_V7()
 5368 %{
 5369   constraint(ALLOC_IN_RC(v7_reg));
 5370   match(RegD);
 5371   op_cost(0);
 5372   format %{ %}
 5373   interface(REG_INTER);
 5374 %}
 5375 
 5376 operand vRegD_V8()
 5377 %{
 5378   constraint(ALLOC_IN_RC(v8_reg));
 5379   match(RegD);
 5380   op_cost(0);
 5381   format %{ %}
 5382   interface(REG_INTER);
 5383 %}
 5384 
 5385 operand vRegD_V9()
 5386 %{
 5387   constraint(ALLOC_IN_RC(v9_reg));
 5388   match(RegD);
 5389   op_cost(0);
 5390   format %{ %}
 5391   interface(REG_INTER);
 5392 %}
 5393 
 5394 operand vRegD_V10()
 5395 %{
 5396   constraint(ALLOC_IN_RC(v10_reg));
 5397   match(RegD);
 5398   op_cost(0);
 5399   format %{ %}
 5400   interface(REG_INTER);
 5401 %}
 5402 
 5403 operand vRegD_V11()
 5404 %{
 5405   constraint(ALLOC_IN_RC(v11_reg));
 5406   match(RegD);
 5407   op_cost(0);
 5408   format %{ %}
 5409   interface(REG_INTER);
 5410 %}
 5411 
 5412 operand vRegD_V12()
 5413 %{
 5414   constraint(ALLOC_IN_RC(v12_reg));
 5415   match(RegD);
 5416   op_cost(0);
 5417   format %{ %}
 5418   interface(REG_INTER);
 5419 %}
 5420 
 5421 operand vRegD_V13()
 5422 %{
 5423   constraint(ALLOC_IN_RC(v13_reg));
 5424   match(RegD);
 5425   op_cost(0);
 5426   format %{ %}
 5427   interface(REG_INTER);
 5428 %}
 5429 
 5430 operand vRegD_V14()
 5431 %{
 5432   constraint(ALLOC_IN_RC(v14_reg));
 5433   match(RegD);
 5434   op_cost(0);
 5435   format %{ %}
 5436   interface(REG_INTER);
 5437 %}
 5438 
 5439 operand vRegD_V15()
 5440 %{
 5441   constraint(ALLOC_IN_RC(v15_reg));
 5442   match(RegD);
 5443   op_cost(0);
 5444   format %{ %}
 5445   interface(REG_INTER);
 5446 %}
 5447 
 5448 operand vRegD_V16()
 5449 %{
 5450   constraint(ALLOC_IN_RC(v16_reg));
 5451   match(RegD);
 5452   op_cost(0);
 5453   format %{ %}
 5454   interface(REG_INTER);
 5455 %}
 5456 
 5457 operand vRegD_V17()
 5458 %{
 5459   constraint(ALLOC_IN_RC(v17_reg));
 5460   match(RegD);
 5461   op_cost(0);
 5462   format %{ %}
 5463   interface(REG_INTER);
 5464 %}
 5465 
 5466 operand vRegD_V18()
 5467 %{
 5468   constraint(ALLOC_IN_RC(v18_reg));
 5469   match(RegD);
 5470   op_cost(0);
 5471   format %{ %}
 5472   interface(REG_INTER);
 5473 %}
 5474 
 5475 operand vRegD_V19()
 5476 %{
 5477   constraint(ALLOC_IN_RC(v19_reg));
 5478   match(RegD);
 5479   op_cost(0);
 5480   format %{ %}
 5481   interface(REG_INTER);
 5482 %}
 5483 
 5484 operand vRegD_V20()
 5485 %{
 5486   constraint(ALLOC_IN_RC(v20_reg));
 5487   match(RegD);
 5488   op_cost(0);
 5489   format %{ %}
 5490   interface(REG_INTER);
 5491 %}
 5492 
 5493 operand vRegD_V21()
 5494 %{
 5495   constraint(ALLOC_IN_RC(v21_reg));
 5496   match(RegD);
 5497   op_cost(0);
 5498   format %{ %}
 5499   interface(REG_INTER);
 5500 %}
 5501 
 5502 operand vRegD_V22()
 5503 %{
 5504   constraint(ALLOC_IN_RC(v22_reg));
 5505   match(RegD);
 5506   op_cost(0);
 5507   format %{ %}
 5508   interface(REG_INTER);
 5509 %}
 5510 
 5511 operand vRegD_V23()
 5512 %{
 5513   constraint(ALLOC_IN_RC(v23_reg));
 5514   match(RegD);
 5515   op_cost(0);
 5516   format %{ %}
 5517   interface(REG_INTER);
 5518 %}
 5519 
 5520 operand vRegD_V24()
 5521 %{
 5522   constraint(ALLOC_IN_RC(v24_reg));
 5523   match(RegD);
 5524   op_cost(0);
 5525   format %{ %}
 5526   interface(REG_INTER);
 5527 %}
 5528 
 5529 operand vRegD_V25()
 5530 %{
 5531   constraint(ALLOC_IN_RC(v25_reg));
 5532   match(RegD);
 5533   op_cost(0);
 5534   format %{ %}
 5535   interface(REG_INTER);
 5536 %}
 5537 
 5538 operand vRegD_V26()
 5539 %{
 5540   constraint(ALLOC_IN_RC(v26_reg));
 5541   match(RegD);
 5542   op_cost(0);
 5543   format %{ %}
 5544   interface(REG_INTER);
 5545 %}
 5546 
 5547 operand vRegD_V27()
 5548 %{
 5549   constraint(ALLOC_IN_RC(v27_reg));
 5550   match(RegD);
 5551   op_cost(0);
 5552   format %{ %}
 5553   interface(REG_INTER);
 5554 %}
 5555 
 5556 operand vRegD_V28()
 5557 %{
 5558   constraint(ALLOC_IN_RC(v28_reg));
 5559   match(RegD);
 5560   op_cost(0);
 5561   format %{ %}
 5562   interface(REG_INTER);
 5563 %}
 5564 
 5565 operand vRegD_V29()
 5566 %{
 5567   constraint(ALLOC_IN_RC(v29_reg));
 5568   match(RegD);
 5569   op_cost(0);
 5570   format %{ %}
 5571   interface(REG_INTER);
 5572 %}
 5573 
 5574 operand vRegD_V30()
 5575 %{
 5576   constraint(ALLOC_IN_RC(v30_reg));
 5577   match(RegD);
 5578   op_cost(0);
 5579   format %{ %}
 5580   interface(REG_INTER);
 5581 %}
 5582 
 5583 operand vRegD_V31()
 5584 %{
 5585   constraint(ALLOC_IN_RC(v31_reg));
 5586   match(RegD);
 5587   op_cost(0);
 5588   format %{ %}
 5589   interface(REG_INTER);
 5590 %}
 5591 
 5592 operand pReg()
 5593 %{
 5594   constraint(ALLOC_IN_RC(pr_reg));
 5595   match(RegVectMask);
 5596   match(pRegGov);
 5597   op_cost(0);
 5598   format %{ %}
 5599   interface(REG_INTER);
 5600 %}
 5601 
 5602 operand pRegGov()
 5603 %{
 5604   constraint(ALLOC_IN_RC(gov_pr));
 5605   match(RegVectMask);
 5606   match(pReg);
 5607   op_cost(0);
 5608   format %{ %}
 5609   interface(REG_INTER);
 5610 %}
 5611 
 5612 operand pRegGov_P0()
 5613 %{
 5614   constraint(ALLOC_IN_RC(p0_reg));
 5615   match(RegVectMask);
 5616   op_cost(0);
 5617   format %{ %}
 5618   interface(REG_INTER);
 5619 %}
 5620 
 5621 operand pRegGov_P1()
 5622 %{
 5623   constraint(ALLOC_IN_RC(p1_reg));
 5624   match(RegVectMask);
 5625   op_cost(0);
 5626   format %{ %}
 5627   interface(REG_INTER);
 5628 %}
 5629 
 5630 // Flags register, used as output of signed compare instructions
 5631 
 5632 // note that on AArch64 we also use this register as the output for
 5633 // for floating point compare instructions (CmpF CmpD). this ensures
 5634 // that ordered inequality tests use GT, GE, LT or LE none of which
 5635 // pass through cases where the result is unordered i.e. one or both
 5636 // inputs to the compare is a NaN. this means that the ideal code can
 5637 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5638 // (where the comparison should always fail). EQ and NE tests are
 5639 // always generated in ideal code so that unordered folds into the NE
 5640 // case, matching the behaviour of AArch64 NE.
 5641 //
 5642 // This differs from x86 where the outputs of FP compares use a
 5643 // special FP flags registers and where compares based on this
 5644 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5645 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5646 // to explicitly handle the unordered case in branches. x86 also has
 5647 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5648 
 5649 operand rFlagsReg()
 5650 %{
 5651   constraint(ALLOC_IN_RC(int_flags));
 5652   match(RegFlags);
 5653 
 5654   op_cost(0);
 5655   format %{ "RFLAGS" %}
 5656   interface(REG_INTER);
 5657 %}
 5658 
 5659 // Flags register, used as output of unsigned compare instructions
 5660 operand rFlagsRegU()
 5661 %{
 5662   constraint(ALLOC_IN_RC(int_flags));
 5663   match(RegFlags);
 5664 
 5665   op_cost(0);
 5666   format %{ "RFLAGSU" %}
 5667   interface(REG_INTER);
 5668 %}
 5669 
 5670 // Special Registers
 5671 
 5672 // Method Register
 5673 operand inline_cache_RegP(iRegP reg)
 5674 %{
 5675   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5676   match(reg);
 5677   match(iRegPNoSp);
 5678   op_cost(0);
 5679   format %{ %}
 5680   interface(REG_INTER);
 5681 %}
 5682 
 5683 // Thread Register
 5684 operand thread_RegP(iRegP reg)
 5685 %{
 5686   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5687   match(reg);
 5688   op_cost(0);
 5689   format %{ %}
 5690   interface(REG_INTER);
 5691 %}
 5692 
 5693 operand lr_RegP(iRegP reg)
 5694 %{
 5695   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5696   match(reg);
 5697   op_cost(0);
 5698   format %{ %}
 5699   interface(REG_INTER);
 5700 %}
 5701 
 5702 //----------Memory Operands----------------------------------------------------
 5703 
 5704 operand indirect(iRegP reg)
 5705 %{
 5706   constraint(ALLOC_IN_RC(ptr_reg));
 5707   match(reg);
 5708   op_cost(0);
 5709   format %{ "[$reg]" %}
 5710   interface(MEMORY_INTER) %{
 5711     base($reg);
 5712     index(0xffffffff);
 5713     scale(0x0);
 5714     disp(0x0);
 5715   %}
 5716 %}
 5717 
 5718 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5719 %{
 5720   constraint(ALLOC_IN_RC(ptr_reg));
 5721   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5722   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5723   op_cost(0);
 5724   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5725   interface(MEMORY_INTER) %{
 5726     base($reg);
 5727     index($ireg);
 5728     scale($scale);
 5729     disp(0x0);
 5730   %}
 5731 %}
 5732 
 5733 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5734 %{
 5735   constraint(ALLOC_IN_RC(ptr_reg));
 5736   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5737   match(AddP reg (LShiftL lreg scale));
 5738   op_cost(0);
 5739   format %{ "$reg, $lreg lsl($scale)" %}
 5740   interface(MEMORY_INTER) %{
 5741     base($reg);
 5742     index($lreg);
 5743     scale($scale);
 5744     disp(0x0);
 5745   %}
 5746 %}
 5747 
 5748 operand indIndexI2L(iRegP reg, iRegI ireg)
 5749 %{
 5750   constraint(ALLOC_IN_RC(ptr_reg));
 5751   match(AddP reg (ConvI2L ireg));
 5752   op_cost(0);
 5753   format %{ "$reg, $ireg, 0, I2L" %}
 5754   interface(MEMORY_INTER) %{
 5755     base($reg);
 5756     index($ireg);
 5757     scale(0x0);
 5758     disp(0x0);
 5759   %}
 5760 %}
 5761 
 5762 operand indIndex(iRegP reg, iRegL lreg)
 5763 %{
 5764   constraint(ALLOC_IN_RC(ptr_reg));
 5765   match(AddP reg lreg);
 5766   op_cost(0);
 5767   format %{ "$reg, $lreg" %}
 5768   interface(MEMORY_INTER) %{
 5769     base($reg);
 5770     index($lreg);
 5771     scale(0x0);
 5772     disp(0x0);
 5773   %}
 5774 %}
 5775 
 5776 operand indOffI(iRegP reg, immIOffset off)
 5777 %{
 5778   constraint(ALLOC_IN_RC(ptr_reg));
 5779   match(AddP reg off);
 5780   op_cost(0);
 5781   format %{ "[$reg, $off]" %}
 5782   interface(MEMORY_INTER) %{
 5783     base($reg);
 5784     index(0xffffffff);
 5785     scale(0x0);
 5786     disp($off);
 5787   %}
 5788 %}
 5789 
 5790 operand indOffI1(iRegP reg, immIOffset1 off)
 5791 %{
 5792   constraint(ALLOC_IN_RC(ptr_reg));
 5793   match(AddP reg off);
 5794   op_cost(0);
 5795   format %{ "[$reg, $off]" %}
 5796   interface(MEMORY_INTER) %{
 5797     base($reg);
 5798     index(0xffffffff);
 5799     scale(0x0);
 5800     disp($off);
 5801   %}
 5802 %}
 5803 
 5804 operand indOffI2(iRegP reg, immIOffset2 off)
 5805 %{
 5806   constraint(ALLOC_IN_RC(ptr_reg));
 5807   match(AddP reg off);
 5808   op_cost(0);
 5809   format %{ "[$reg, $off]" %}
 5810   interface(MEMORY_INTER) %{
 5811     base($reg);
 5812     index(0xffffffff);
 5813     scale(0x0);
 5814     disp($off);
 5815   %}
 5816 %}
 5817 
 5818 operand indOffI4(iRegP reg, immIOffset4 off)
 5819 %{
 5820   constraint(ALLOC_IN_RC(ptr_reg));
 5821   match(AddP reg off);
 5822   op_cost(0);
 5823   format %{ "[$reg, $off]" %}
 5824   interface(MEMORY_INTER) %{
 5825     base($reg);
 5826     index(0xffffffff);
 5827     scale(0x0);
 5828     disp($off);
 5829   %}
 5830 %}
 5831 
 5832 operand indOffI8(iRegP reg, immIOffset8 off)
 5833 %{
 5834   constraint(ALLOC_IN_RC(ptr_reg));
 5835   match(AddP reg off);
 5836   op_cost(0);
 5837   format %{ "[$reg, $off]" %}
 5838   interface(MEMORY_INTER) %{
 5839     base($reg);
 5840     index(0xffffffff);
 5841     scale(0x0);
 5842     disp($off);
 5843   %}
 5844 %}
 5845 
 5846 operand indOffI16(iRegP reg, immIOffset16 off)
 5847 %{
 5848   constraint(ALLOC_IN_RC(ptr_reg));
 5849   match(AddP reg off);
 5850   op_cost(0);
 5851   format %{ "[$reg, $off]" %}
 5852   interface(MEMORY_INTER) %{
 5853     base($reg);
 5854     index(0xffffffff);
 5855     scale(0x0);
 5856     disp($off);
 5857   %}
 5858 %}
 5859 
 5860 operand indOffL(iRegP reg, immLoffset off)
 5861 %{
 5862   constraint(ALLOC_IN_RC(ptr_reg));
 5863   match(AddP reg off);
 5864   op_cost(0);
 5865   format %{ "[$reg, $off]" %}
 5866   interface(MEMORY_INTER) %{
 5867     base($reg);
 5868     index(0xffffffff);
 5869     scale(0x0);
 5870     disp($off);
 5871   %}
 5872 %}
 5873 
 5874 operand indOffL1(iRegP reg, immLoffset1 off)
 5875 %{
 5876   constraint(ALLOC_IN_RC(ptr_reg));
 5877   match(AddP reg off);
 5878   op_cost(0);
 5879   format %{ "[$reg, $off]" %}
 5880   interface(MEMORY_INTER) %{
 5881     base($reg);
 5882     index(0xffffffff);
 5883     scale(0x0);
 5884     disp($off);
 5885   %}
 5886 %}
 5887 
 5888 operand indOffL2(iRegP reg, immLoffset2 off)
 5889 %{
 5890   constraint(ALLOC_IN_RC(ptr_reg));
 5891   match(AddP reg off);
 5892   op_cost(0);
 5893   format %{ "[$reg, $off]" %}
 5894   interface(MEMORY_INTER) %{
 5895     base($reg);
 5896     index(0xffffffff);
 5897     scale(0x0);
 5898     disp($off);
 5899   %}
 5900 %}
 5901 
 5902 operand indOffL4(iRegP reg, immLoffset4 off)
 5903 %{
 5904   constraint(ALLOC_IN_RC(ptr_reg));
 5905   match(AddP reg off);
 5906   op_cost(0);
 5907   format %{ "[$reg, $off]" %}
 5908   interface(MEMORY_INTER) %{
 5909     base($reg);
 5910     index(0xffffffff);
 5911     scale(0x0);
 5912     disp($off);
 5913   %}
 5914 %}
 5915 
 5916 operand indOffL8(iRegP reg, immLoffset8 off)
 5917 %{
 5918   constraint(ALLOC_IN_RC(ptr_reg));
 5919   match(AddP reg off);
 5920   op_cost(0);
 5921   format %{ "[$reg, $off]" %}
 5922   interface(MEMORY_INTER) %{
 5923     base($reg);
 5924     index(0xffffffff);
 5925     scale(0x0);
 5926     disp($off);
 5927   %}
 5928 %}
 5929 
 5930 operand indOffL16(iRegP reg, immLoffset16 off)
 5931 %{
 5932   constraint(ALLOC_IN_RC(ptr_reg));
 5933   match(AddP reg off);
 5934   op_cost(0);
 5935   format %{ "[$reg, $off]" %}
 5936   interface(MEMORY_INTER) %{
 5937     base($reg);
 5938     index(0xffffffff);
 5939     scale(0x0);
 5940     disp($off);
 5941   %}
 5942 %}
 5943 
 5944 operand indirectN(iRegN reg)
 5945 %{
 5946   predicate(CompressedOops::shift() == 0);
 5947   constraint(ALLOC_IN_RC(ptr_reg));
 5948   match(DecodeN reg);
 5949   op_cost(0);
 5950   format %{ "[$reg]\t# narrow" %}
 5951   interface(MEMORY_INTER) %{
 5952     base($reg);
 5953     index(0xffffffff);
 5954     scale(0x0);
 5955     disp(0x0);
 5956   %}
 5957 %}
 5958 
 5959 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5960 %{
 5961   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5962   constraint(ALLOC_IN_RC(ptr_reg));
 5963   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5964   op_cost(0);
 5965   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5966   interface(MEMORY_INTER) %{
 5967     base($reg);
 5968     index($ireg);
 5969     scale($scale);
 5970     disp(0x0);
 5971   %}
 5972 %}
 5973 
 5974 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5975 %{
 5976   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5977   constraint(ALLOC_IN_RC(ptr_reg));
 5978   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5979   op_cost(0);
 5980   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5981   interface(MEMORY_INTER) %{
 5982     base($reg);
 5983     index($lreg);
 5984     scale($scale);
 5985     disp(0x0);
 5986   %}
 5987 %}
 5988 
 5989 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5990 %{
 5991   predicate(CompressedOops::shift() == 0);
 5992   constraint(ALLOC_IN_RC(ptr_reg));
 5993   match(AddP (DecodeN reg) (ConvI2L ireg));
 5994   op_cost(0);
 5995   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5996   interface(MEMORY_INTER) %{
 5997     base($reg);
 5998     index($ireg);
 5999     scale(0x0);
 6000     disp(0x0);
 6001   %}
 6002 %}
 6003 
 6004 operand indIndexN(iRegN reg, iRegL lreg)
 6005 %{
 6006   predicate(CompressedOops::shift() == 0);
 6007   constraint(ALLOC_IN_RC(ptr_reg));
 6008   match(AddP (DecodeN reg) lreg);
 6009   op_cost(0);
 6010   format %{ "$reg, $lreg\t# narrow" %}
 6011   interface(MEMORY_INTER) %{
 6012     base($reg);
 6013     index($lreg);
 6014     scale(0x0);
 6015     disp(0x0);
 6016   %}
 6017 %}
 6018 
 6019 operand indOffIN(iRegN reg, immIOffset off)
 6020 %{
 6021   predicate(CompressedOops::shift() == 0);
 6022   constraint(ALLOC_IN_RC(ptr_reg));
 6023   match(AddP (DecodeN reg) off);
 6024   op_cost(0);
 6025   format %{ "[$reg, $off]\t# narrow" %}
 6026   interface(MEMORY_INTER) %{
 6027     base($reg);
 6028     index(0xffffffff);
 6029     scale(0x0);
 6030     disp($off);
 6031   %}
 6032 %}
 6033 
 6034 operand indOffLN(iRegN reg, immLoffset off)
 6035 %{
 6036   predicate(CompressedOops::shift() == 0);
 6037   constraint(ALLOC_IN_RC(ptr_reg));
 6038   match(AddP (DecodeN reg) off);
 6039   op_cost(0);
 6040   format %{ "[$reg, $off]\t# narrow" %}
 6041   interface(MEMORY_INTER) %{
 6042     base($reg);
 6043     index(0xffffffff);
 6044     scale(0x0);
 6045     disp($off);
 6046   %}
 6047 %}
 6048 
 6049 
 6050 
 6051 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 6052 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 6053 %{
 6054   constraint(ALLOC_IN_RC(ptr_reg));
 6055   match(AddP reg off);
 6056   op_cost(0);
 6057   format %{ "[$reg, $off]" %}
 6058   interface(MEMORY_INTER) %{
 6059     base($reg);
 6060     index(0xffffffff);
 6061     scale(0x0);
 6062     disp($off);
 6063   %}
 6064 %}
 6065 
 6066 //----------Special Memory Operands--------------------------------------------
 6067 // Stack Slot Operand - This operand is used for loading and storing temporary
 6068 //                      values on the stack where a match requires a value to
 6069 //                      flow through memory.
 6070 operand stackSlotP(sRegP reg)
 6071 %{
 6072   constraint(ALLOC_IN_RC(stack_slots));
 6073   op_cost(100);
 6074   // No match rule because this operand is only generated in matching
 6075   // match(RegP);
 6076   format %{ "[$reg]" %}
 6077   interface(MEMORY_INTER) %{
 6078     base(0x1e);  // RSP
 6079     index(0x0);  // No Index
 6080     scale(0x0);  // No Scale
 6081     disp($reg);  // Stack Offset
 6082   %}
 6083 %}
 6084 
 6085 operand stackSlotI(sRegI reg)
 6086 %{
 6087   constraint(ALLOC_IN_RC(stack_slots));
 6088   // No match rule because this operand is only generated in matching
 6089   // match(RegI);
 6090   format %{ "[$reg]" %}
 6091   interface(MEMORY_INTER) %{
 6092     base(0x1e);  // RSP
 6093     index(0x0);  // No Index
 6094     scale(0x0);  // No Scale
 6095     disp($reg);  // Stack Offset
 6096   %}
 6097 %}
 6098 
 6099 operand stackSlotF(sRegF reg)
 6100 %{
 6101   constraint(ALLOC_IN_RC(stack_slots));
 6102   // No match rule because this operand is only generated in matching
 6103   // match(RegF);
 6104   format %{ "[$reg]" %}
 6105   interface(MEMORY_INTER) %{
 6106     base(0x1e);  // RSP
 6107     index(0x0);  // No Index
 6108     scale(0x0);  // No Scale
 6109     disp($reg);  // Stack Offset
 6110   %}
 6111 %}
 6112 
 6113 operand stackSlotD(sRegD reg)
 6114 %{
 6115   constraint(ALLOC_IN_RC(stack_slots));
 6116   // No match rule because this operand is only generated in matching
 6117   // match(RegD);
 6118   format %{ "[$reg]" %}
 6119   interface(MEMORY_INTER) %{
 6120     base(0x1e);  // RSP
 6121     index(0x0);  // No Index
 6122     scale(0x0);  // No Scale
 6123     disp($reg);  // Stack Offset
 6124   %}
 6125 %}
 6126 
 6127 operand stackSlotL(sRegL reg)
 6128 %{
 6129   constraint(ALLOC_IN_RC(stack_slots));
 6130   // No match rule because this operand is only generated in matching
 6131   // match(RegL);
 6132   format %{ "[$reg]" %}
 6133   interface(MEMORY_INTER) %{
 6134     base(0x1e);  // RSP
 6135     index(0x0);  // No Index
 6136     scale(0x0);  // No Scale
 6137     disp($reg);  // Stack Offset
 6138   %}
 6139 %}
 6140 
 6141 // Operands for expressing Control Flow
 6142 // NOTE: Label is a predefined operand which should not be redefined in
 6143 //       the AD file. It is generically handled within the ADLC.
 6144 
 6145 //----------Conditional Branch Operands----------------------------------------
 6146 // Comparison Op  - This is the operation of the comparison, and is limited to
 6147 //                  the following set of codes:
 6148 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6149 //
 6150 // Other attributes of the comparison, such as unsignedness, are specified
 6151 // by the comparison instruction that sets a condition code flags register.
 6152 // That result is represented by a flags operand whose subtype is appropriate
 6153 // to the unsignedness (etc.) of the comparison.
 6154 //
 6155 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6156 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6157 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6158 
 6159 // used for signed integral comparisons and fp comparisons
 6160 
 6161 operand cmpOp()
 6162 %{
 6163   match(Bool);
 6164 
 6165   format %{ "" %}
 6166   interface(COND_INTER) %{
 6167     equal(0x0, "eq");
 6168     not_equal(0x1, "ne");
 6169     less(0xb, "lt");
 6170     greater_equal(0xa, "ge");
 6171     less_equal(0xd, "le");
 6172     greater(0xc, "gt");
 6173     overflow(0x6, "vs");
 6174     no_overflow(0x7, "vc");
 6175   %}
 6176 %}
 6177 
 6178 // used for unsigned integral comparisons
 6179 
 6180 operand cmpOpU()
 6181 %{
 6182   match(Bool);
 6183 
 6184   format %{ "" %}
 6185   interface(COND_INTER) %{
 6186     equal(0x0, "eq");
 6187     not_equal(0x1, "ne");
 6188     less(0x3, "lo");
 6189     greater_equal(0x2, "hs");
 6190     less_equal(0x9, "ls");
 6191     greater(0x8, "hi");
 6192     overflow(0x6, "vs");
 6193     no_overflow(0x7, "vc");
 6194   %}
 6195 %}
 6196 
 6197 // used for certain integral comparisons which can be
 6198 // converted to cbxx or tbxx instructions
 6199 
 6200 operand cmpOpEqNe()
 6201 %{
 6202   match(Bool);
 6203   op_cost(0);
 6204   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6205             || n->as_Bool()->_test._test == BoolTest::eq);
 6206 
 6207   format %{ "" %}
 6208   interface(COND_INTER) %{
 6209     equal(0x0, "eq");
 6210     not_equal(0x1, "ne");
 6211     less(0xb, "lt");
 6212     greater_equal(0xa, "ge");
 6213     less_equal(0xd, "le");
 6214     greater(0xc, "gt");
 6215     overflow(0x6, "vs");
 6216     no_overflow(0x7, "vc");
 6217   %}
 6218 %}
 6219 
 6220 // used for certain integral comparisons which can be
 6221 // converted to cbxx or tbxx instructions
 6222 
 6223 operand cmpOpLtGe()
 6224 %{
 6225   match(Bool);
 6226   op_cost(0);
 6227 
 6228   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6229             || n->as_Bool()->_test._test == BoolTest::ge);
 6230 
 6231   format %{ "" %}
 6232   interface(COND_INTER) %{
 6233     equal(0x0, "eq");
 6234     not_equal(0x1, "ne");
 6235     less(0xb, "lt");
 6236     greater_equal(0xa, "ge");
 6237     less_equal(0xd, "le");
 6238     greater(0xc, "gt");
 6239     overflow(0x6, "vs");
 6240     no_overflow(0x7, "vc");
 6241   %}
 6242 %}
 6243 
 6244 // used for certain unsigned integral comparisons which can be
 6245 // converted to cbxx or tbxx instructions
 6246 
 6247 operand cmpOpUEqNeLtGe()
 6248 %{
 6249   match(Bool);
 6250   op_cost(0);
 6251 
 6252   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6253             || n->as_Bool()->_test._test == BoolTest::ne
 6254             || n->as_Bool()->_test._test == BoolTest::lt
 6255             || n->as_Bool()->_test._test == BoolTest::ge);
 6256 
 6257   format %{ "" %}
 6258   interface(COND_INTER) %{
 6259     equal(0x0, "eq");
 6260     not_equal(0x1, "ne");
 6261     less(0xb, "lt");
 6262     greater_equal(0xa, "ge");
 6263     less_equal(0xd, "le");
 6264     greater(0xc, "gt");
 6265     overflow(0x6, "vs");
 6266     no_overflow(0x7, "vc");
 6267   %}
 6268 %}
 6269 
 6270 // Special operand allowing long args to int ops to be truncated for free
 6271 
 6272 operand iRegL2I(iRegL reg) %{
 6273 
 6274   op_cost(0);
 6275 
 6276   match(ConvL2I reg);
 6277 
 6278   format %{ "l2i($reg)" %}
 6279 
 6280   interface(REG_INTER)
 6281 %}
 6282 
 6283 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6284 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6285 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6286 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6287 
 6288 //----------OPERAND CLASSES----------------------------------------------------
 6289 // Operand Classes are groups of operands that are used as to simplify
 6290 // instruction definitions by not requiring the AD writer to specify
 6291 // separate instructions for every form of operand when the
 6292 // instruction accepts multiple operand types with the same basic
 6293 // encoding and format. The classic case of this is memory operands.
 6294 
 6295 // memory is used to define read/write location for load/store
 6296 // instruction defs. we can turn a memory op into an Address
 6297 
 6298 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6299                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6300 
 6301 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6302                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6303 
 6304 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6305                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6306 
 6307 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6308                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6309 
 6310 // All of the memory operands. For the pipeline description.
 6311 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6312                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6313                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6314 
 6315 
 6316 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6317 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6318 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6319 // can be elided because the 32-bit instruction will just employ the
 6320 // lower 32 bits anyway.
 6321 //
 6322 // n.b. this does not elide all L2I conversions. if the truncated
 6323 // value is consumed by more than one operation then the ConvL2I
 6324 // cannot be bundled into the consuming nodes so an l2i gets planted
 6325 // (actually a movw $dst $src) and the downstream instructions consume
 6326 // the result of the l2i as an iRegI input. That's a shame since the
 6327 // movw is actually redundant but its not too costly.
 6328 
 6329 opclass iRegIorL2I(iRegI, iRegL2I);
 6330 
 6331 //----------PIPELINE-----------------------------------------------------------
 6332 // Rules which define the behavior of the target architectures pipeline.
 6333 
 6334 // For specific pipelines, eg A53, define the stages of that pipeline
 6335 //pipe_desc(ISS, EX1, EX2, WR);
 6336 #define ISS S0
 6337 #define EX1 S1
 6338 #define EX2 S2
 6339 #define WR  S3
 6340 
 6341 // Integer ALU reg operation
 6342 pipeline %{
 6343 
 6344 attributes %{
 6345   // ARM instructions are of fixed length
 6346   fixed_size_instructions;        // Fixed size instructions TODO does
 6347   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6348   // ARM instructions come in 32-bit word units
 6349   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6350   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6351   instruction_fetch_units = 1;       // of 64 bytes
 6352 
 6353   // List of nop instructions
 6354   nops( MachNop );
 6355 %}
 6356 
 6357 // We don't use an actual pipeline model so don't care about resources
 6358 // or description. we do use pipeline classes to introduce fixed
 6359 // latencies
 6360 
 6361 //----------RESOURCES----------------------------------------------------------
 6362 // Resources are the functional units available to the machine
 6363 
 6364 resources( INS0, INS1, INS01 = INS0 | INS1,
 6365            ALU0, ALU1, ALU = ALU0 | ALU1,
 6366            MAC,
 6367            DIV,
 6368            BRANCH,
 6369            LDST,
 6370            NEON_FP);
 6371 
 6372 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6373 // Pipeline Description specifies the stages in the machine's pipeline
 6374 
 6375 // Define the pipeline as a generic 6 stage pipeline
 6376 pipe_desc(S0, S1, S2, S3, S4, S5);
 6377 
 6378 //----------PIPELINE CLASSES---------------------------------------------------
 6379 // Pipeline Classes describe the stages in which input and output are
 6380 // referenced by the hardware pipeline.
 6381 
 6382 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6383 %{
 6384   single_instruction;
 6385   src1   : S1(read);
 6386   src2   : S2(read);
 6387   dst    : S5(write);
 6388   INS01  : ISS;
 6389   NEON_FP : S5;
 6390 %}
 6391 
 6392 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6393 %{
 6394   single_instruction;
 6395   src1   : S1(read);
 6396   src2   : S2(read);
 6397   dst    : S5(write);
 6398   INS01  : ISS;
 6399   NEON_FP : S5;
 6400 %}
 6401 
 6402 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6403 %{
 6404   single_instruction;
 6405   src    : S1(read);
 6406   dst    : S5(write);
 6407   INS01  : ISS;
 6408   NEON_FP : S5;
 6409 %}
 6410 
 6411 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6412 %{
 6413   single_instruction;
 6414   src    : S1(read);
 6415   dst    : S5(write);
 6416   INS01  : ISS;
 6417   NEON_FP : S5;
 6418 %}
 6419 
 6420 pipe_class fp_d2f(vRegF dst, vRegD src)
 6421 %{
 6422   single_instruction;
 6423   src    : S1(read);
 6424   dst    : S5(write);
 6425   INS01  : ISS;
 6426   NEON_FP : S5;
 6427 %}
 6428 
 6429 pipe_class fp_f2d(vRegD dst, vRegF src)
 6430 %{
 6431   single_instruction;
 6432   src    : S1(read);
 6433   dst    : S5(write);
 6434   INS01  : ISS;
 6435   NEON_FP : S5;
 6436 %}
 6437 
 6438 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6439 %{
 6440   single_instruction;
 6441   src    : S1(read);
 6442   dst    : S5(write);
 6443   INS01  : ISS;
 6444   NEON_FP : S5;
 6445 %}
 6446 
 6447 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6448 %{
 6449   single_instruction;
 6450   src    : S1(read);
 6451   dst    : S5(write);
 6452   INS01  : ISS;
 6453   NEON_FP : S5;
 6454 %}
 6455 
 6456 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6457 %{
 6458   single_instruction;
 6459   src    : S1(read);
 6460   dst    : S5(write);
 6461   INS01  : ISS;
 6462   NEON_FP : S5;
 6463 %}
 6464 
 6465 pipe_class fp_l2f(vRegF dst, iRegL src)
 6466 %{
 6467   single_instruction;
 6468   src    : S1(read);
 6469   dst    : S5(write);
 6470   INS01  : ISS;
 6471   NEON_FP : S5;
 6472 %}
 6473 
 6474 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6475 %{
 6476   single_instruction;
 6477   src    : S1(read);
 6478   dst    : S5(write);
 6479   INS01  : ISS;
 6480   NEON_FP : S5;
 6481 %}
 6482 
 6483 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6484 %{
 6485   single_instruction;
 6486   src    : S1(read);
 6487   dst    : S5(write);
 6488   INS01  : ISS;
 6489   NEON_FP : S5;
 6490 %}
 6491 
 6492 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6493 %{
 6494   single_instruction;
 6495   src    : S1(read);
 6496   dst    : S5(write);
 6497   INS01  : ISS;
 6498   NEON_FP : S5;
 6499 %}
 6500 
 6501 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6502 %{
 6503   single_instruction;
 6504   src    : S1(read);
 6505   dst    : S5(write);
 6506   INS01  : ISS;
 6507   NEON_FP : S5;
 6508 %}
 6509 
 6510 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6511 %{
 6512   single_instruction;
 6513   src1   : S1(read);
 6514   src2   : S2(read);
 6515   dst    : S5(write);
 6516   INS0   : ISS;
 6517   NEON_FP : S5;
 6518 %}
 6519 
 6520 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6521 %{
 6522   single_instruction;
 6523   src1   : S1(read);
 6524   src2   : S2(read);
 6525   dst    : S5(write);
 6526   INS0   : ISS;
 6527   NEON_FP : S5;
 6528 %}
 6529 
 6530 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6531 %{
 6532   single_instruction;
 6533   cr     : S1(read);
 6534   src1   : S1(read);
 6535   src2   : S1(read);
 6536   dst    : S3(write);
 6537   INS01  : ISS;
 6538   NEON_FP : S3;
 6539 %}
 6540 
 6541 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6542 %{
 6543   single_instruction;
 6544   cr     : S1(read);
 6545   src1   : S1(read);
 6546   src2   : S1(read);
 6547   dst    : S3(write);
 6548   INS01  : ISS;
 6549   NEON_FP : S3;
 6550 %}
 6551 
 6552 pipe_class fp_imm_s(vRegF dst)
 6553 %{
 6554   single_instruction;
 6555   dst    : S3(write);
 6556   INS01  : ISS;
 6557   NEON_FP : S3;
 6558 %}
 6559 
 6560 pipe_class fp_imm_d(vRegD dst)
 6561 %{
 6562   single_instruction;
 6563   dst    : S3(write);
 6564   INS01  : ISS;
 6565   NEON_FP : S3;
 6566 %}
 6567 
 6568 pipe_class fp_load_constant_s(vRegF dst)
 6569 %{
 6570   single_instruction;
 6571   dst    : S4(write);
 6572   INS01  : ISS;
 6573   NEON_FP : S4;
 6574 %}
 6575 
 6576 pipe_class fp_load_constant_d(vRegD dst)
 6577 %{
 6578   single_instruction;
 6579   dst    : S4(write);
 6580   INS01  : ISS;
 6581   NEON_FP : S4;
 6582 %}
 6583 
 6584 //------- Integer ALU operations --------------------------
 6585 
 6586 // Integer ALU reg-reg operation
 6587 // Operands needed in EX1, result generated in EX2
 6588 // Eg.  ADD     x0, x1, x2
 6589 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6590 %{
 6591   single_instruction;
 6592   dst    : EX2(write);
 6593   src1   : EX1(read);
 6594   src2   : EX1(read);
 6595   INS01  : ISS; // Dual issue as instruction 0 or 1
 6596   ALU    : EX2;
 6597 %}
 6598 
 6599 // Integer ALU reg-reg operation with constant shift
 6600 // Shifted register must be available in LATE_ISS instead of EX1
 6601 // Eg.  ADD     x0, x1, x2, LSL #2
 6602 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6603 %{
 6604   single_instruction;
 6605   dst    : EX2(write);
 6606   src1   : EX1(read);
 6607   src2   : ISS(read);
 6608   INS01  : ISS;
 6609   ALU    : EX2;
 6610 %}
 6611 
 6612 // Integer ALU reg operation with constant shift
 6613 // Eg.  LSL     x0, x1, #shift
 6614 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6615 %{
 6616   single_instruction;
 6617   dst    : EX2(write);
 6618   src1   : ISS(read);
 6619   INS01  : ISS;
 6620   ALU    : EX2;
 6621 %}
 6622 
 6623 // Integer ALU reg-reg operation with variable shift
 6624 // Both operands must be available in LATE_ISS instead of EX1
 6625 // Result is available in EX1 instead of EX2
 6626 // Eg.  LSLV    x0, x1, x2
 6627 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6628 %{
 6629   single_instruction;
 6630   dst    : EX1(write);
 6631   src1   : ISS(read);
 6632   src2   : ISS(read);
 6633   INS01  : ISS;
 6634   ALU    : EX1;
 6635 %}
 6636 
 6637 // Integer ALU reg-reg operation with extract
 6638 // As for _vshift above, but result generated in EX2
 6639 // Eg.  EXTR    x0, x1, x2, #N
 6640 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6641 %{
 6642   single_instruction;
 6643   dst    : EX2(write);
 6644   src1   : ISS(read);
 6645   src2   : ISS(read);
 6646   INS1   : ISS; // Can only dual issue as Instruction 1
 6647   ALU    : EX1;
 6648 %}
 6649 
 6650 // Integer ALU reg operation
 6651 // Eg.  NEG     x0, x1
 6652 pipe_class ialu_reg(iRegI dst, iRegI src)
 6653 %{
 6654   single_instruction;
 6655   dst    : EX2(write);
 6656   src    : EX1(read);
 6657   INS01  : ISS;
 6658   ALU    : EX2;
 6659 %}
 6660 
 6661 // Integer ALU reg mmediate operation
 6662 // Eg.  ADD     x0, x1, #N
 6663 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6664 %{
 6665   single_instruction;
 6666   dst    : EX2(write);
 6667   src1   : EX1(read);
 6668   INS01  : ISS;
 6669   ALU    : EX2;
 6670 %}
 6671 
 6672 // Integer ALU immediate operation (no source operands)
 6673 // Eg.  MOV     x0, #N
 6674 pipe_class ialu_imm(iRegI dst)
 6675 %{
 6676   single_instruction;
 6677   dst    : EX1(write);
 6678   INS01  : ISS;
 6679   ALU    : EX1;
 6680 %}
 6681 
 6682 //------- Compare operation -------------------------------
 6683 
 6684 // Compare reg-reg
 6685 // Eg.  CMP     x0, x1
 6686 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6687 %{
 6688   single_instruction;
 6689 //  fixed_latency(16);
 6690   cr     : EX2(write);
 6691   op1    : EX1(read);
 6692   op2    : EX1(read);
 6693   INS01  : ISS;
 6694   ALU    : EX2;
 6695 %}
 6696 
 6697 // Compare reg-reg
 6698 // Eg.  CMP     x0, #N
 6699 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6700 %{
 6701   single_instruction;
 6702 //  fixed_latency(16);
 6703   cr     : EX2(write);
 6704   op1    : EX1(read);
 6705   INS01  : ISS;
 6706   ALU    : EX2;
 6707 %}
 6708 
 6709 //------- Conditional instructions ------------------------
 6710 
 6711 // Conditional no operands
 6712 // Eg.  CSINC   x0, zr, zr, <cond>
 6713 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6714 %{
 6715   single_instruction;
 6716   cr     : EX1(read);
 6717   dst    : EX2(write);
 6718   INS01  : ISS;
 6719   ALU    : EX2;
 6720 %}
 6721 
 6722 // Conditional 2 operand
 6723 // EG.  CSEL    X0, X1, X2, <cond>
 6724 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6725 %{
 6726   single_instruction;
 6727   cr     : EX1(read);
 6728   src1   : EX1(read);
 6729   src2   : EX1(read);
 6730   dst    : EX2(write);
 6731   INS01  : ISS;
 6732   ALU    : EX2;
 6733 %}
 6734 
 6735 // Conditional 2 operand
 6736 // EG.  CSEL    X0, X1, X2, <cond>
 6737 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6738 %{
 6739   single_instruction;
 6740   cr     : EX1(read);
 6741   src    : EX1(read);
 6742   dst    : EX2(write);
 6743   INS01  : ISS;
 6744   ALU    : EX2;
 6745 %}
 6746 
 6747 //------- Multiply pipeline operations --------------------
 6748 
 6749 // Multiply reg-reg
 6750 // Eg.  MUL     w0, w1, w2
 6751 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6752 %{
 6753   single_instruction;
 6754   dst    : WR(write);
 6755   src1   : ISS(read);
 6756   src2   : ISS(read);
 6757   INS01  : ISS;
 6758   MAC    : WR;
 6759 %}
 6760 
 6761 // Multiply accumulate
 6762 // Eg.  MADD    w0, w1, w2, w3
 6763 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6764 %{
 6765   single_instruction;
 6766   dst    : WR(write);
 6767   src1   : ISS(read);
 6768   src2   : ISS(read);
 6769   src3   : ISS(read);
 6770   INS01  : ISS;
 6771   MAC    : WR;
 6772 %}
 6773 
 6774 // Eg.  MUL     w0, w1, w2
 6775 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6776 %{
 6777   single_instruction;
 6778   fixed_latency(3); // Maximum latency for 64 bit mul
 6779   dst    : WR(write);
 6780   src1   : ISS(read);
 6781   src2   : ISS(read);
 6782   INS01  : ISS;
 6783   MAC    : WR;
 6784 %}
 6785 
 6786 // Multiply accumulate
 6787 // Eg.  MADD    w0, w1, w2, w3
 6788 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6789 %{
 6790   single_instruction;
 6791   fixed_latency(3); // Maximum latency for 64 bit mul
 6792   dst    : WR(write);
 6793   src1   : ISS(read);
 6794   src2   : ISS(read);
 6795   src3   : ISS(read);
 6796   INS01  : ISS;
 6797   MAC    : WR;
 6798 %}
 6799 
 6800 //------- Divide pipeline operations --------------------
 6801 
 6802 // Eg.  SDIV    w0, w1, w2
 6803 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6804 %{
 6805   single_instruction;
 6806   fixed_latency(8); // Maximum latency for 32 bit divide
 6807   dst    : WR(write);
 6808   src1   : ISS(read);
 6809   src2   : ISS(read);
 6810   INS0   : ISS; // Can only dual issue as instruction 0
 6811   DIV    : WR;
 6812 %}
 6813 
 6814 // Eg.  SDIV    x0, x1, x2
 6815 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6816 %{
 6817   single_instruction;
 6818   fixed_latency(16); // Maximum latency for 64 bit divide
 6819   dst    : WR(write);
 6820   src1   : ISS(read);
 6821   src2   : ISS(read);
 6822   INS0   : ISS; // Can only dual issue as instruction 0
 6823   DIV    : WR;
 6824 %}
 6825 
 6826 //------- Load pipeline operations ------------------------
 6827 
 6828 // Load - prefetch
 6829 // Eg.  PFRM    <mem>
 6830 pipe_class iload_prefetch(memory mem)
 6831 %{
 6832   single_instruction;
 6833   mem    : ISS(read);
 6834   INS01  : ISS;
 6835   LDST   : WR;
 6836 %}
 6837 
 6838 // Load - reg, mem
 6839 // Eg.  LDR     x0, <mem>
 6840 pipe_class iload_reg_mem(iRegI dst, memory mem)
 6841 %{
 6842   single_instruction;
 6843   dst    : WR(write);
 6844   mem    : ISS(read);
 6845   INS01  : ISS;
 6846   LDST   : WR;
 6847 %}
 6848 
 6849 // Load - reg, reg
 6850 // Eg.  LDR     x0, [sp, x1]
 6851 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 6852 %{
 6853   single_instruction;
 6854   dst    : WR(write);
 6855   src    : ISS(read);
 6856   INS01  : ISS;
 6857   LDST   : WR;
 6858 %}
 6859 
 6860 //------- Store pipeline operations -----------------------
 6861 
 6862 // Store - zr, mem
 6863 // Eg.  STR     zr, <mem>
 6864 pipe_class istore_mem(memory mem)
 6865 %{
 6866   single_instruction;
 6867   mem    : ISS(read);
 6868   INS01  : ISS;
 6869   LDST   : WR;
 6870 %}
 6871 
 6872 // Store - reg, mem
 6873 // Eg.  STR     x0, <mem>
 6874 pipe_class istore_reg_mem(iRegI src, memory mem)
 6875 %{
 6876   single_instruction;
 6877   mem    : ISS(read);
 6878   src    : EX2(read);
 6879   INS01  : ISS;
 6880   LDST   : WR;
 6881 %}
 6882 
 6883 // Store - reg, reg
 6884 // Eg. STR      x0, [sp, x1]
 6885 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 6886 %{
 6887   single_instruction;
 6888   dst    : ISS(read);
 6889   src    : EX2(read);
 6890   INS01  : ISS;
 6891   LDST   : WR;
 6892 %}
 6893 
 6894 //------- Store pipeline operations -----------------------
 6895 
 6896 // Branch
 6897 pipe_class pipe_branch()
 6898 %{
 6899   single_instruction;
 6900   INS01  : ISS;
 6901   BRANCH : EX1;
 6902 %}
 6903 
 6904 // Conditional branch
 6905 pipe_class pipe_branch_cond(rFlagsReg cr)
 6906 %{
 6907   single_instruction;
 6908   cr     : EX1(read);
 6909   INS01  : ISS;
 6910   BRANCH : EX1;
 6911 %}
 6912 
 6913 // Compare & Branch
 6914 // EG.  CBZ/CBNZ
 6915 pipe_class pipe_cmp_branch(iRegI op1)
 6916 %{
 6917   single_instruction;
 6918   op1    : EX1(read);
 6919   INS01  : ISS;
 6920   BRANCH : EX1;
 6921 %}
 6922 
 6923 //------- Synchronisation operations ----------------------
 6924 
 6925 // Any operation requiring serialization.
 6926 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 6927 pipe_class pipe_serial()
 6928 %{
 6929   single_instruction;
 6930   force_serialization;
 6931   fixed_latency(16);
 6932   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6933   LDST   : WR;
 6934 %}
 6935 
 6936 // Generic big/slow expanded idiom - also serialized
 6937 pipe_class pipe_slow()
 6938 %{
 6939   instruction_count(10);
 6940   multiple_bundles;
 6941   force_serialization;
 6942   fixed_latency(16);
 6943   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6944   LDST   : WR;
 6945 %}
 6946 
 6947 // Empty pipeline class
 6948 pipe_class pipe_class_empty()
 6949 %{
 6950   single_instruction;
 6951   fixed_latency(0);
 6952 %}
 6953 
 6954 // Default pipeline class.
 6955 pipe_class pipe_class_default()
 6956 %{
 6957   single_instruction;
 6958   fixed_latency(2);
 6959 %}
 6960 
 6961 // Pipeline class for compares.
 6962 pipe_class pipe_class_compare()
 6963 %{
 6964   single_instruction;
 6965   fixed_latency(16);
 6966 %}
 6967 
 6968 // Pipeline class for memory operations.
 6969 pipe_class pipe_class_memory()
 6970 %{
 6971   single_instruction;
 6972   fixed_latency(16);
 6973 %}
 6974 
 6975 // Pipeline class for call.
 6976 pipe_class pipe_class_call()
 6977 %{
 6978   single_instruction;
 6979   fixed_latency(100);
 6980 %}
 6981 
 6982 // Define the class for the Nop node.
 6983 define %{
 6984    MachNop = pipe_class_empty;
 6985 %}
 6986 
 6987 %}
 6988 //----------INSTRUCTIONS-------------------------------------------------------
 6989 //
 6990 // match      -- States which machine-independent subtree may be replaced
 6991 //               by this instruction.
 6992 // ins_cost   -- The estimated cost of this instruction is used by instruction
 6993 //               selection to identify a minimum cost tree of machine
 6994 //               instructions that matches a tree of machine-independent
 6995 //               instructions.
 6996 // format     -- A string providing the disassembly for this instruction.
 6997 //               The value of an instruction's operand may be inserted
 6998 //               by referring to it with a '$' prefix.
 6999 // opcode     -- Three instruction opcodes may be provided.  These are referred
 7000 //               to within an encode class as $primary, $secondary, and $tertiary
 7001 //               rrspectively.  The primary opcode is commonly used to
 7002 //               indicate the type of machine instruction, while secondary
 7003 //               and tertiary are often used for prefix options or addressing
 7004 //               modes.
 7005 // ins_encode -- A list of encode classes with parameters. The encode class
 7006 //               name must have been defined in an 'enc_class' specification
 7007 //               in the encode section of the architecture description.
 7008 
 7009 // ============================================================================
 7010 // Memory (Load/Store) Instructions
 7011 
 7012 // Load Instructions
 7013 
 7014 // Load Byte (8 bit signed)
 7015 instruct loadB(iRegINoSp dst, memory1 mem)
 7016 %{
 7017   match(Set dst (LoadB mem));
 7018   predicate(!needs_acquiring_load(n));
 7019 
 7020   ins_cost(4 * INSN_COST);
 7021   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 7022 
 7023   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 7024 
 7025   ins_pipe(iload_reg_mem);
 7026 %}
 7027 
 7028 // Load Byte (8 bit signed) into long
 7029 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 7030 %{
 7031   match(Set dst (ConvI2L (LoadB mem)));
 7032   predicate(!needs_acquiring_load(n->in(1)));
 7033 
 7034   ins_cost(4 * INSN_COST);
 7035   format %{ "ldrsb  $dst, $mem\t# byte" %}
 7036 
 7037   ins_encode(aarch64_enc_ldrsb(dst, mem));
 7038 
 7039   ins_pipe(iload_reg_mem);
 7040 %}
 7041 
 7042 // Load Byte (8 bit unsigned)
 7043 instruct loadUB(iRegINoSp dst, memory1 mem)
 7044 %{
 7045   match(Set dst (LoadUB mem));
 7046   predicate(!needs_acquiring_load(n));
 7047 
 7048   ins_cost(4 * INSN_COST);
 7049   format %{ "ldrbw  $dst, $mem\t# byte" %}
 7050 
 7051   ins_encode(aarch64_enc_ldrb(dst, mem));
 7052 
 7053   ins_pipe(iload_reg_mem);
 7054 %}
 7055 
 7056 // Load Byte (8 bit unsigned) into long
 7057 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7058 %{
 7059   match(Set dst (ConvI2L (LoadUB mem)));
 7060   predicate(!needs_acquiring_load(n->in(1)));
 7061 
 7062   ins_cost(4 * INSN_COST);
 7063   format %{ "ldrb  $dst, $mem\t# byte" %}
 7064 
 7065   ins_encode(aarch64_enc_ldrb(dst, mem));
 7066 
 7067   ins_pipe(iload_reg_mem);
 7068 %}
 7069 
 7070 // Load Short (16 bit signed)
 7071 instruct loadS(iRegINoSp dst, memory2 mem)
 7072 %{
 7073   match(Set dst (LoadS mem));
 7074   predicate(!needs_acquiring_load(n));
 7075 
 7076   ins_cost(4 * INSN_COST);
 7077   format %{ "ldrshw  $dst, $mem\t# short" %}
 7078 
 7079   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7080 
 7081   ins_pipe(iload_reg_mem);
 7082 %}
 7083 
 7084 // Load Short (16 bit signed) into long
 7085 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7086 %{
 7087   match(Set dst (ConvI2L (LoadS mem)));
 7088   predicate(!needs_acquiring_load(n->in(1)));
 7089 
 7090   ins_cost(4 * INSN_COST);
 7091   format %{ "ldrsh  $dst, $mem\t# short" %}
 7092 
 7093   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7094 
 7095   ins_pipe(iload_reg_mem);
 7096 %}
 7097 
 7098 // Load Char (16 bit unsigned)
 7099 instruct loadUS(iRegINoSp dst, memory2 mem)
 7100 %{
 7101   match(Set dst (LoadUS mem));
 7102   predicate(!needs_acquiring_load(n));
 7103 
 7104   ins_cost(4 * INSN_COST);
 7105   format %{ "ldrh  $dst, $mem\t# short" %}
 7106 
 7107   ins_encode(aarch64_enc_ldrh(dst, mem));
 7108 
 7109   ins_pipe(iload_reg_mem);
 7110 %}
 7111 
 7112 // Load Short/Char (16 bit unsigned) into long
 7113 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7114 %{
 7115   match(Set dst (ConvI2L (LoadUS mem)));
 7116   predicate(!needs_acquiring_load(n->in(1)));
 7117 
 7118   ins_cost(4 * INSN_COST);
 7119   format %{ "ldrh  $dst, $mem\t# short" %}
 7120 
 7121   ins_encode(aarch64_enc_ldrh(dst, mem));
 7122 
 7123   ins_pipe(iload_reg_mem);
 7124 %}
 7125 
 7126 // Load Integer (32 bit signed)
 7127 instruct loadI(iRegINoSp dst, memory4 mem)
 7128 %{
 7129   match(Set dst (LoadI mem));
 7130   predicate(!needs_acquiring_load(n));
 7131 
 7132   ins_cost(4 * INSN_COST);
 7133   format %{ "ldrw  $dst, $mem\t# int" %}
 7134 
 7135   ins_encode(aarch64_enc_ldrw(dst, mem));
 7136 
 7137   ins_pipe(iload_reg_mem);
 7138 %}
 7139 
 7140 // Load Integer (32 bit signed) into long
 7141 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7142 %{
 7143   match(Set dst (ConvI2L (LoadI mem)));
 7144   predicate(!needs_acquiring_load(n->in(1)));
 7145 
 7146   ins_cost(4 * INSN_COST);
 7147   format %{ "ldrsw  $dst, $mem\t# int" %}
 7148 
 7149   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7150 
 7151   ins_pipe(iload_reg_mem);
 7152 %}
 7153 
 7154 // Load Integer (32 bit unsigned) into long
 7155 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7156 %{
 7157   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7158   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7159 
 7160   ins_cost(4 * INSN_COST);
 7161   format %{ "ldrw  $dst, $mem\t# int" %}
 7162 
 7163   ins_encode(aarch64_enc_ldrw(dst, mem));
 7164 
 7165   ins_pipe(iload_reg_mem);
 7166 %}
 7167 
 7168 // Load Long (64 bit signed)
 7169 instruct loadL(iRegLNoSp dst, memory8 mem)
 7170 %{
 7171   match(Set dst (LoadL mem));
 7172   predicate(!needs_acquiring_load(n));
 7173 
 7174   ins_cost(4 * INSN_COST);
 7175   format %{ "ldr  $dst, $mem\t# int" %}
 7176 
 7177   ins_encode(aarch64_enc_ldr(dst, mem));
 7178 
 7179   ins_pipe(iload_reg_mem);
 7180 %}
 7181 
 7182 // Load Range
 7183 instruct loadRange(iRegINoSp dst, memory4 mem)
 7184 %{
 7185   match(Set dst (LoadRange mem));
 7186 
 7187   ins_cost(4 * INSN_COST);
 7188   format %{ "ldrw  $dst, $mem\t# range" %}
 7189 
 7190   ins_encode(aarch64_enc_ldrw(dst, mem));
 7191 
 7192   ins_pipe(iload_reg_mem);
 7193 %}
 7194 
 7195 // Load Pointer
 7196 instruct loadP(iRegPNoSp dst, memory8 mem)
 7197 %{
 7198   match(Set dst (LoadP mem));
 7199   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7200 
 7201   ins_cost(4 * INSN_COST);
 7202   format %{ "ldr  $dst, $mem\t# ptr" %}
 7203 
 7204   ins_encode(aarch64_enc_ldr(dst, mem));
 7205 
 7206   ins_pipe(iload_reg_mem);
 7207 %}
 7208 
 7209 // Load Compressed Pointer
 7210 instruct loadN(iRegNNoSp dst, memory4 mem)
 7211 %{
 7212   match(Set dst (LoadN mem));
 7213   predicate(!needs_acquiring_load(n));
 7214 
 7215   ins_cost(4 * INSN_COST);
 7216   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7217 
 7218   ins_encode(aarch64_enc_ldrw(dst, mem));
 7219 
 7220   ins_pipe(iload_reg_mem);
 7221 %}
 7222 
 7223 // Load Klass Pointer
 7224 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7225 %{
 7226   match(Set dst (LoadKlass mem));
 7227   predicate(!needs_acquiring_load(n));
 7228 
 7229   ins_cost(4 * INSN_COST);
 7230   format %{ "ldr  $dst, $mem\t# class" %}
 7231 
 7232   ins_encode(aarch64_enc_ldr(dst, mem));
 7233 
 7234   ins_pipe(iload_reg_mem);
 7235 %}
 7236 
 7237 // Load Narrow Klass Pointer
 7238 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7239 %{
 7240   match(Set dst (LoadNKlass mem));
 7241   predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
 7242 
 7243   ins_cost(4 * INSN_COST);
 7244   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7245 
 7246   ins_encode(aarch64_enc_ldrw(dst, mem));
 7247 
 7248   ins_pipe(iload_reg_mem);
 7249 %}
 7250 
 7251 instruct loadNKlassLilliput(iRegNNoSp dst, memory4 mem, rFlagsReg cr)
 7252 %{
 7253   match(Set dst (LoadNKlass mem));
 7254   effect(TEMP_DEF dst, KILL cr);
 7255   predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
 7256 
 7257   ins_cost(4 * INSN_COST);
 7258   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7259   ins_encode %{
 7260     assert($mem$$disp == oopDesc::klass_offset_in_bytes(), "expect correct offset");
 7261     assert($mem$$index$$Register == noreg, "expect no index");
 7262     Register dst = $dst$$Register;
 7263     Register obj = $mem$$base$$Register;
 7264     C2LoadNKlassStub* stub = new (Compile::current()->comp_arena()) C2LoadNKlassStub(dst);
 7265     Compile::current()->output()->add_stub(stub);
 7266     __ ldr(dst, Address(obj, oopDesc::mark_offset_in_bytes()));
 7267     // NOTE: We can't use tbnz here, because the target is sometimes too far away
 7268     // and cannot be encoded.
 7269     __ tst(dst, markWord::monitor_value);
 7270     __ br(Assembler::NE, stub->entry());
 7271     __ bind(stub->continuation());
 7272     __ lsr(dst, dst, markWord::klass_shift);
 7273   %}
 7274   ins_pipe(pipe_slow);
 7275 %}
 7276 
 7277 // Load Float
 7278 instruct loadF(vRegF dst, memory4 mem)
 7279 %{
 7280   match(Set dst (LoadF mem));
 7281   predicate(!needs_acquiring_load(n));
 7282 
 7283   ins_cost(4 * INSN_COST);
 7284   format %{ "ldrs  $dst, $mem\t# float" %}
 7285 
 7286   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7287 
 7288   ins_pipe(pipe_class_memory);
 7289 %}
 7290 
 7291 // Load Double
 7292 instruct loadD(vRegD dst, memory8 mem)
 7293 %{
 7294   match(Set dst (LoadD mem));
 7295   predicate(!needs_acquiring_load(n));
 7296 
 7297   ins_cost(4 * INSN_COST);
 7298   format %{ "ldrd  $dst, $mem\t# double" %}
 7299 
 7300   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7301 
 7302   ins_pipe(pipe_class_memory);
 7303 %}
 7304 
 7305 
 7306 // Load Int Constant
 7307 instruct loadConI(iRegINoSp dst, immI src)
 7308 %{
 7309   match(Set dst src);
 7310 
 7311   ins_cost(INSN_COST);
 7312   format %{ "mov $dst, $src\t# int" %}
 7313 
 7314   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7315 
 7316   ins_pipe(ialu_imm);
 7317 %}
 7318 
 7319 // Load Long Constant
 7320 instruct loadConL(iRegLNoSp dst, immL src)
 7321 %{
 7322   match(Set dst src);
 7323 
 7324   ins_cost(INSN_COST);
 7325   format %{ "mov $dst, $src\t# long" %}
 7326 
 7327   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7328 
 7329   ins_pipe(ialu_imm);
 7330 %}
 7331 
 7332 // Load Pointer Constant
 7333 
 7334 instruct loadConP(iRegPNoSp dst, immP con)
 7335 %{
 7336   match(Set dst con);
 7337 
 7338   ins_cost(INSN_COST * 4);
 7339   format %{
 7340     "mov  $dst, $con\t# ptr\n\t"
 7341   %}
 7342 
 7343   ins_encode(aarch64_enc_mov_p(dst, con));
 7344 
 7345   ins_pipe(ialu_imm);
 7346 %}
 7347 
 7348 // Load Null Pointer Constant
 7349 
 7350 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7351 %{
 7352   match(Set dst con);
 7353 
 7354   ins_cost(INSN_COST);
 7355   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7356 
 7357   ins_encode(aarch64_enc_mov_p0(dst, con));
 7358 
 7359   ins_pipe(ialu_imm);
 7360 %}
 7361 
 7362 // Load Pointer Constant One
 7363 
 7364 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7365 %{
 7366   match(Set dst con);
 7367 
 7368   ins_cost(INSN_COST);
 7369   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7370 
 7371   ins_encode(aarch64_enc_mov_p1(dst, con));
 7372 
 7373   ins_pipe(ialu_imm);
 7374 %}
 7375 
 7376 // Load Byte Map Base Constant
 7377 
 7378 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7379 %{
 7380   match(Set dst con);
 7381 
 7382   ins_cost(INSN_COST);
 7383   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7384 
 7385   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7386 
 7387   ins_pipe(ialu_imm);
 7388 %}
 7389 
 7390 // Load Narrow Pointer Constant
 7391 
 7392 instruct loadConN(iRegNNoSp dst, immN con)
 7393 %{
 7394   match(Set dst con);
 7395 
 7396   ins_cost(INSN_COST * 4);
 7397   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7398 
 7399   ins_encode(aarch64_enc_mov_n(dst, con));
 7400 
 7401   ins_pipe(ialu_imm);
 7402 %}
 7403 
 7404 // Load Narrow Null Pointer Constant
 7405 
 7406 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7407 %{
 7408   match(Set dst con);
 7409 
 7410   ins_cost(INSN_COST);
 7411   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7412 
 7413   ins_encode(aarch64_enc_mov_n0(dst, con));
 7414 
 7415   ins_pipe(ialu_imm);
 7416 %}
 7417 
 7418 // Load Narrow Klass Constant
 7419 
 7420 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7421 %{
 7422   match(Set dst con);
 7423 
 7424   ins_cost(INSN_COST);
 7425   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7426 
 7427   ins_encode(aarch64_enc_mov_nk(dst, con));
 7428 
 7429   ins_pipe(ialu_imm);
 7430 %}
 7431 
 7432 // Load Packed Float Constant
 7433 
 7434 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7435   match(Set dst con);
 7436   ins_cost(INSN_COST * 4);
 7437   format %{ "fmovs  $dst, $con"%}
 7438   ins_encode %{
 7439     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7440   %}
 7441 
 7442   ins_pipe(fp_imm_s);
 7443 %}
 7444 
 7445 // Load Float Constant
 7446 
 7447 instruct loadConF(vRegF dst, immF con) %{
 7448   match(Set dst con);
 7449 
 7450   ins_cost(INSN_COST * 4);
 7451 
 7452   format %{
 7453     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7454   %}
 7455 
 7456   ins_encode %{
 7457     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7458   %}
 7459 
 7460   ins_pipe(fp_load_constant_s);
 7461 %}
 7462 
 7463 // Load Packed Double Constant
 7464 
 7465 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7466   match(Set dst con);
 7467   ins_cost(INSN_COST);
 7468   format %{ "fmovd  $dst, $con"%}
 7469   ins_encode %{
 7470     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7471   %}
 7472 
 7473   ins_pipe(fp_imm_d);
 7474 %}
 7475 
 7476 // Load Double Constant
 7477 
 7478 instruct loadConD(vRegD dst, immD con) %{
 7479   match(Set dst con);
 7480 
 7481   ins_cost(INSN_COST * 5);
 7482   format %{
 7483     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7484   %}
 7485 
 7486   ins_encode %{
 7487     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7488   %}
 7489 
 7490   ins_pipe(fp_load_constant_d);
 7491 %}
 7492 
 7493 // Store Instructions
 7494 
 7495 // Store CMS card-mark Immediate
 7496 instruct storeimmCM0(immI0 zero, memory1 mem)
 7497 %{
 7498   match(Set mem (StoreCM mem zero));
 7499 
 7500   ins_cost(INSN_COST);
 7501   format %{ "storestore (elided)\n\t"
 7502             "strb zr, $mem\t# byte" %}
 7503 
 7504   ins_encode(aarch64_enc_strb0(mem));
 7505 
 7506   ins_pipe(istore_mem);
 7507 %}
 7508 
 7509 // Store CMS card-mark Immediate with intervening StoreStore
 7510 // needed when using CMS with no conditional card marking
 7511 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7512 %{
 7513   match(Set mem (StoreCM mem zero));
 7514 
 7515   ins_cost(INSN_COST * 2);
 7516   format %{ "storestore\n\t"
 7517             "dmb ishst"
 7518             "\n\tstrb zr, $mem\t# byte" %}
 7519 
 7520   ins_encode(aarch64_enc_strb0_ordered(mem));
 7521 
 7522   ins_pipe(istore_mem);
 7523 %}
 7524 
 7525 // Store Byte
 7526 instruct storeB(iRegIorL2I src, memory1 mem)
 7527 %{
 7528   match(Set mem (StoreB mem src));
 7529   predicate(!needs_releasing_store(n));
 7530 
 7531   ins_cost(INSN_COST);
 7532   format %{ "strb  $src, $mem\t# byte" %}
 7533 
 7534   ins_encode(aarch64_enc_strb(src, mem));
 7535 
 7536   ins_pipe(istore_reg_mem);
 7537 %}
 7538 
 7539 
 7540 instruct storeimmB0(immI0 zero, memory1 mem)
 7541 %{
 7542   match(Set mem (StoreB mem zero));
 7543   predicate(!needs_releasing_store(n));
 7544 
 7545   ins_cost(INSN_COST);
 7546   format %{ "strb rscractch2, $mem\t# byte" %}
 7547 
 7548   ins_encode(aarch64_enc_strb0(mem));
 7549 
 7550   ins_pipe(istore_mem);
 7551 %}
 7552 
 7553 // Store Char/Short
 7554 instruct storeC(iRegIorL2I src, memory2 mem)
 7555 %{
 7556   match(Set mem (StoreC mem src));
 7557   predicate(!needs_releasing_store(n));
 7558 
 7559   ins_cost(INSN_COST);
 7560   format %{ "strh  $src, $mem\t# short" %}
 7561 
 7562   ins_encode(aarch64_enc_strh(src, mem));
 7563 
 7564   ins_pipe(istore_reg_mem);
 7565 %}
 7566 
 7567 instruct storeimmC0(immI0 zero, memory2 mem)
 7568 %{
 7569   match(Set mem (StoreC mem zero));
 7570   predicate(!needs_releasing_store(n));
 7571 
 7572   ins_cost(INSN_COST);
 7573   format %{ "strh  zr, $mem\t# short" %}
 7574 
 7575   ins_encode(aarch64_enc_strh0(mem));
 7576 
 7577   ins_pipe(istore_mem);
 7578 %}
 7579 
 7580 // Store Integer
 7581 
 7582 instruct storeI(iRegIorL2I src, memory4 mem)
 7583 %{
 7584   match(Set mem(StoreI mem src));
 7585   predicate(!needs_releasing_store(n));
 7586 
 7587   ins_cost(INSN_COST);
 7588   format %{ "strw  $src, $mem\t# int" %}
 7589 
 7590   ins_encode(aarch64_enc_strw(src, mem));
 7591 
 7592   ins_pipe(istore_reg_mem);
 7593 %}
 7594 
 7595 instruct storeimmI0(immI0 zero, memory4 mem)
 7596 %{
 7597   match(Set mem(StoreI mem zero));
 7598   predicate(!needs_releasing_store(n));
 7599 
 7600   ins_cost(INSN_COST);
 7601   format %{ "strw  zr, $mem\t# int" %}
 7602 
 7603   ins_encode(aarch64_enc_strw0(mem));
 7604 
 7605   ins_pipe(istore_mem);
 7606 %}
 7607 
 7608 // Store Long (64 bit signed)
 7609 instruct storeL(iRegL src, memory8 mem)
 7610 %{
 7611   match(Set mem (StoreL mem src));
 7612   predicate(!needs_releasing_store(n));
 7613 
 7614   ins_cost(INSN_COST);
 7615   format %{ "str  $src, $mem\t# int" %}
 7616 
 7617   ins_encode(aarch64_enc_str(src, mem));
 7618 
 7619   ins_pipe(istore_reg_mem);
 7620 %}
 7621 
 7622 // Store Long (64 bit signed)
 7623 instruct storeimmL0(immL0 zero, memory8 mem)
 7624 %{
 7625   match(Set mem (StoreL mem zero));
 7626   predicate(!needs_releasing_store(n));
 7627 
 7628   ins_cost(INSN_COST);
 7629   format %{ "str  zr, $mem\t# int" %}
 7630 
 7631   ins_encode(aarch64_enc_str0(mem));
 7632 
 7633   ins_pipe(istore_mem);
 7634 %}
 7635 
 7636 // Store Pointer
 7637 instruct storeP(iRegP src, memory8 mem)
 7638 %{
 7639   match(Set mem (StoreP mem src));
 7640   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7641 
 7642   ins_cost(INSN_COST);
 7643   format %{ "str  $src, $mem\t# ptr" %}
 7644 
 7645   ins_encode(aarch64_enc_str(src, mem));
 7646 
 7647   ins_pipe(istore_reg_mem);
 7648 %}
 7649 
 7650 // Store Pointer
 7651 instruct storeimmP0(immP0 zero, memory8 mem)
 7652 %{
 7653   match(Set mem (StoreP mem zero));
 7654   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7655 
 7656   ins_cost(INSN_COST);
 7657   format %{ "str zr, $mem\t# ptr" %}
 7658 
 7659   ins_encode(aarch64_enc_str0(mem));
 7660 
 7661   ins_pipe(istore_mem);
 7662 %}
 7663 
 7664 // Store Compressed Pointer
 7665 instruct storeN(iRegN src, memory4 mem)
 7666 %{
 7667   match(Set mem (StoreN mem src));
 7668   predicate(!needs_releasing_store(n));
 7669 
 7670   ins_cost(INSN_COST);
 7671   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7672 
 7673   ins_encode(aarch64_enc_strw(src, mem));
 7674 
 7675   ins_pipe(istore_reg_mem);
 7676 %}
 7677 
 7678 instruct storeImmN0(immN0 zero, memory4 mem)
 7679 %{
 7680   match(Set mem (StoreN mem zero));
 7681   predicate(!needs_releasing_store(n));
 7682 
 7683   ins_cost(INSN_COST);
 7684   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7685 
 7686   ins_encode(aarch64_enc_strw0(mem));
 7687 
 7688   ins_pipe(istore_mem);
 7689 %}
 7690 
 7691 // Store Float
 7692 instruct storeF(vRegF src, memory4 mem)
 7693 %{
 7694   match(Set mem (StoreF mem src));
 7695   predicate(!needs_releasing_store(n));
 7696 
 7697   ins_cost(INSN_COST);
 7698   format %{ "strs  $src, $mem\t# float" %}
 7699 
 7700   ins_encode( aarch64_enc_strs(src, mem) );
 7701 
 7702   ins_pipe(pipe_class_memory);
 7703 %}
 7704 
 7705 // TODO
 7706 // implement storeImmF0 and storeFImmPacked
 7707 
 7708 // Store Double
 7709 instruct storeD(vRegD src, memory8 mem)
 7710 %{
 7711   match(Set mem (StoreD mem src));
 7712   predicate(!needs_releasing_store(n));
 7713 
 7714   ins_cost(INSN_COST);
 7715   format %{ "strd  $src, $mem\t# double" %}
 7716 
 7717   ins_encode( aarch64_enc_strd(src, mem) );
 7718 
 7719   ins_pipe(pipe_class_memory);
 7720 %}
 7721 
 7722 // Store Compressed Klass Pointer
 7723 instruct storeNKlass(iRegN src, memory4 mem)
 7724 %{
 7725   predicate(!needs_releasing_store(n));
 7726   match(Set mem (StoreNKlass mem src));
 7727 
 7728   ins_cost(INSN_COST);
 7729   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7730 
 7731   ins_encode(aarch64_enc_strw(src, mem));
 7732 
 7733   ins_pipe(istore_reg_mem);
 7734 %}
 7735 
 7736 // TODO
 7737 // implement storeImmD0 and storeDImmPacked
 7738 
 7739 // prefetch instructions
 7740 // Must be safe to execute with invalid address (cannot fault).
 7741 
 7742 instruct prefetchalloc( memory8 mem ) %{
 7743   match(PrefetchAllocation mem);
 7744 
 7745   ins_cost(INSN_COST);
 7746   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7747 
 7748   ins_encode( aarch64_enc_prefetchw(mem) );
 7749 
 7750   ins_pipe(iload_prefetch);
 7751 %}
 7752 
 7753 //  ---------------- volatile loads and stores ----------------
 7754 
 7755 // Load Byte (8 bit signed)
 7756 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7757 %{
 7758   match(Set dst (LoadB mem));
 7759 
 7760   ins_cost(VOLATILE_REF_COST);
 7761   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7762 
 7763   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7764 
 7765   ins_pipe(pipe_serial);
 7766 %}
 7767 
 7768 // Load Byte (8 bit signed) into long
 7769 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7770 %{
 7771   match(Set dst (ConvI2L (LoadB mem)));
 7772 
 7773   ins_cost(VOLATILE_REF_COST);
 7774   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7775 
 7776   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7777 
 7778   ins_pipe(pipe_serial);
 7779 %}
 7780 
 7781 // Load Byte (8 bit unsigned)
 7782 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7783 %{
 7784   match(Set dst (LoadUB mem));
 7785 
 7786   ins_cost(VOLATILE_REF_COST);
 7787   format %{ "ldarb  $dst, $mem\t# byte" %}
 7788 
 7789   ins_encode(aarch64_enc_ldarb(dst, mem));
 7790 
 7791   ins_pipe(pipe_serial);
 7792 %}
 7793 
 7794 // Load Byte (8 bit unsigned) into long
 7795 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7796 %{
 7797   match(Set dst (ConvI2L (LoadUB mem)));
 7798 
 7799   ins_cost(VOLATILE_REF_COST);
 7800   format %{ "ldarb  $dst, $mem\t# byte" %}
 7801 
 7802   ins_encode(aarch64_enc_ldarb(dst, mem));
 7803 
 7804   ins_pipe(pipe_serial);
 7805 %}
 7806 
 7807 // Load Short (16 bit signed)
 7808 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7809 %{
 7810   match(Set dst (LoadS mem));
 7811 
 7812   ins_cost(VOLATILE_REF_COST);
 7813   format %{ "ldarshw  $dst, $mem\t# short" %}
 7814 
 7815   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7816 
 7817   ins_pipe(pipe_serial);
 7818 %}
 7819 
 7820 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7821 %{
 7822   match(Set dst (LoadUS mem));
 7823 
 7824   ins_cost(VOLATILE_REF_COST);
 7825   format %{ "ldarhw  $dst, $mem\t# short" %}
 7826 
 7827   ins_encode(aarch64_enc_ldarhw(dst, mem));
 7828 
 7829   ins_pipe(pipe_serial);
 7830 %}
 7831 
 7832 // Load Short/Char (16 bit unsigned) into long
 7833 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7834 %{
 7835   match(Set dst (ConvI2L (LoadUS mem)));
 7836 
 7837   ins_cost(VOLATILE_REF_COST);
 7838   format %{ "ldarh  $dst, $mem\t# short" %}
 7839 
 7840   ins_encode(aarch64_enc_ldarh(dst, mem));
 7841 
 7842   ins_pipe(pipe_serial);
 7843 %}
 7844 
 7845 // Load Short/Char (16 bit signed) into long
 7846 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7847 %{
 7848   match(Set dst (ConvI2L (LoadS mem)));
 7849 
 7850   ins_cost(VOLATILE_REF_COST);
 7851   format %{ "ldarh  $dst, $mem\t# short" %}
 7852 
 7853   ins_encode(aarch64_enc_ldarsh(dst, mem));
 7854 
 7855   ins_pipe(pipe_serial);
 7856 %}
 7857 
 7858 // Load Integer (32 bit signed)
 7859 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7860 %{
 7861   match(Set dst (LoadI mem));
 7862 
 7863   ins_cost(VOLATILE_REF_COST);
 7864   format %{ "ldarw  $dst, $mem\t# int" %}
 7865 
 7866   ins_encode(aarch64_enc_ldarw(dst, mem));
 7867 
 7868   ins_pipe(pipe_serial);
 7869 %}
 7870 
 7871 // Load Integer (32 bit unsigned) into long
 7872 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 7873 %{
 7874   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7875 
 7876   ins_cost(VOLATILE_REF_COST);
 7877   format %{ "ldarw  $dst, $mem\t# int" %}
 7878 
 7879   ins_encode(aarch64_enc_ldarw(dst, mem));
 7880 
 7881   ins_pipe(pipe_serial);
 7882 %}
 7883 
 7884 // Load Long (64 bit signed)
 7885 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7886 %{
 7887   match(Set dst (LoadL mem));
 7888 
 7889   ins_cost(VOLATILE_REF_COST);
 7890   format %{ "ldar  $dst, $mem\t# int" %}
 7891 
 7892   ins_encode(aarch64_enc_ldar(dst, mem));
 7893 
 7894   ins_pipe(pipe_serial);
 7895 %}
 7896 
 7897 // Load Pointer
 7898 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 7899 %{
 7900   match(Set dst (LoadP mem));
 7901   predicate(n->as_Load()->barrier_data() == 0);
 7902 
 7903   ins_cost(VOLATILE_REF_COST);
 7904   format %{ "ldar  $dst, $mem\t# ptr" %}
 7905 
 7906   ins_encode(aarch64_enc_ldar(dst, mem));
 7907 
 7908   ins_pipe(pipe_serial);
 7909 %}
 7910 
 7911 // Load Compressed Pointer
 7912 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 7913 %{
 7914   match(Set dst (LoadN mem));
 7915 
 7916   ins_cost(VOLATILE_REF_COST);
 7917   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 7918 
 7919   ins_encode(aarch64_enc_ldarw(dst, mem));
 7920 
 7921   ins_pipe(pipe_serial);
 7922 %}
 7923 
 7924 // Load Float
 7925 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 7926 %{
 7927   match(Set dst (LoadF mem));
 7928 
 7929   ins_cost(VOLATILE_REF_COST);
 7930   format %{ "ldars  $dst, $mem\t# float" %}
 7931 
 7932   ins_encode( aarch64_enc_fldars(dst, mem) );
 7933 
 7934   ins_pipe(pipe_serial);
 7935 %}
 7936 
 7937 // Load Double
 7938 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 7939 %{
 7940   match(Set dst (LoadD mem));
 7941 
 7942   ins_cost(VOLATILE_REF_COST);
 7943   format %{ "ldard  $dst, $mem\t# double" %}
 7944 
 7945   ins_encode( aarch64_enc_fldard(dst, mem) );
 7946 
 7947   ins_pipe(pipe_serial);
 7948 %}
 7949 
 7950 // Store Byte
 7951 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7952 %{
 7953   match(Set mem (StoreB mem src));
 7954 
 7955   ins_cost(VOLATILE_REF_COST);
 7956   format %{ "stlrb  $src, $mem\t# byte" %}
 7957 
 7958   ins_encode(aarch64_enc_stlrb(src, mem));
 7959 
 7960   ins_pipe(pipe_class_memory);
 7961 %}
 7962 
 7963 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7964 %{
 7965   match(Set mem (StoreB mem zero));
 7966 
 7967   ins_cost(VOLATILE_REF_COST);
 7968   format %{ "stlrb  zr, $mem\t# byte" %}
 7969 
 7970   ins_encode(aarch64_enc_stlrb0(mem));
 7971 
 7972   ins_pipe(pipe_class_memory);
 7973 %}
 7974 
 7975 // Store Char/Short
 7976 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7977 %{
 7978   match(Set mem (StoreC mem src));
 7979 
 7980   ins_cost(VOLATILE_REF_COST);
 7981   format %{ "stlrh  $src, $mem\t# short" %}
 7982 
 7983   ins_encode(aarch64_enc_stlrh(src, mem));
 7984 
 7985   ins_pipe(pipe_class_memory);
 7986 %}
 7987 
 7988 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7989 %{
 7990   match(Set mem (StoreC mem zero));
 7991 
 7992   ins_cost(VOLATILE_REF_COST);
 7993   format %{ "stlrh  zr, $mem\t# short" %}
 7994 
 7995   ins_encode(aarch64_enc_stlrh0(mem));
 7996 
 7997   ins_pipe(pipe_class_memory);
 7998 %}
 7999 
 8000 // Store Integer
 8001 
 8002 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8003 %{
 8004   match(Set mem(StoreI mem src));
 8005 
 8006   ins_cost(VOLATILE_REF_COST);
 8007   format %{ "stlrw  $src, $mem\t# int" %}
 8008 
 8009   ins_encode(aarch64_enc_stlrw(src, mem));
 8010 
 8011   ins_pipe(pipe_class_memory);
 8012 %}
 8013 
 8014 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8015 %{
 8016   match(Set mem(StoreI mem zero));
 8017 
 8018   ins_cost(VOLATILE_REF_COST);
 8019   format %{ "stlrw  zr, $mem\t# int" %}
 8020 
 8021   ins_encode(aarch64_enc_stlrw0(mem));
 8022 
 8023   ins_pipe(pipe_class_memory);
 8024 %}
 8025 
 8026 // Store Long (64 bit signed)
 8027 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 8028 %{
 8029   match(Set mem (StoreL mem src));
 8030 
 8031   ins_cost(VOLATILE_REF_COST);
 8032   format %{ "stlr  $src, $mem\t# int" %}
 8033 
 8034   ins_encode(aarch64_enc_stlr(src, mem));
 8035 
 8036   ins_pipe(pipe_class_memory);
 8037 %}
 8038 
 8039 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 8040 %{
 8041   match(Set mem (StoreL mem zero));
 8042 
 8043   ins_cost(VOLATILE_REF_COST);
 8044   format %{ "stlr  zr, $mem\t# int" %}
 8045 
 8046   ins_encode(aarch64_enc_stlr0(mem));
 8047 
 8048   ins_pipe(pipe_class_memory);
 8049 %}
 8050 
 8051 // Store Pointer
 8052 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 8053 %{
 8054   match(Set mem (StoreP mem src));
 8055   predicate(n->as_Store()->barrier_data() == 0);
 8056 
 8057   ins_cost(VOLATILE_REF_COST);
 8058   format %{ "stlr  $src, $mem\t# ptr" %}
 8059 
 8060   ins_encode(aarch64_enc_stlr(src, mem));
 8061 
 8062   ins_pipe(pipe_class_memory);
 8063 %}
 8064 
 8065 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 8066 %{
 8067   match(Set mem (StoreP mem zero));
 8068   predicate(n->as_Store()->barrier_data() == 0);
 8069 
 8070   ins_cost(VOLATILE_REF_COST);
 8071   format %{ "stlr  zr, $mem\t# ptr" %}
 8072 
 8073   ins_encode(aarch64_enc_stlr0(mem));
 8074 
 8075   ins_pipe(pipe_class_memory);
 8076 %}
 8077 
 8078 // Store Compressed Pointer
 8079 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8080 %{
 8081   match(Set mem (StoreN mem src));
 8082 
 8083   ins_cost(VOLATILE_REF_COST);
 8084   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8085 
 8086   ins_encode(aarch64_enc_stlrw(src, mem));
 8087 
 8088   ins_pipe(pipe_class_memory);
 8089 %}
 8090 
 8091 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 8092 %{
 8093   match(Set mem (StoreN mem zero));
 8094 
 8095   ins_cost(VOLATILE_REF_COST);
 8096   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 8097 
 8098   ins_encode(aarch64_enc_stlrw0(mem));
 8099 
 8100   ins_pipe(pipe_class_memory);
 8101 %}
 8102 
 8103 // Store Float
 8104 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8105 %{
 8106   match(Set mem (StoreF mem src));
 8107 
 8108   ins_cost(VOLATILE_REF_COST);
 8109   format %{ "stlrs  $src, $mem\t# float" %}
 8110 
 8111   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8112 
 8113   ins_pipe(pipe_class_memory);
 8114 %}
 8115 
 8116 // TODO
 8117 // implement storeImmF0 and storeFImmPacked
 8118 
 8119 // Store Double
 8120 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8121 %{
 8122   match(Set mem (StoreD mem src));
 8123 
 8124   ins_cost(VOLATILE_REF_COST);
 8125   format %{ "stlrd  $src, $mem\t# double" %}
 8126 
 8127   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8128 
 8129   ins_pipe(pipe_class_memory);
 8130 %}
 8131 
 8132 //  ---------------- end of volatile loads and stores ----------------
 8133 
 8134 instruct cacheWB(indirect addr)
 8135 %{
 8136   predicate(VM_Version::supports_data_cache_line_flush());
 8137   match(CacheWB addr);
 8138 
 8139   ins_cost(100);
 8140   format %{"cache wb $addr" %}
 8141   ins_encode %{
 8142     assert($addr->index_position() < 0, "should be");
 8143     assert($addr$$disp == 0, "should be");
 8144     __ cache_wb(Address($addr$$base$$Register, 0));
 8145   %}
 8146   ins_pipe(pipe_slow); // XXX
 8147 %}
 8148 
 8149 instruct cacheWBPreSync()
 8150 %{
 8151   predicate(VM_Version::supports_data_cache_line_flush());
 8152   match(CacheWBPreSync);
 8153 
 8154   ins_cost(100);
 8155   format %{"cache wb presync" %}
 8156   ins_encode %{
 8157     __ cache_wbsync(true);
 8158   %}
 8159   ins_pipe(pipe_slow); // XXX
 8160 %}
 8161 
 8162 instruct cacheWBPostSync()
 8163 %{
 8164   predicate(VM_Version::supports_data_cache_line_flush());
 8165   match(CacheWBPostSync);
 8166 
 8167   ins_cost(100);
 8168   format %{"cache wb postsync" %}
 8169   ins_encode %{
 8170     __ cache_wbsync(false);
 8171   %}
 8172   ins_pipe(pipe_slow); // XXX
 8173 %}
 8174 
 8175 // ============================================================================
 8176 // BSWAP Instructions
 8177 
 8178 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8179   match(Set dst (ReverseBytesI src));
 8180 
 8181   ins_cost(INSN_COST);
 8182   format %{ "revw  $dst, $src" %}
 8183 
 8184   ins_encode %{
 8185     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8186   %}
 8187 
 8188   ins_pipe(ialu_reg);
 8189 %}
 8190 
 8191 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8192   match(Set dst (ReverseBytesL src));
 8193 
 8194   ins_cost(INSN_COST);
 8195   format %{ "rev  $dst, $src" %}
 8196 
 8197   ins_encode %{
 8198     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8199   %}
 8200 
 8201   ins_pipe(ialu_reg);
 8202 %}
 8203 
 8204 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8205   match(Set dst (ReverseBytesUS src));
 8206 
 8207   ins_cost(INSN_COST);
 8208   format %{ "rev16w  $dst, $src" %}
 8209 
 8210   ins_encode %{
 8211     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8212   %}
 8213 
 8214   ins_pipe(ialu_reg);
 8215 %}
 8216 
 8217 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8218   match(Set dst (ReverseBytesS src));
 8219 
 8220   ins_cost(INSN_COST);
 8221   format %{ "rev16w  $dst, $src\n\t"
 8222             "sbfmw $dst, $dst, #0, #15" %}
 8223 
 8224   ins_encode %{
 8225     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8226     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8227   %}
 8228 
 8229   ins_pipe(ialu_reg);
 8230 %}
 8231 
 8232 // ============================================================================
 8233 // Zero Count Instructions
 8234 
 8235 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8236   match(Set dst (CountLeadingZerosI src));
 8237 
 8238   ins_cost(INSN_COST);
 8239   format %{ "clzw  $dst, $src" %}
 8240   ins_encode %{
 8241     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8242   %}
 8243 
 8244   ins_pipe(ialu_reg);
 8245 %}
 8246 
 8247 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8248   match(Set dst (CountLeadingZerosL src));
 8249 
 8250   ins_cost(INSN_COST);
 8251   format %{ "clz   $dst, $src" %}
 8252   ins_encode %{
 8253     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8254   %}
 8255 
 8256   ins_pipe(ialu_reg);
 8257 %}
 8258 
 8259 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8260   match(Set dst (CountTrailingZerosI src));
 8261 
 8262   ins_cost(INSN_COST * 2);
 8263   format %{ "rbitw  $dst, $src\n\t"
 8264             "clzw   $dst, $dst" %}
 8265   ins_encode %{
 8266     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8267     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8268   %}
 8269 
 8270   ins_pipe(ialu_reg);
 8271 %}
 8272 
 8273 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8274   match(Set dst (CountTrailingZerosL src));
 8275 
 8276   ins_cost(INSN_COST * 2);
 8277   format %{ "rbit   $dst, $src\n\t"
 8278             "clz    $dst, $dst" %}
 8279   ins_encode %{
 8280     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8281     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8282   %}
 8283 
 8284   ins_pipe(ialu_reg);
 8285 %}
 8286 
 8287 //---------- Population Count Instructions -------------------------------------
 8288 //
 8289 
 8290 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8291   match(Set dst (PopCountI src));
 8292   effect(TEMP tmp);
 8293   ins_cost(INSN_COST * 13);
 8294 
 8295   format %{ "movw   $src, $src\n\t"
 8296             "mov    $tmp, $src\t# vector (1D)\n\t"
 8297             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8298             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8299             "mov    $dst, $tmp\t# vector (1D)" %}
 8300   ins_encode %{
 8301     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8302     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8303     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8304     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8305     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8306   %}
 8307 
 8308   ins_pipe(pipe_class_default);
 8309 %}
 8310 
 8311 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8312   match(Set dst (PopCountI (LoadI mem)));
 8313   effect(TEMP tmp);
 8314   ins_cost(INSN_COST * 13);
 8315 
 8316   format %{ "ldrs   $tmp, $mem\n\t"
 8317             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8318             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8319             "mov    $dst, $tmp\t# vector (1D)" %}
 8320   ins_encode %{
 8321     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8322     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8323               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8324     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8325     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8326     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8327   %}
 8328 
 8329   ins_pipe(pipe_class_default);
 8330 %}
 8331 
 8332 // Note: Long.bitCount(long) returns an int.
 8333 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8334   match(Set dst (PopCountL src));
 8335   effect(TEMP tmp);
 8336   ins_cost(INSN_COST * 13);
 8337 
 8338   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8339             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8340             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8341             "mov    $dst, $tmp\t# vector (1D)" %}
 8342   ins_encode %{
 8343     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8344     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8345     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8346     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8347   %}
 8348 
 8349   ins_pipe(pipe_class_default);
 8350 %}
 8351 
 8352 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8353   match(Set dst (PopCountL (LoadL mem)));
 8354   effect(TEMP tmp);
 8355   ins_cost(INSN_COST * 13);
 8356 
 8357   format %{ "ldrd   $tmp, $mem\n\t"
 8358             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8359             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8360             "mov    $dst, $tmp\t# vector (1D)" %}
 8361   ins_encode %{
 8362     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8363     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8364               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8365     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8366     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8367     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8368   %}
 8369 
 8370   ins_pipe(pipe_class_default);
 8371 %}
 8372 
 8373 // ============================================================================
 8374 // MemBar Instruction
 8375 
 8376 instruct load_fence() %{
 8377   match(LoadFence);
 8378   ins_cost(VOLATILE_REF_COST);
 8379 
 8380   format %{ "load_fence" %}
 8381 
 8382   ins_encode %{
 8383     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8384   %}
 8385   ins_pipe(pipe_serial);
 8386 %}
 8387 
 8388 instruct unnecessary_membar_acquire() %{
 8389   predicate(unnecessary_acquire(n));
 8390   match(MemBarAcquire);
 8391   ins_cost(0);
 8392 
 8393   format %{ "membar_acquire (elided)" %}
 8394 
 8395   ins_encode %{
 8396     __ block_comment("membar_acquire (elided)");
 8397   %}
 8398 
 8399   ins_pipe(pipe_class_empty);
 8400 %}
 8401 
 8402 instruct membar_acquire() %{
 8403   match(MemBarAcquire);
 8404   ins_cost(VOLATILE_REF_COST);
 8405 
 8406   format %{ "membar_acquire\n\t"
 8407             "dmb ish" %}
 8408 
 8409   ins_encode %{
 8410     __ block_comment("membar_acquire");
 8411     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8412   %}
 8413 
 8414   ins_pipe(pipe_serial);
 8415 %}
 8416 
 8417 
 8418 instruct membar_acquire_lock() %{
 8419   match(MemBarAcquireLock);
 8420   ins_cost(VOLATILE_REF_COST);
 8421 
 8422   format %{ "membar_acquire_lock (elided)" %}
 8423 
 8424   ins_encode %{
 8425     __ block_comment("membar_acquire_lock (elided)");
 8426   %}
 8427 
 8428   ins_pipe(pipe_serial);
 8429 %}
 8430 
 8431 instruct store_fence() %{
 8432   match(StoreFence);
 8433   ins_cost(VOLATILE_REF_COST);
 8434 
 8435   format %{ "store_fence" %}
 8436 
 8437   ins_encode %{
 8438     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8439   %}
 8440   ins_pipe(pipe_serial);
 8441 %}
 8442 
 8443 instruct unnecessary_membar_release() %{
 8444   predicate(unnecessary_release(n));
 8445   match(MemBarRelease);
 8446   ins_cost(0);
 8447 
 8448   format %{ "membar_release (elided)" %}
 8449 
 8450   ins_encode %{
 8451     __ block_comment("membar_release (elided)");
 8452   %}
 8453   ins_pipe(pipe_serial);
 8454 %}
 8455 
 8456 instruct membar_release() %{
 8457   match(MemBarRelease);
 8458   ins_cost(VOLATILE_REF_COST);
 8459 
 8460   format %{ "membar_release\n\t"
 8461             "dmb ish" %}
 8462 
 8463   ins_encode %{
 8464     __ block_comment("membar_release");
 8465     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8466   %}
 8467   ins_pipe(pipe_serial);
 8468 %}
 8469 
 8470 instruct membar_storestore() %{
 8471   match(MemBarStoreStore);
 8472   match(StoreStoreFence);
 8473   ins_cost(VOLATILE_REF_COST);
 8474 
 8475   format %{ "MEMBAR-store-store" %}
 8476 
 8477   ins_encode %{
 8478     __ membar(Assembler::StoreStore);
 8479   %}
 8480   ins_pipe(pipe_serial);
 8481 %}
 8482 
 8483 instruct membar_release_lock() %{
 8484   match(MemBarReleaseLock);
 8485   ins_cost(VOLATILE_REF_COST);
 8486 
 8487   format %{ "membar_release_lock (elided)" %}
 8488 
 8489   ins_encode %{
 8490     __ block_comment("membar_release_lock (elided)");
 8491   %}
 8492 
 8493   ins_pipe(pipe_serial);
 8494 %}
 8495 
 8496 instruct unnecessary_membar_volatile() %{
 8497   predicate(unnecessary_volatile(n));
 8498   match(MemBarVolatile);
 8499   ins_cost(0);
 8500 
 8501   format %{ "membar_volatile (elided)" %}
 8502 
 8503   ins_encode %{
 8504     __ block_comment("membar_volatile (elided)");
 8505   %}
 8506 
 8507   ins_pipe(pipe_serial);
 8508 %}
 8509 
 8510 instruct membar_volatile() %{
 8511   match(MemBarVolatile);
 8512   ins_cost(VOLATILE_REF_COST*100);
 8513 
 8514   format %{ "membar_volatile\n\t"
 8515              "dmb ish"%}
 8516 
 8517   ins_encode %{
 8518     __ block_comment("membar_volatile");
 8519     __ membar(Assembler::StoreLoad);
 8520   %}
 8521 
 8522   ins_pipe(pipe_serial);
 8523 %}
 8524 
 8525 // ============================================================================
 8526 // Cast/Convert Instructions
 8527 
 8528 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8529   match(Set dst (CastX2P src));
 8530 
 8531   ins_cost(INSN_COST);
 8532   format %{ "mov $dst, $src\t# long -> ptr" %}
 8533 
 8534   ins_encode %{
 8535     if ($dst$$reg != $src$$reg) {
 8536       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8537     }
 8538   %}
 8539 
 8540   ins_pipe(ialu_reg);
 8541 %}
 8542 
 8543 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8544   match(Set dst (CastP2X src));
 8545 
 8546   ins_cost(INSN_COST);
 8547   format %{ "mov $dst, $src\t# ptr -> long" %}
 8548 
 8549   ins_encode %{
 8550     if ($dst$$reg != $src$$reg) {
 8551       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8552     }
 8553   %}
 8554 
 8555   ins_pipe(ialu_reg);
 8556 %}
 8557 
 8558 // Convert oop into int for vectors alignment masking
 8559 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8560   match(Set dst (ConvL2I (CastP2X src)));
 8561 
 8562   ins_cost(INSN_COST);
 8563   format %{ "movw $dst, $src\t# ptr -> int" %}
 8564   ins_encode %{
 8565     __ movw($dst$$Register, $src$$Register);
 8566   %}
 8567 
 8568   ins_pipe(ialu_reg);
 8569 %}
 8570 
 8571 // Convert compressed oop into int for vectors alignment masking
 8572 // in case of 32bit oops (heap < 4Gb).
 8573 instruct convN2I(iRegINoSp dst, iRegN src)
 8574 %{
 8575   predicate(CompressedOops::shift() == 0);
 8576   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8577 
 8578   ins_cost(INSN_COST);
 8579   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8580   ins_encode %{
 8581     __ movw($dst$$Register, $src$$Register);
 8582   %}
 8583 
 8584   ins_pipe(ialu_reg);
 8585 %}
 8586 
 8587 
 8588 // Convert oop pointer into compressed form
 8589 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8590   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8591   match(Set dst (EncodeP src));
 8592   effect(KILL cr);
 8593   ins_cost(INSN_COST * 3);
 8594   format %{ "encode_heap_oop $dst, $src" %}
 8595   ins_encode %{
 8596     Register s = $src$$Register;
 8597     Register d = $dst$$Register;
 8598     __ encode_heap_oop(d, s);
 8599   %}
 8600   ins_pipe(ialu_reg);
 8601 %}
 8602 
 8603 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8604   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8605   match(Set dst (EncodeP src));
 8606   ins_cost(INSN_COST * 3);
 8607   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8608   ins_encode %{
 8609     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8610   %}
 8611   ins_pipe(ialu_reg);
 8612 %}
 8613 
 8614 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8615   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8616             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8617   match(Set dst (DecodeN src));
 8618   ins_cost(INSN_COST * 3);
 8619   format %{ "decode_heap_oop $dst, $src" %}
 8620   ins_encode %{
 8621     Register s = $src$$Register;
 8622     Register d = $dst$$Register;
 8623     __ decode_heap_oop(d, s);
 8624   %}
 8625   ins_pipe(ialu_reg);
 8626 %}
 8627 
 8628 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8629   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8630             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8631   match(Set dst (DecodeN src));
 8632   ins_cost(INSN_COST * 3);
 8633   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8634   ins_encode %{
 8635     Register s = $src$$Register;
 8636     Register d = $dst$$Register;
 8637     __ decode_heap_oop_not_null(d, s);
 8638   %}
 8639   ins_pipe(ialu_reg);
 8640 %}
 8641 
 8642 // n.b. AArch64 implementations of encode_klass_not_null and
 8643 // decode_klass_not_null do not modify the flags register so, unlike
 8644 // Intel, we don't kill CR as a side effect here
 8645 
 8646 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8647   match(Set dst (EncodePKlass src));
 8648 
 8649   ins_cost(INSN_COST * 3);
 8650   format %{ "encode_klass_not_null $dst,$src" %}
 8651 
 8652   ins_encode %{
 8653     Register src_reg = as_Register($src$$reg);
 8654     Register dst_reg = as_Register($dst$$reg);
 8655     __ encode_klass_not_null(dst_reg, src_reg);
 8656   %}
 8657 
 8658    ins_pipe(ialu_reg);
 8659 %}
 8660 
 8661 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8662   match(Set dst (DecodeNKlass src));
 8663 
 8664   ins_cost(INSN_COST * 3);
 8665   format %{ "decode_klass_not_null $dst,$src" %}
 8666 
 8667   ins_encode %{
 8668     Register src_reg = as_Register($src$$reg);
 8669     Register dst_reg = as_Register($dst$$reg);
 8670     if (dst_reg != src_reg) {
 8671       __ decode_klass_not_null(dst_reg, src_reg);
 8672     } else {
 8673       __ decode_klass_not_null(dst_reg);
 8674     }
 8675   %}
 8676 
 8677    ins_pipe(ialu_reg);
 8678 %}
 8679 
 8680 instruct checkCastPP(iRegPNoSp dst)
 8681 %{
 8682   match(Set dst (CheckCastPP dst));
 8683 
 8684   size(0);
 8685   format %{ "# checkcastPP of $dst" %}
 8686   ins_encode(/* empty encoding */);
 8687   ins_pipe(pipe_class_empty);
 8688 %}
 8689 
 8690 instruct castPP(iRegPNoSp dst)
 8691 %{
 8692   match(Set dst (CastPP dst));
 8693 
 8694   size(0);
 8695   format %{ "# castPP of $dst" %}
 8696   ins_encode(/* empty encoding */);
 8697   ins_pipe(pipe_class_empty);
 8698 %}
 8699 
 8700 instruct castII(iRegI dst)
 8701 %{
 8702   match(Set dst (CastII dst));
 8703 
 8704   size(0);
 8705   format %{ "# castII of $dst" %}
 8706   ins_encode(/* empty encoding */);
 8707   ins_cost(0);
 8708   ins_pipe(pipe_class_empty);
 8709 %}
 8710 
 8711 instruct castLL(iRegL dst)
 8712 %{
 8713   match(Set dst (CastLL dst));
 8714 
 8715   size(0);
 8716   format %{ "# castLL of $dst" %}
 8717   ins_encode(/* empty encoding */);
 8718   ins_cost(0);
 8719   ins_pipe(pipe_class_empty);
 8720 %}
 8721 
 8722 instruct castFF(vRegF dst)
 8723 %{
 8724   match(Set dst (CastFF dst));
 8725 
 8726   size(0);
 8727   format %{ "# castFF of $dst" %}
 8728   ins_encode(/* empty encoding */);
 8729   ins_cost(0);
 8730   ins_pipe(pipe_class_empty);
 8731 %}
 8732 
 8733 instruct castDD(vRegD dst)
 8734 %{
 8735   match(Set dst (CastDD dst));
 8736 
 8737   size(0);
 8738   format %{ "# castDD of $dst" %}
 8739   ins_encode(/* empty encoding */);
 8740   ins_cost(0);
 8741   ins_pipe(pipe_class_empty);
 8742 %}
 8743 
 8744 instruct castVV(vReg dst)
 8745 %{
 8746   match(Set dst (CastVV dst));
 8747 
 8748   size(0);
 8749   format %{ "# castVV of $dst" %}
 8750   ins_encode(/* empty encoding */);
 8751   ins_cost(0);
 8752   ins_pipe(pipe_class_empty);
 8753 %}
 8754 
 8755 instruct castVVMask(pRegGov dst)
 8756 %{
 8757   match(Set dst (CastVV dst));
 8758 
 8759   size(0);
 8760   format %{ "# castVV of $dst" %}
 8761   ins_encode(/* empty encoding */);
 8762   ins_cost(0);
 8763   ins_pipe(pipe_class_empty);
 8764 %}
 8765 
 8766 // ============================================================================
 8767 // Atomic operation instructions
 8768 //
 8769 
 8770 // standard CompareAndSwapX when we are using barriers
 8771 // these have higher priority than the rules selected by a predicate
 8772 
 8773 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 8774 // can't match them
 8775 
 8776 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8777 
 8778   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8779   ins_cost(2 * VOLATILE_REF_COST);
 8780 
 8781   effect(KILL cr);
 8782 
 8783   format %{
 8784     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8785     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8786   %}
 8787 
 8788   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 8789             aarch64_enc_cset_eq(res));
 8790 
 8791   ins_pipe(pipe_slow);
 8792 %}
 8793 
 8794 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8795 
 8796   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8797   ins_cost(2 * VOLATILE_REF_COST);
 8798 
 8799   effect(KILL cr);
 8800 
 8801   format %{
 8802     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8803     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8804   %}
 8805 
 8806   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 8807             aarch64_enc_cset_eq(res));
 8808 
 8809   ins_pipe(pipe_slow);
 8810 %}
 8811 
 8812 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8813 
 8814   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8815   ins_cost(2 * VOLATILE_REF_COST);
 8816 
 8817   effect(KILL cr);
 8818 
 8819  format %{
 8820     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8821     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8822  %}
 8823 
 8824  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8825             aarch64_enc_cset_eq(res));
 8826 
 8827   ins_pipe(pipe_slow);
 8828 %}
 8829 
 8830 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8831 
 8832   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8833   ins_cost(2 * VOLATILE_REF_COST);
 8834 
 8835   effect(KILL cr);
 8836 
 8837  format %{
 8838     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8839     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8840  %}
 8841 
 8842  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8843             aarch64_enc_cset_eq(res));
 8844 
 8845   ins_pipe(pipe_slow);
 8846 %}
 8847 
 8848 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8849 
 8850   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8851   predicate(n->as_LoadStore()->barrier_data() == 0);
 8852   ins_cost(2 * VOLATILE_REF_COST);
 8853 
 8854   effect(KILL cr);
 8855 
 8856  format %{
 8857     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8858     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8859  %}
 8860 
 8861  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8862             aarch64_enc_cset_eq(res));
 8863 
 8864   ins_pipe(pipe_slow);
 8865 %}
 8866 
 8867 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8868 
 8869   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8870   ins_cost(2 * VOLATILE_REF_COST);
 8871 
 8872   effect(KILL cr);
 8873 
 8874  format %{
 8875     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8876     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8877  %}
 8878 
 8879  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8880             aarch64_enc_cset_eq(res));
 8881 
 8882   ins_pipe(pipe_slow);
 8883 %}
 8884 
 8885 // alternative CompareAndSwapX when we are eliding barriers
 8886 
 8887 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8888 
 8889   predicate(needs_acquiring_load_exclusive(n));
 8890   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8891   ins_cost(VOLATILE_REF_COST);
 8892 
 8893   effect(KILL cr);
 8894 
 8895   format %{
 8896     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8897     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8898   %}
 8899 
 8900   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 8901             aarch64_enc_cset_eq(res));
 8902 
 8903   ins_pipe(pipe_slow);
 8904 %}
 8905 
 8906 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8907 
 8908   predicate(needs_acquiring_load_exclusive(n));
 8909   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8910   ins_cost(VOLATILE_REF_COST);
 8911 
 8912   effect(KILL cr);
 8913 
 8914   format %{
 8915     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8916     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8917   %}
 8918 
 8919   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 8920             aarch64_enc_cset_eq(res));
 8921 
 8922   ins_pipe(pipe_slow);
 8923 %}
 8924 
 8925 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8926 
 8927   predicate(needs_acquiring_load_exclusive(n));
 8928   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8929   ins_cost(VOLATILE_REF_COST);
 8930 
 8931   effect(KILL cr);
 8932 
 8933  format %{
 8934     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8935     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8936  %}
 8937 
 8938  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8939             aarch64_enc_cset_eq(res));
 8940 
 8941   ins_pipe(pipe_slow);
 8942 %}
 8943 
 8944 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8945 
 8946   predicate(needs_acquiring_load_exclusive(n));
 8947   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8948   ins_cost(VOLATILE_REF_COST);
 8949 
 8950   effect(KILL cr);
 8951 
 8952  format %{
 8953     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8954     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8955  %}
 8956 
 8957  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8958             aarch64_enc_cset_eq(res));
 8959 
 8960   ins_pipe(pipe_slow);
 8961 %}
 8962 
 8963 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8964 
 8965   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8966   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8967   ins_cost(VOLATILE_REF_COST);
 8968 
 8969   effect(KILL cr);
 8970 
 8971  format %{
 8972     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8973     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8974  %}
 8975 
 8976  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8977             aarch64_enc_cset_eq(res));
 8978 
 8979   ins_pipe(pipe_slow);
 8980 %}
 8981 
 8982 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8983 
 8984   predicate(needs_acquiring_load_exclusive(n));
 8985   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8986   ins_cost(VOLATILE_REF_COST);
 8987 
 8988   effect(KILL cr);
 8989 
 8990  format %{
 8991     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8992     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8993  %}
 8994 
 8995  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8996             aarch64_enc_cset_eq(res));
 8997 
 8998   ins_pipe(pipe_slow);
 8999 %}
 9000 
 9001 
 9002 // ---------------------------------------------------------------------
 9003 
 9004 // BEGIN This section of the file is automatically generated. Do not edit --------------
 9005 
 9006 // Sundry CAS operations.  Note that release is always true,
 9007 // regardless of the memory ordering of the CAS.  This is because we
 9008 // need the volatile case to be sequentially consistent but there is
 9009 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 9010 // can't check the type of memory ordering here, so we always emit a
 9011 // STLXR.
 9012 
 9013 // This section is generated from aarch64_ad_cas.m4
 9014 
 9015 
 9016 
 9017 // This pattern is generated automatically from cas.m4.
 9018 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9019 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9020 
 9021   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9022   ins_cost(2 * VOLATILE_REF_COST);
 9023   effect(TEMP_DEF res, KILL cr);
 9024   format %{
 9025     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9026   %}
 9027   ins_encode %{
 9028     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9029                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9030                /*weak*/ false, $res$$Register);
 9031     __ sxtbw($res$$Register, $res$$Register);
 9032   %}
 9033   ins_pipe(pipe_slow);
 9034 %}
 9035 
 9036 // This pattern is generated automatically from cas.m4.
 9037 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9038 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9039 
 9040   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9041   ins_cost(2 * VOLATILE_REF_COST);
 9042   effect(TEMP_DEF res, KILL cr);
 9043   format %{
 9044     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9045   %}
 9046   ins_encode %{
 9047     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9048                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9049                /*weak*/ false, $res$$Register);
 9050     __ sxthw($res$$Register, $res$$Register);
 9051   %}
 9052   ins_pipe(pipe_slow);
 9053 %}
 9054 
 9055 // This pattern is generated automatically from cas.m4.
 9056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9057 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9058 
 9059   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9060   ins_cost(2 * VOLATILE_REF_COST);
 9061   effect(TEMP_DEF res, KILL cr);
 9062   format %{
 9063     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9064   %}
 9065   ins_encode %{
 9066     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9067                Assembler::word, /*acquire*/ false, /*release*/ true,
 9068                /*weak*/ false, $res$$Register);
 9069   %}
 9070   ins_pipe(pipe_slow);
 9071 %}
 9072 
 9073 // This pattern is generated automatically from cas.m4.
 9074 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9075 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9076 
 9077   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9078   ins_cost(2 * VOLATILE_REF_COST);
 9079   effect(TEMP_DEF res, KILL cr);
 9080   format %{
 9081     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9082   %}
 9083   ins_encode %{
 9084     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9085                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9086                /*weak*/ false, $res$$Register);
 9087   %}
 9088   ins_pipe(pipe_slow);
 9089 %}
 9090 
 9091 // This pattern is generated automatically from cas.m4.
 9092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9093 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9094 
 9095   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9096   ins_cost(2 * VOLATILE_REF_COST);
 9097   effect(TEMP_DEF res, KILL cr);
 9098   format %{
 9099     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9100   %}
 9101   ins_encode %{
 9102     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9103                Assembler::word, /*acquire*/ false, /*release*/ true,
 9104                /*weak*/ false, $res$$Register);
 9105   %}
 9106   ins_pipe(pipe_slow);
 9107 %}
 9108 
 9109 // This pattern is generated automatically from cas.m4.
 9110 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9111 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9112   predicate(n->as_LoadStore()->barrier_data() == 0);
 9113   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9114   ins_cost(2 * VOLATILE_REF_COST);
 9115   effect(TEMP_DEF res, KILL cr);
 9116   format %{
 9117     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9118   %}
 9119   ins_encode %{
 9120     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9121                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9122                /*weak*/ false, $res$$Register);
 9123   %}
 9124   ins_pipe(pipe_slow);
 9125 %}
 9126 
 9127 // This pattern is generated automatically from cas.m4.
 9128 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9129 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9130   predicate(needs_acquiring_load_exclusive(n));
 9131   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9132   ins_cost(VOLATILE_REF_COST);
 9133   effect(TEMP_DEF res, KILL cr);
 9134   format %{
 9135     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9136   %}
 9137   ins_encode %{
 9138     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9139                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9140                /*weak*/ false, $res$$Register);
 9141     __ sxtbw($res$$Register, $res$$Register);
 9142   %}
 9143   ins_pipe(pipe_slow);
 9144 %}
 9145 
 9146 // This pattern is generated automatically from cas.m4.
 9147 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9148 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9149   predicate(needs_acquiring_load_exclusive(n));
 9150   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9151   ins_cost(VOLATILE_REF_COST);
 9152   effect(TEMP_DEF res, KILL cr);
 9153   format %{
 9154     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9155   %}
 9156   ins_encode %{
 9157     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9158                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9159                /*weak*/ false, $res$$Register);
 9160     __ sxthw($res$$Register, $res$$Register);
 9161   %}
 9162   ins_pipe(pipe_slow);
 9163 %}
 9164 
 9165 // This pattern is generated automatically from cas.m4.
 9166 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9167 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9168   predicate(needs_acquiring_load_exclusive(n));
 9169   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9170   ins_cost(VOLATILE_REF_COST);
 9171   effect(TEMP_DEF res, KILL cr);
 9172   format %{
 9173     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9174   %}
 9175   ins_encode %{
 9176     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9177                Assembler::word, /*acquire*/ true, /*release*/ true,
 9178                /*weak*/ false, $res$$Register);
 9179   %}
 9180   ins_pipe(pipe_slow);
 9181 %}
 9182 
 9183 // This pattern is generated automatically from cas.m4.
 9184 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9185 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9186   predicate(needs_acquiring_load_exclusive(n));
 9187   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9188   ins_cost(VOLATILE_REF_COST);
 9189   effect(TEMP_DEF res, KILL cr);
 9190   format %{
 9191     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9192   %}
 9193   ins_encode %{
 9194     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9195                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9196                /*weak*/ false, $res$$Register);
 9197   %}
 9198   ins_pipe(pipe_slow);
 9199 %}
 9200 
 9201 // This pattern is generated automatically from cas.m4.
 9202 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9203 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9204   predicate(needs_acquiring_load_exclusive(n));
 9205   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9206   ins_cost(VOLATILE_REF_COST);
 9207   effect(TEMP_DEF res, KILL cr);
 9208   format %{
 9209     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9210   %}
 9211   ins_encode %{
 9212     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9213                Assembler::word, /*acquire*/ true, /*release*/ true,
 9214                /*weak*/ false, $res$$Register);
 9215   %}
 9216   ins_pipe(pipe_slow);
 9217 %}
 9218 
 9219 // This pattern is generated automatically from cas.m4.
 9220 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9221 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9222   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9223   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9224   ins_cost(VOLATILE_REF_COST);
 9225   effect(TEMP_DEF res, KILL cr);
 9226   format %{
 9227     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9228   %}
 9229   ins_encode %{
 9230     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9231                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9232                /*weak*/ false, $res$$Register);
 9233   %}
 9234   ins_pipe(pipe_slow);
 9235 %}
 9236 
 9237 // This pattern is generated automatically from cas.m4.
 9238 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9239 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9240 
 9241   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9242   ins_cost(2 * VOLATILE_REF_COST);
 9243   effect(KILL cr);
 9244   format %{
 9245     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9246     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9247   %}
 9248   ins_encode %{
 9249     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9250                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9251                /*weak*/ true, noreg);
 9252     __ csetw($res$$Register, Assembler::EQ);
 9253   %}
 9254   ins_pipe(pipe_slow);
 9255 %}
 9256 
 9257 // This pattern is generated automatically from cas.m4.
 9258 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9259 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9260 
 9261   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9262   ins_cost(2 * VOLATILE_REF_COST);
 9263   effect(KILL cr);
 9264   format %{
 9265     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9266     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9267   %}
 9268   ins_encode %{
 9269     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9270                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9271                /*weak*/ true, noreg);
 9272     __ csetw($res$$Register, Assembler::EQ);
 9273   %}
 9274   ins_pipe(pipe_slow);
 9275 %}
 9276 
 9277 // This pattern is generated automatically from cas.m4.
 9278 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9279 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9280 
 9281   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9282   ins_cost(2 * VOLATILE_REF_COST);
 9283   effect(KILL cr);
 9284   format %{
 9285     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9286     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9287   %}
 9288   ins_encode %{
 9289     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9290                Assembler::word, /*acquire*/ false, /*release*/ true,
 9291                /*weak*/ true, noreg);
 9292     __ csetw($res$$Register, Assembler::EQ);
 9293   %}
 9294   ins_pipe(pipe_slow);
 9295 %}
 9296 
 9297 // This pattern is generated automatically from cas.m4.
 9298 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9299 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9300 
 9301   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9302   ins_cost(2 * VOLATILE_REF_COST);
 9303   effect(KILL cr);
 9304   format %{
 9305     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9306     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9307   %}
 9308   ins_encode %{
 9309     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9310                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9311                /*weak*/ true, noreg);
 9312     __ csetw($res$$Register, Assembler::EQ);
 9313   %}
 9314   ins_pipe(pipe_slow);
 9315 %}
 9316 
 9317 // This pattern is generated automatically from cas.m4.
 9318 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9319 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9320 
 9321   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9322   ins_cost(2 * VOLATILE_REF_COST);
 9323   effect(KILL cr);
 9324   format %{
 9325     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9326     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9327   %}
 9328   ins_encode %{
 9329     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9330                Assembler::word, /*acquire*/ false, /*release*/ true,
 9331                /*weak*/ true, noreg);
 9332     __ csetw($res$$Register, Assembler::EQ);
 9333   %}
 9334   ins_pipe(pipe_slow);
 9335 %}
 9336 
 9337 // This pattern is generated automatically from cas.m4.
 9338 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9339 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9340   predicate(n->as_LoadStore()->barrier_data() == 0);
 9341   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9342   ins_cost(2 * VOLATILE_REF_COST);
 9343   effect(KILL cr);
 9344   format %{
 9345     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9346     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9347   %}
 9348   ins_encode %{
 9349     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9350                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9351                /*weak*/ true, noreg);
 9352     __ csetw($res$$Register, Assembler::EQ);
 9353   %}
 9354   ins_pipe(pipe_slow);
 9355 %}
 9356 
 9357 // This pattern is generated automatically from cas.m4.
 9358 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9359 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9360   predicate(needs_acquiring_load_exclusive(n));
 9361   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9362   ins_cost(VOLATILE_REF_COST);
 9363   effect(KILL cr);
 9364   format %{
 9365     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9366     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9367   %}
 9368   ins_encode %{
 9369     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9370                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9371                /*weak*/ true, noreg);
 9372     __ csetw($res$$Register, Assembler::EQ);
 9373   %}
 9374   ins_pipe(pipe_slow);
 9375 %}
 9376 
 9377 // This pattern is generated automatically from cas.m4.
 9378 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9379 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9380   predicate(needs_acquiring_load_exclusive(n));
 9381   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9382   ins_cost(VOLATILE_REF_COST);
 9383   effect(KILL cr);
 9384   format %{
 9385     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9386     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9387   %}
 9388   ins_encode %{
 9389     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9390                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9391                /*weak*/ true, noreg);
 9392     __ csetw($res$$Register, Assembler::EQ);
 9393   %}
 9394   ins_pipe(pipe_slow);
 9395 %}
 9396 
 9397 // This pattern is generated automatically from cas.m4.
 9398 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9399 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9400   predicate(needs_acquiring_load_exclusive(n));
 9401   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9402   ins_cost(VOLATILE_REF_COST);
 9403   effect(KILL cr);
 9404   format %{
 9405     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9406     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9407   %}
 9408   ins_encode %{
 9409     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9410                Assembler::word, /*acquire*/ true, /*release*/ true,
 9411                /*weak*/ true, noreg);
 9412     __ csetw($res$$Register, Assembler::EQ);
 9413   %}
 9414   ins_pipe(pipe_slow);
 9415 %}
 9416 
 9417 // This pattern is generated automatically from cas.m4.
 9418 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9419 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9420   predicate(needs_acquiring_load_exclusive(n));
 9421   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9422   ins_cost(VOLATILE_REF_COST);
 9423   effect(KILL cr);
 9424   format %{
 9425     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9426     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9427   %}
 9428   ins_encode %{
 9429     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9430                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9431                /*weak*/ true, noreg);
 9432     __ csetw($res$$Register, Assembler::EQ);
 9433   %}
 9434   ins_pipe(pipe_slow);
 9435 %}
 9436 
 9437 // This pattern is generated automatically from cas.m4.
 9438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9439 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9440   predicate(needs_acquiring_load_exclusive(n));
 9441   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9442   ins_cost(VOLATILE_REF_COST);
 9443   effect(KILL cr);
 9444   format %{
 9445     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9446     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9447   %}
 9448   ins_encode %{
 9449     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9450                Assembler::word, /*acquire*/ true, /*release*/ true,
 9451                /*weak*/ true, noreg);
 9452     __ csetw($res$$Register, Assembler::EQ);
 9453   %}
 9454   ins_pipe(pipe_slow);
 9455 %}
 9456 
 9457 // This pattern is generated automatically from cas.m4.
 9458 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9459 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9460   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9461   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9462   ins_cost(VOLATILE_REF_COST);
 9463   effect(KILL cr);
 9464   format %{
 9465     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9466     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9467   %}
 9468   ins_encode %{
 9469     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9470                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9471                /*weak*/ true, noreg);
 9472     __ csetw($res$$Register, Assembler::EQ);
 9473   %}
 9474   ins_pipe(pipe_slow);
 9475 %}
 9476 
 9477 // END This section of the file is automatically generated. Do not edit --------------
 9478 // ---------------------------------------------------------------------
 9479 
 9480 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9481   match(Set prev (GetAndSetI mem newv));
 9482   ins_cost(2 * VOLATILE_REF_COST);
 9483   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9484   ins_encode %{
 9485     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9486   %}
 9487   ins_pipe(pipe_serial);
 9488 %}
 9489 
 9490 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9491   match(Set prev (GetAndSetL mem newv));
 9492   ins_cost(2 * VOLATILE_REF_COST);
 9493   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9494   ins_encode %{
 9495     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9496   %}
 9497   ins_pipe(pipe_serial);
 9498 %}
 9499 
 9500 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9501   match(Set prev (GetAndSetN mem newv));
 9502   ins_cost(2 * VOLATILE_REF_COST);
 9503   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9504   ins_encode %{
 9505     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9506   %}
 9507   ins_pipe(pipe_serial);
 9508 %}
 9509 
 9510 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9511   predicate(n->as_LoadStore()->barrier_data() == 0);
 9512   match(Set prev (GetAndSetP mem newv));
 9513   ins_cost(2 * VOLATILE_REF_COST);
 9514   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9515   ins_encode %{
 9516     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9517   %}
 9518   ins_pipe(pipe_serial);
 9519 %}
 9520 
 9521 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9522   predicate(needs_acquiring_load_exclusive(n));
 9523   match(Set prev (GetAndSetI mem newv));
 9524   ins_cost(VOLATILE_REF_COST);
 9525   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9526   ins_encode %{
 9527     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9528   %}
 9529   ins_pipe(pipe_serial);
 9530 %}
 9531 
 9532 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9533   predicate(needs_acquiring_load_exclusive(n));
 9534   match(Set prev (GetAndSetL mem newv));
 9535   ins_cost(VOLATILE_REF_COST);
 9536   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9537   ins_encode %{
 9538     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9539   %}
 9540   ins_pipe(pipe_serial);
 9541 %}
 9542 
 9543 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9544   predicate(needs_acquiring_load_exclusive(n));
 9545   match(Set prev (GetAndSetN mem newv));
 9546   ins_cost(VOLATILE_REF_COST);
 9547   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9548   ins_encode %{
 9549     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9550   %}
 9551   ins_pipe(pipe_serial);
 9552 %}
 9553 
 9554 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9555   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9556   match(Set prev (GetAndSetP mem newv));
 9557   ins_cost(VOLATILE_REF_COST);
 9558   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9559   ins_encode %{
 9560     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9561   %}
 9562   ins_pipe(pipe_serial);
 9563 %}
 9564 
 9565 
 9566 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9567   match(Set newval (GetAndAddL mem incr));
 9568   ins_cost(2 * VOLATILE_REF_COST + 1);
 9569   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9570   ins_encode %{
 9571     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9572   %}
 9573   ins_pipe(pipe_serial);
 9574 %}
 9575 
 9576 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9577   predicate(n->as_LoadStore()->result_not_used());
 9578   match(Set dummy (GetAndAddL mem incr));
 9579   ins_cost(2 * VOLATILE_REF_COST);
 9580   format %{ "get_and_addL [$mem], $incr" %}
 9581   ins_encode %{
 9582     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9583   %}
 9584   ins_pipe(pipe_serial);
 9585 %}
 9586 
 9587 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9588   match(Set newval (GetAndAddL mem incr));
 9589   ins_cost(2 * VOLATILE_REF_COST + 1);
 9590   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9591   ins_encode %{
 9592     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9593   %}
 9594   ins_pipe(pipe_serial);
 9595 %}
 9596 
 9597 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9598   predicate(n->as_LoadStore()->result_not_used());
 9599   match(Set dummy (GetAndAddL mem incr));
 9600   ins_cost(2 * VOLATILE_REF_COST);
 9601   format %{ "get_and_addL [$mem], $incr" %}
 9602   ins_encode %{
 9603     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9604   %}
 9605   ins_pipe(pipe_serial);
 9606 %}
 9607 
 9608 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9609   match(Set newval (GetAndAddI mem incr));
 9610   ins_cost(2 * VOLATILE_REF_COST + 1);
 9611   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9612   ins_encode %{
 9613     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9614   %}
 9615   ins_pipe(pipe_serial);
 9616 %}
 9617 
 9618 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9619   predicate(n->as_LoadStore()->result_not_used());
 9620   match(Set dummy (GetAndAddI mem incr));
 9621   ins_cost(2 * VOLATILE_REF_COST);
 9622   format %{ "get_and_addI [$mem], $incr" %}
 9623   ins_encode %{
 9624     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9625   %}
 9626   ins_pipe(pipe_serial);
 9627 %}
 9628 
 9629 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9630   match(Set newval (GetAndAddI mem incr));
 9631   ins_cost(2 * VOLATILE_REF_COST + 1);
 9632   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9633   ins_encode %{
 9634     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9635   %}
 9636   ins_pipe(pipe_serial);
 9637 %}
 9638 
 9639 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9640   predicate(n->as_LoadStore()->result_not_used());
 9641   match(Set dummy (GetAndAddI mem incr));
 9642   ins_cost(2 * VOLATILE_REF_COST);
 9643   format %{ "get_and_addI [$mem], $incr" %}
 9644   ins_encode %{
 9645     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9646   %}
 9647   ins_pipe(pipe_serial);
 9648 %}
 9649 
 9650 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9651   predicate(needs_acquiring_load_exclusive(n));
 9652   match(Set newval (GetAndAddL mem incr));
 9653   ins_cost(VOLATILE_REF_COST + 1);
 9654   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9655   ins_encode %{
 9656     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9657   %}
 9658   ins_pipe(pipe_serial);
 9659 %}
 9660 
 9661 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9662   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9663   match(Set dummy (GetAndAddL mem incr));
 9664   ins_cost(VOLATILE_REF_COST);
 9665   format %{ "get_and_addL_acq [$mem], $incr" %}
 9666   ins_encode %{
 9667     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9668   %}
 9669   ins_pipe(pipe_serial);
 9670 %}
 9671 
 9672 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9673   predicate(needs_acquiring_load_exclusive(n));
 9674   match(Set newval (GetAndAddL mem incr));
 9675   ins_cost(VOLATILE_REF_COST + 1);
 9676   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9677   ins_encode %{
 9678     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9679   %}
 9680   ins_pipe(pipe_serial);
 9681 %}
 9682 
 9683 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9684   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9685   match(Set dummy (GetAndAddL mem incr));
 9686   ins_cost(VOLATILE_REF_COST);
 9687   format %{ "get_and_addL_acq [$mem], $incr" %}
 9688   ins_encode %{
 9689     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9690   %}
 9691   ins_pipe(pipe_serial);
 9692 %}
 9693 
 9694 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9695   predicate(needs_acquiring_load_exclusive(n));
 9696   match(Set newval (GetAndAddI mem incr));
 9697   ins_cost(VOLATILE_REF_COST + 1);
 9698   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9699   ins_encode %{
 9700     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9701   %}
 9702   ins_pipe(pipe_serial);
 9703 %}
 9704 
 9705 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9706   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9707   match(Set dummy (GetAndAddI mem incr));
 9708   ins_cost(VOLATILE_REF_COST);
 9709   format %{ "get_and_addI_acq [$mem], $incr" %}
 9710   ins_encode %{
 9711     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9712   %}
 9713   ins_pipe(pipe_serial);
 9714 %}
 9715 
 9716 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9717   predicate(needs_acquiring_load_exclusive(n));
 9718   match(Set newval (GetAndAddI mem incr));
 9719   ins_cost(VOLATILE_REF_COST + 1);
 9720   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9721   ins_encode %{
 9722     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9723   %}
 9724   ins_pipe(pipe_serial);
 9725 %}
 9726 
 9727 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9728   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9729   match(Set dummy (GetAndAddI mem incr));
 9730   ins_cost(VOLATILE_REF_COST);
 9731   format %{ "get_and_addI_acq [$mem], $incr" %}
 9732   ins_encode %{
 9733     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9734   %}
 9735   ins_pipe(pipe_serial);
 9736 %}
 9737 
 9738 // Manifest a CmpU result in an integer register.
 9739 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9740 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
 9741 %{
 9742   match(Set dst (CmpU3 src1 src2));
 9743   effect(KILL flags);
 9744 
 9745   ins_cost(INSN_COST * 3);
 9746   format %{
 9747       "cmpw $src1, $src2\n\t"
 9748       "csetw $dst, ne\n\t"
 9749       "cnegw $dst, lo\t# CmpU3(reg)"
 9750   %}
 9751   ins_encode %{
 9752     __ cmpw($src1$$Register, $src2$$Register);
 9753     __ csetw($dst$$Register, Assembler::NE);
 9754     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9755   %}
 9756 
 9757   ins_pipe(pipe_class_default);
 9758 %}
 9759 
 9760 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
 9761 %{
 9762   match(Set dst (CmpU3 src1 src2));
 9763   effect(KILL flags);
 9764 
 9765   ins_cost(INSN_COST * 3);
 9766   format %{
 9767       "subsw zr, $src1, $src2\n\t"
 9768       "csetw $dst, ne\n\t"
 9769       "cnegw $dst, lo\t# CmpU3(imm)"
 9770   %}
 9771   ins_encode %{
 9772     __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
 9773     __ csetw($dst$$Register, Assembler::NE);
 9774     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9775   %}
 9776 
 9777   ins_pipe(pipe_class_default);
 9778 %}
 9779 
 9780 // Manifest a CmpUL result in an integer register.
 9781 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9782 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9783 %{
 9784   match(Set dst (CmpUL3 src1 src2));
 9785   effect(KILL flags);
 9786 
 9787   ins_cost(INSN_COST * 3);
 9788   format %{
 9789       "cmp $src1, $src2\n\t"
 9790       "csetw $dst, ne\n\t"
 9791       "cnegw $dst, lo\t# CmpUL3(reg)"
 9792   %}
 9793   ins_encode %{
 9794     __ cmp($src1$$Register, $src2$$Register);
 9795     __ csetw($dst$$Register, Assembler::NE);
 9796     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9797   %}
 9798 
 9799   ins_pipe(pipe_class_default);
 9800 %}
 9801 
 9802 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9803 %{
 9804   match(Set dst (CmpUL3 src1 src2));
 9805   effect(KILL flags);
 9806 
 9807   ins_cost(INSN_COST * 3);
 9808   format %{
 9809       "subs zr, $src1, $src2\n\t"
 9810       "csetw $dst, ne\n\t"
 9811       "cnegw $dst, lo\t# CmpUL3(imm)"
 9812   %}
 9813   ins_encode %{
 9814     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9815     __ csetw($dst$$Register, Assembler::NE);
 9816     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9817   %}
 9818 
 9819   ins_pipe(pipe_class_default);
 9820 %}
 9821 
 9822 // Manifest a CmpL result in an integer register.
 9823 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9824 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9825 %{
 9826   match(Set dst (CmpL3 src1 src2));
 9827   effect(KILL flags);
 9828 
 9829   ins_cost(INSN_COST * 3);
 9830   format %{
 9831       "cmp $src1, $src2\n\t"
 9832       "csetw $dst, ne\n\t"
 9833       "cnegw $dst, lt\t# CmpL3(reg)"
 9834   %}
 9835   ins_encode %{
 9836     __ cmp($src1$$Register, $src2$$Register);
 9837     __ csetw($dst$$Register, Assembler::NE);
 9838     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9839   %}
 9840 
 9841   ins_pipe(pipe_class_default);
 9842 %}
 9843 
 9844 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9845 %{
 9846   match(Set dst (CmpL3 src1 src2));
 9847   effect(KILL flags);
 9848 
 9849   ins_cost(INSN_COST * 3);
 9850   format %{
 9851       "subs zr, $src1, $src2\n\t"
 9852       "csetw $dst, ne\n\t"
 9853       "cnegw $dst, lt\t# CmpL3(imm)"
 9854   %}
 9855   ins_encode %{
 9856     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9857     __ csetw($dst$$Register, Assembler::NE);
 9858     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9859   %}
 9860 
 9861   ins_pipe(pipe_class_default);
 9862 %}
 9863 
 9864 // ============================================================================
 9865 // Conditional Move Instructions
 9866 
 9867 // n.b. we have identical rules for both a signed compare op (cmpOp)
 9868 // and an unsigned compare op (cmpOpU). it would be nice if we could
 9869 // define an op class which merged both inputs and use it to type the
 9870 // argument to a single rule. unfortunatelyt his fails because the
 9871 // opclass does not live up to the COND_INTER interface of its
 9872 // component operands. When the generic code tries to negate the
 9873 // operand it ends up running the generci Machoper::negate method
 9874 // which throws a ShouldNotHappen. So, we have to provide two flavours
 9875 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
 9876 
 9877 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9878   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9879 
 9880   ins_cost(INSN_COST * 2);
 9881   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
 9882 
 9883   ins_encode %{
 9884     __ cselw(as_Register($dst$$reg),
 9885              as_Register($src2$$reg),
 9886              as_Register($src1$$reg),
 9887              (Assembler::Condition)$cmp$$cmpcode);
 9888   %}
 9889 
 9890   ins_pipe(icond_reg_reg);
 9891 %}
 9892 
 9893 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9894   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9895 
 9896   ins_cost(INSN_COST * 2);
 9897   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
 9898 
 9899   ins_encode %{
 9900     __ cselw(as_Register($dst$$reg),
 9901              as_Register($src2$$reg),
 9902              as_Register($src1$$reg),
 9903              (Assembler::Condition)$cmp$$cmpcode);
 9904   %}
 9905 
 9906   ins_pipe(icond_reg_reg);
 9907 %}
 9908 
 9909 // special cases where one arg is zero
 9910 
 9911 // n.b. this is selected in preference to the rule above because it
 9912 // avoids loading constant 0 into a source register
 9913 
 9914 // TODO
 9915 // we ought only to be able to cull one of these variants as the ideal
 9916 // transforms ought always to order the zero consistently (to left/right?)
 9917 
 9918 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9919   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9920 
 9921   ins_cost(INSN_COST * 2);
 9922   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
 9923 
 9924   ins_encode %{
 9925     __ cselw(as_Register($dst$$reg),
 9926              as_Register($src$$reg),
 9927              zr,
 9928              (Assembler::Condition)$cmp$$cmpcode);
 9929   %}
 9930 
 9931   ins_pipe(icond_reg);
 9932 %}
 9933 
 9934 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9935   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9936 
 9937   ins_cost(INSN_COST * 2);
 9938   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
 9939 
 9940   ins_encode %{
 9941     __ cselw(as_Register($dst$$reg),
 9942              as_Register($src$$reg),
 9943              zr,
 9944              (Assembler::Condition)$cmp$$cmpcode);
 9945   %}
 9946 
 9947   ins_pipe(icond_reg);
 9948 %}
 9949 
 9950 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9951   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9952 
 9953   ins_cost(INSN_COST * 2);
 9954   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
 9955 
 9956   ins_encode %{
 9957     __ cselw(as_Register($dst$$reg),
 9958              zr,
 9959              as_Register($src$$reg),
 9960              (Assembler::Condition)$cmp$$cmpcode);
 9961   %}
 9962 
 9963   ins_pipe(icond_reg);
 9964 %}
 9965 
 9966 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9967   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9968 
 9969   ins_cost(INSN_COST * 2);
 9970   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
 9971 
 9972   ins_encode %{
 9973     __ cselw(as_Register($dst$$reg),
 9974              zr,
 9975              as_Register($src$$reg),
 9976              (Assembler::Condition)$cmp$$cmpcode);
 9977   %}
 9978 
 9979   ins_pipe(icond_reg);
 9980 %}
 9981 
 9982 // special case for creating a boolean 0 or 1
 9983 
 9984 // n.b. this is selected in preference to the rule above because it
 9985 // avoids loading constants 0 and 1 into a source register
 9986 
 9987 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9988   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9989 
 9990   ins_cost(INSN_COST * 2);
 9991   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
 9992 
 9993   ins_encode %{
 9994     // equivalently
 9995     // cset(as_Register($dst$$reg),
 9996     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9997     __ csincw(as_Register($dst$$reg),
 9998              zr,
 9999              zr,
10000              (Assembler::Condition)$cmp$$cmpcode);
10001   %}
10002 
10003   ins_pipe(icond_none);
10004 %}
10005 
10006 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10007   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10008 
10009   ins_cost(INSN_COST * 2);
10010   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
10011 
10012   ins_encode %{
10013     // equivalently
10014     // cset(as_Register($dst$$reg),
10015     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10016     __ csincw(as_Register($dst$$reg),
10017              zr,
10018              zr,
10019              (Assembler::Condition)$cmp$$cmpcode);
10020   %}
10021 
10022   ins_pipe(icond_none);
10023 %}
10024 
10025 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10026   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10027 
10028   ins_cost(INSN_COST * 2);
10029   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
10030 
10031   ins_encode %{
10032     __ csel(as_Register($dst$$reg),
10033             as_Register($src2$$reg),
10034             as_Register($src1$$reg),
10035             (Assembler::Condition)$cmp$$cmpcode);
10036   %}
10037 
10038   ins_pipe(icond_reg_reg);
10039 %}
10040 
10041 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10042   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10043 
10044   ins_cost(INSN_COST * 2);
10045   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
10046 
10047   ins_encode %{
10048     __ csel(as_Register($dst$$reg),
10049             as_Register($src2$$reg),
10050             as_Register($src1$$reg),
10051             (Assembler::Condition)$cmp$$cmpcode);
10052   %}
10053 
10054   ins_pipe(icond_reg_reg);
10055 %}
10056 
10057 // special cases where one arg is zero
10058 
10059 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10060   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10061 
10062   ins_cost(INSN_COST * 2);
10063   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
10064 
10065   ins_encode %{
10066     __ csel(as_Register($dst$$reg),
10067             zr,
10068             as_Register($src$$reg),
10069             (Assembler::Condition)$cmp$$cmpcode);
10070   %}
10071 
10072   ins_pipe(icond_reg);
10073 %}
10074 
10075 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10076   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10077 
10078   ins_cost(INSN_COST * 2);
10079   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10080 
10081   ins_encode %{
10082     __ csel(as_Register($dst$$reg),
10083             zr,
10084             as_Register($src$$reg),
10085             (Assembler::Condition)$cmp$$cmpcode);
10086   %}
10087 
10088   ins_pipe(icond_reg);
10089 %}
10090 
10091 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10092   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10093 
10094   ins_cost(INSN_COST * 2);
10095   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10096 
10097   ins_encode %{
10098     __ csel(as_Register($dst$$reg),
10099             as_Register($src$$reg),
10100             zr,
10101             (Assembler::Condition)$cmp$$cmpcode);
10102   %}
10103 
10104   ins_pipe(icond_reg);
10105 %}
10106 
10107 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10108   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10109 
10110   ins_cost(INSN_COST * 2);
10111   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10112 
10113   ins_encode %{
10114     __ csel(as_Register($dst$$reg),
10115             as_Register($src$$reg),
10116             zr,
10117             (Assembler::Condition)$cmp$$cmpcode);
10118   %}
10119 
10120   ins_pipe(icond_reg);
10121 %}
10122 
10123 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10124   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10125 
10126   ins_cost(INSN_COST * 2);
10127   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10128 
10129   ins_encode %{
10130     __ csel(as_Register($dst$$reg),
10131             as_Register($src2$$reg),
10132             as_Register($src1$$reg),
10133             (Assembler::Condition)$cmp$$cmpcode);
10134   %}
10135 
10136   ins_pipe(icond_reg_reg);
10137 %}
10138 
10139 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10140   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10141 
10142   ins_cost(INSN_COST * 2);
10143   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10144 
10145   ins_encode %{
10146     __ csel(as_Register($dst$$reg),
10147             as_Register($src2$$reg),
10148             as_Register($src1$$reg),
10149             (Assembler::Condition)$cmp$$cmpcode);
10150   %}
10151 
10152   ins_pipe(icond_reg_reg);
10153 %}
10154 
10155 // special cases where one arg is zero
10156 
10157 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10158   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10159 
10160   ins_cost(INSN_COST * 2);
10161   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10162 
10163   ins_encode %{
10164     __ csel(as_Register($dst$$reg),
10165             zr,
10166             as_Register($src$$reg),
10167             (Assembler::Condition)$cmp$$cmpcode);
10168   %}
10169 
10170   ins_pipe(icond_reg);
10171 %}
10172 
10173 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10174   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10175 
10176   ins_cost(INSN_COST * 2);
10177   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10178 
10179   ins_encode %{
10180     __ csel(as_Register($dst$$reg),
10181             zr,
10182             as_Register($src$$reg),
10183             (Assembler::Condition)$cmp$$cmpcode);
10184   %}
10185 
10186   ins_pipe(icond_reg);
10187 %}
10188 
10189 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10190   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10191 
10192   ins_cost(INSN_COST * 2);
10193   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10194 
10195   ins_encode %{
10196     __ csel(as_Register($dst$$reg),
10197             as_Register($src$$reg),
10198             zr,
10199             (Assembler::Condition)$cmp$$cmpcode);
10200   %}
10201 
10202   ins_pipe(icond_reg);
10203 %}
10204 
10205 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10206   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10207 
10208   ins_cost(INSN_COST * 2);
10209   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10210 
10211   ins_encode %{
10212     __ csel(as_Register($dst$$reg),
10213             as_Register($src$$reg),
10214             zr,
10215             (Assembler::Condition)$cmp$$cmpcode);
10216   %}
10217 
10218   ins_pipe(icond_reg);
10219 %}
10220 
10221 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10222   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10223 
10224   ins_cost(INSN_COST * 2);
10225   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10226 
10227   ins_encode %{
10228     __ cselw(as_Register($dst$$reg),
10229              as_Register($src2$$reg),
10230              as_Register($src1$$reg),
10231              (Assembler::Condition)$cmp$$cmpcode);
10232   %}
10233 
10234   ins_pipe(icond_reg_reg);
10235 %}
10236 
10237 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10238   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10239 
10240   ins_cost(INSN_COST * 2);
10241   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10242 
10243   ins_encode %{
10244     __ cselw(as_Register($dst$$reg),
10245              as_Register($src2$$reg),
10246              as_Register($src1$$reg),
10247              (Assembler::Condition)$cmp$$cmpcode);
10248   %}
10249 
10250   ins_pipe(icond_reg_reg);
10251 %}
10252 
10253 // special cases where one arg is zero
10254 
10255 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10256   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10257 
10258   ins_cost(INSN_COST * 2);
10259   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10260 
10261   ins_encode %{
10262     __ cselw(as_Register($dst$$reg),
10263              zr,
10264              as_Register($src$$reg),
10265              (Assembler::Condition)$cmp$$cmpcode);
10266   %}
10267 
10268   ins_pipe(icond_reg);
10269 %}
10270 
10271 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10272   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10273 
10274   ins_cost(INSN_COST * 2);
10275   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10276 
10277   ins_encode %{
10278     __ cselw(as_Register($dst$$reg),
10279              zr,
10280              as_Register($src$$reg),
10281              (Assembler::Condition)$cmp$$cmpcode);
10282   %}
10283 
10284   ins_pipe(icond_reg);
10285 %}
10286 
10287 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10288   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10289 
10290   ins_cost(INSN_COST * 2);
10291   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10292 
10293   ins_encode %{
10294     __ cselw(as_Register($dst$$reg),
10295              as_Register($src$$reg),
10296              zr,
10297              (Assembler::Condition)$cmp$$cmpcode);
10298   %}
10299 
10300   ins_pipe(icond_reg);
10301 %}
10302 
10303 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10304   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10305 
10306   ins_cost(INSN_COST * 2);
10307   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10308 
10309   ins_encode %{
10310     __ cselw(as_Register($dst$$reg),
10311              as_Register($src$$reg),
10312              zr,
10313              (Assembler::Condition)$cmp$$cmpcode);
10314   %}
10315 
10316   ins_pipe(icond_reg);
10317 %}
10318 
10319 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10320 %{
10321   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10322 
10323   ins_cost(INSN_COST * 3);
10324 
10325   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10326   ins_encode %{
10327     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10328     __ fcsels(as_FloatRegister($dst$$reg),
10329               as_FloatRegister($src2$$reg),
10330               as_FloatRegister($src1$$reg),
10331               cond);
10332   %}
10333 
10334   ins_pipe(fp_cond_reg_reg_s);
10335 %}
10336 
10337 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10338 %{
10339   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10340 
10341   ins_cost(INSN_COST * 3);
10342 
10343   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10344   ins_encode %{
10345     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10346     __ fcsels(as_FloatRegister($dst$$reg),
10347               as_FloatRegister($src2$$reg),
10348               as_FloatRegister($src1$$reg),
10349               cond);
10350   %}
10351 
10352   ins_pipe(fp_cond_reg_reg_s);
10353 %}
10354 
10355 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10356 %{
10357   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10358 
10359   ins_cost(INSN_COST * 3);
10360 
10361   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10362   ins_encode %{
10363     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10364     __ fcseld(as_FloatRegister($dst$$reg),
10365               as_FloatRegister($src2$$reg),
10366               as_FloatRegister($src1$$reg),
10367               cond);
10368   %}
10369 
10370   ins_pipe(fp_cond_reg_reg_d);
10371 %}
10372 
10373 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10374 %{
10375   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10376 
10377   ins_cost(INSN_COST * 3);
10378 
10379   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10380   ins_encode %{
10381     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10382     __ fcseld(as_FloatRegister($dst$$reg),
10383               as_FloatRegister($src2$$reg),
10384               as_FloatRegister($src1$$reg),
10385               cond);
10386   %}
10387 
10388   ins_pipe(fp_cond_reg_reg_d);
10389 %}
10390 
10391 // ============================================================================
10392 // Arithmetic Instructions
10393 //
10394 
10395 // Integer Addition
10396 
10397 // TODO
10398 // these currently employ operations which do not set CR and hence are
10399 // not flagged as killing CR but we would like to isolate the cases
10400 // where we want to set flags from those where we don't. need to work
10401 // out how to do that.
10402 
10403 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10404   match(Set dst (AddI src1 src2));
10405 
10406   ins_cost(INSN_COST);
10407   format %{ "addw  $dst, $src1, $src2" %}
10408 
10409   ins_encode %{
10410     __ addw(as_Register($dst$$reg),
10411             as_Register($src1$$reg),
10412             as_Register($src2$$reg));
10413   %}
10414 
10415   ins_pipe(ialu_reg_reg);
10416 %}
10417 
10418 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10419   match(Set dst (AddI src1 src2));
10420 
10421   ins_cost(INSN_COST);
10422   format %{ "addw $dst, $src1, $src2" %}
10423 
10424   // use opcode to indicate that this is an add not a sub
10425   opcode(0x0);
10426 
10427   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10428 
10429   ins_pipe(ialu_reg_imm);
10430 %}
10431 
10432 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10433   match(Set dst (AddI (ConvL2I src1) src2));
10434 
10435   ins_cost(INSN_COST);
10436   format %{ "addw $dst, $src1, $src2" %}
10437 
10438   // use opcode to indicate that this is an add not a sub
10439   opcode(0x0);
10440 
10441   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10442 
10443   ins_pipe(ialu_reg_imm);
10444 %}
10445 
10446 // Pointer Addition
10447 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10448   match(Set dst (AddP src1 src2));
10449 
10450   ins_cost(INSN_COST);
10451   format %{ "add $dst, $src1, $src2\t# ptr" %}
10452 
10453   ins_encode %{
10454     __ add(as_Register($dst$$reg),
10455            as_Register($src1$$reg),
10456            as_Register($src2$$reg));
10457   %}
10458 
10459   ins_pipe(ialu_reg_reg);
10460 %}
10461 
10462 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10463   match(Set dst (AddP src1 (ConvI2L src2)));
10464 
10465   ins_cost(1.9 * INSN_COST);
10466   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10467 
10468   ins_encode %{
10469     __ add(as_Register($dst$$reg),
10470            as_Register($src1$$reg),
10471            as_Register($src2$$reg), ext::sxtw);
10472   %}
10473 
10474   ins_pipe(ialu_reg_reg);
10475 %}
10476 
10477 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10478   match(Set dst (AddP src1 (LShiftL src2 scale)));
10479 
10480   ins_cost(1.9 * INSN_COST);
10481   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10482 
10483   ins_encode %{
10484     __ lea(as_Register($dst$$reg),
10485            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10486                    Address::lsl($scale$$constant)));
10487   %}
10488 
10489   ins_pipe(ialu_reg_reg_shift);
10490 %}
10491 
10492 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10493   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10494 
10495   ins_cost(1.9 * INSN_COST);
10496   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10497 
10498   ins_encode %{
10499     __ lea(as_Register($dst$$reg),
10500            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10501                    Address::sxtw($scale$$constant)));
10502   %}
10503 
10504   ins_pipe(ialu_reg_reg_shift);
10505 %}
10506 
10507 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10508   match(Set dst (LShiftL (ConvI2L src) scale));
10509 
10510   ins_cost(INSN_COST);
10511   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10512 
10513   ins_encode %{
10514     __ sbfiz(as_Register($dst$$reg),
10515           as_Register($src$$reg),
10516           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10517   %}
10518 
10519   ins_pipe(ialu_reg_shift);
10520 %}
10521 
10522 // Pointer Immediate Addition
10523 // n.b. this needs to be more expensive than using an indirect memory
10524 // operand
10525 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10526   match(Set dst (AddP src1 src2));
10527 
10528   ins_cost(INSN_COST);
10529   format %{ "add $dst, $src1, $src2\t# ptr" %}
10530 
10531   // use opcode to indicate that this is an add not a sub
10532   opcode(0x0);
10533 
10534   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10535 
10536   ins_pipe(ialu_reg_imm);
10537 %}
10538 
10539 // Long Addition
10540 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10541 
10542   match(Set dst (AddL src1 src2));
10543 
10544   ins_cost(INSN_COST);
10545   format %{ "add  $dst, $src1, $src2" %}
10546 
10547   ins_encode %{
10548     __ add(as_Register($dst$$reg),
10549            as_Register($src1$$reg),
10550            as_Register($src2$$reg));
10551   %}
10552 
10553   ins_pipe(ialu_reg_reg);
10554 %}
10555 
10556 // No constant pool entries requiredLong Immediate Addition.
10557 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10558   match(Set dst (AddL src1 src2));
10559 
10560   ins_cost(INSN_COST);
10561   format %{ "add $dst, $src1, $src2" %}
10562 
10563   // use opcode to indicate that this is an add not a sub
10564   opcode(0x0);
10565 
10566   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10567 
10568   ins_pipe(ialu_reg_imm);
10569 %}
10570 
10571 // Integer Subtraction
10572 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10573   match(Set dst (SubI src1 src2));
10574 
10575   ins_cost(INSN_COST);
10576   format %{ "subw  $dst, $src1, $src2" %}
10577 
10578   ins_encode %{
10579     __ subw(as_Register($dst$$reg),
10580             as_Register($src1$$reg),
10581             as_Register($src2$$reg));
10582   %}
10583 
10584   ins_pipe(ialu_reg_reg);
10585 %}
10586 
10587 // Immediate Subtraction
10588 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10589   match(Set dst (SubI src1 src2));
10590 
10591   ins_cost(INSN_COST);
10592   format %{ "subw $dst, $src1, $src2" %}
10593 
10594   // use opcode to indicate that this is a sub not an add
10595   opcode(0x1);
10596 
10597   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10598 
10599   ins_pipe(ialu_reg_imm);
10600 %}
10601 
10602 // Long Subtraction
10603 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10604 
10605   match(Set dst (SubL src1 src2));
10606 
10607   ins_cost(INSN_COST);
10608   format %{ "sub  $dst, $src1, $src2" %}
10609 
10610   ins_encode %{
10611     __ sub(as_Register($dst$$reg),
10612            as_Register($src1$$reg),
10613            as_Register($src2$$reg));
10614   %}
10615 
10616   ins_pipe(ialu_reg_reg);
10617 %}
10618 
10619 // No constant pool entries requiredLong Immediate Subtraction.
10620 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10621   match(Set dst (SubL src1 src2));
10622 
10623   ins_cost(INSN_COST);
10624   format %{ "sub$dst, $src1, $src2" %}
10625 
10626   // use opcode to indicate that this is a sub not an add
10627   opcode(0x1);
10628 
10629   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10630 
10631   ins_pipe(ialu_reg_imm);
10632 %}
10633 
10634 // Integer Negation (special case for sub)
10635 
10636 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10637   match(Set dst (SubI zero src));
10638 
10639   ins_cost(INSN_COST);
10640   format %{ "negw $dst, $src\t# int" %}
10641 
10642   ins_encode %{
10643     __ negw(as_Register($dst$$reg),
10644             as_Register($src$$reg));
10645   %}
10646 
10647   ins_pipe(ialu_reg);
10648 %}
10649 
10650 // Long Negation
10651 
10652 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10653   match(Set dst (SubL zero src));
10654 
10655   ins_cost(INSN_COST);
10656   format %{ "neg $dst, $src\t# long" %}
10657 
10658   ins_encode %{
10659     __ neg(as_Register($dst$$reg),
10660            as_Register($src$$reg));
10661   %}
10662 
10663   ins_pipe(ialu_reg);
10664 %}
10665 
10666 // Integer Multiply
10667 
10668 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10669   match(Set dst (MulI src1 src2));
10670 
10671   ins_cost(INSN_COST * 3);
10672   format %{ "mulw  $dst, $src1, $src2" %}
10673 
10674   ins_encode %{
10675     __ mulw(as_Register($dst$$reg),
10676             as_Register($src1$$reg),
10677             as_Register($src2$$reg));
10678   %}
10679 
10680   ins_pipe(imul_reg_reg);
10681 %}
10682 
10683 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10684   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10685 
10686   ins_cost(INSN_COST * 3);
10687   format %{ "smull  $dst, $src1, $src2" %}
10688 
10689   ins_encode %{
10690     __ smull(as_Register($dst$$reg),
10691              as_Register($src1$$reg),
10692              as_Register($src2$$reg));
10693   %}
10694 
10695   ins_pipe(imul_reg_reg);
10696 %}
10697 
10698 // Long Multiply
10699 
10700 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10701   match(Set dst (MulL src1 src2));
10702 
10703   ins_cost(INSN_COST * 5);
10704   format %{ "mul  $dst, $src1, $src2" %}
10705 
10706   ins_encode %{
10707     __ mul(as_Register($dst$$reg),
10708            as_Register($src1$$reg),
10709            as_Register($src2$$reg));
10710   %}
10711 
10712   ins_pipe(lmul_reg_reg);
10713 %}
10714 
10715 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10716 %{
10717   match(Set dst (MulHiL src1 src2));
10718 
10719   ins_cost(INSN_COST * 7);
10720   format %{ "smulh   $dst, $src1, $src2\t# mulhi" %}
10721 
10722   ins_encode %{
10723     __ smulh(as_Register($dst$$reg),
10724              as_Register($src1$$reg),
10725              as_Register($src2$$reg));
10726   %}
10727 
10728   ins_pipe(lmul_reg_reg);
10729 %}
10730 
10731 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10732 %{
10733   match(Set dst (UMulHiL src1 src2));
10734 
10735   ins_cost(INSN_COST * 7);
10736   format %{ "umulh   $dst, $src1, $src2\t# umulhi" %}
10737 
10738   ins_encode %{
10739     __ umulh(as_Register($dst$$reg),
10740              as_Register($src1$$reg),
10741              as_Register($src2$$reg));
10742   %}
10743 
10744   ins_pipe(lmul_reg_reg);
10745 %}
10746 
10747 // Combined Integer Multiply & Add/Sub
10748 
10749 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10750   match(Set dst (AddI src3 (MulI src1 src2)));
10751 
10752   ins_cost(INSN_COST * 3);
10753   format %{ "madd  $dst, $src1, $src2, $src3" %}
10754 
10755   ins_encode %{
10756     __ maddw(as_Register($dst$$reg),
10757              as_Register($src1$$reg),
10758              as_Register($src2$$reg),
10759              as_Register($src3$$reg));
10760   %}
10761 
10762   ins_pipe(imac_reg_reg);
10763 %}
10764 
10765 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10766   match(Set dst (SubI src3 (MulI src1 src2)));
10767 
10768   ins_cost(INSN_COST * 3);
10769   format %{ "msub  $dst, $src1, $src2, $src3" %}
10770 
10771   ins_encode %{
10772     __ msubw(as_Register($dst$$reg),
10773              as_Register($src1$$reg),
10774              as_Register($src2$$reg),
10775              as_Register($src3$$reg));
10776   %}
10777 
10778   ins_pipe(imac_reg_reg);
10779 %}
10780 
10781 // Combined Integer Multiply & Neg
10782 
10783 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10784   match(Set dst (MulI (SubI zero src1) src2));
10785 
10786   ins_cost(INSN_COST * 3);
10787   format %{ "mneg  $dst, $src1, $src2" %}
10788 
10789   ins_encode %{
10790     __ mnegw(as_Register($dst$$reg),
10791              as_Register($src1$$reg),
10792              as_Register($src2$$reg));
10793   %}
10794 
10795   ins_pipe(imac_reg_reg);
10796 %}
10797 
10798 // Combined Long Multiply & Add/Sub
10799 
10800 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10801   match(Set dst (AddL src3 (MulL src1 src2)));
10802 
10803   ins_cost(INSN_COST * 5);
10804   format %{ "madd  $dst, $src1, $src2, $src3" %}
10805 
10806   ins_encode %{
10807     __ madd(as_Register($dst$$reg),
10808             as_Register($src1$$reg),
10809             as_Register($src2$$reg),
10810             as_Register($src3$$reg));
10811   %}
10812 
10813   ins_pipe(lmac_reg_reg);
10814 %}
10815 
10816 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10817   match(Set dst (SubL src3 (MulL src1 src2)));
10818 
10819   ins_cost(INSN_COST * 5);
10820   format %{ "msub  $dst, $src1, $src2, $src3" %}
10821 
10822   ins_encode %{
10823     __ msub(as_Register($dst$$reg),
10824             as_Register($src1$$reg),
10825             as_Register($src2$$reg),
10826             as_Register($src3$$reg));
10827   %}
10828 
10829   ins_pipe(lmac_reg_reg);
10830 %}
10831 
10832 // Combined Long Multiply & Neg
10833 
10834 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10835   match(Set dst (MulL (SubL zero src1) src2));
10836 
10837   ins_cost(INSN_COST * 5);
10838   format %{ "mneg  $dst, $src1, $src2" %}
10839 
10840   ins_encode %{
10841     __ mneg(as_Register($dst$$reg),
10842             as_Register($src1$$reg),
10843             as_Register($src2$$reg));
10844   %}
10845 
10846   ins_pipe(lmac_reg_reg);
10847 %}
10848 
10849 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10850 
10851 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10852   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10853 
10854   ins_cost(INSN_COST * 3);
10855   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10856 
10857   ins_encode %{
10858     __ smaddl(as_Register($dst$$reg),
10859               as_Register($src1$$reg),
10860               as_Register($src2$$reg),
10861               as_Register($src3$$reg));
10862   %}
10863 
10864   ins_pipe(imac_reg_reg);
10865 %}
10866 
10867 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10868   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10869 
10870   ins_cost(INSN_COST * 3);
10871   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10872 
10873   ins_encode %{
10874     __ smsubl(as_Register($dst$$reg),
10875               as_Register($src1$$reg),
10876               as_Register($src2$$reg),
10877               as_Register($src3$$reg));
10878   %}
10879 
10880   ins_pipe(imac_reg_reg);
10881 %}
10882 
10883 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10884   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10885 
10886   ins_cost(INSN_COST * 3);
10887   format %{ "smnegl  $dst, $src1, $src2" %}
10888 
10889   ins_encode %{
10890     __ smnegl(as_Register($dst$$reg),
10891               as_Register($src1$$reg),
10892               as_Register($src2$$reg));
10893   %}
10894 
10895   ins_pipe(imac_reg_reg);
10896 %}
10897 
10898 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10899 
10900 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10901   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10902 
10903   ins_cost(INSN_COST * 5);
10904   format %{ "mulw  rscratch1, $src1, $src2\n\t"
10905             "maddw $dst, $src3, $src4, rscratch1" %}
10906 
10907   ins_encode %{
10908     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10909     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10910 
10911   ins_pipe(imac_reg_reg);
10912 %}
10913 
10914 // Integer Divide
10915 
10916 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10917   match(Set dst (DivI src1 src2));
10918 
10919   ins_cost(INSN_COST * 19);
10920   format %{ "sdivw  $dst, $src1, $src2" %}
10921 
10922   ins_encode(aarch64_enc_divw(dst, src1, src2));
10923   ins_pipe(idiv_reg_reg);
10924 %}
10925 
10926 // Long Divide
10927 
10928 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10929   match(Set dst (DivL src1 src2));
10930 
10931   ins_cost(INSN_COST * 35);
10932   format %{ "sdiv   $dst, $src1, $src2" %}
10933 
10934   ins_encode(aarch64_enc_div(dst, src1, src2));
10935   ins_pipe(ldiv_reg_reg);
10936 %}
10937 
10938 // Integer Remainder
10939 
10940 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10941   match(Set dst (ModI src1 src2));
10942 
10943   ins_cost(INSN_COST * 22);
10944   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
10945             "msubw  $dst, rscratch1, $src2, $src1" %}
10946 
10947   ins_encode(aarch64_enc_modw(dst, src1, src2));
10948   ins_pipe(idiv_reg_reg);
10949 %}
10950 
10951 // Long Remainder
10952 
10953 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10954   match(Set dst (ModL src1 src2));
10955 
10956   ins_cost(INSN_COST * 38);
10957   format %{ "sdiv   rscratch1, $src1, $src2\n"
10958             "msub   $dst, rscratch1, $src2, $src1" %}
10959 
10960   ins_encode(aarch64_enc_mod(dst, src1, src2));
10961   ins_pipe(ldiv_reg_reg);
10962 %}
10963 
10964 // Unsigned Integer Divide
10965 
10966 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10967   match(Set dst (UDivI src1 src2));
10968 
10969   ins_cost(INSN_COST * 19);
10970   format %{ "udivw  $dst, $src1, $src2" %}
10971 
10972   ins_encode %{
10973     __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10974   %}
10975 
10976   ins_pipe(idiv_reg_reg);
10977 %}
10978 
10979 //  Unsigned Long Divide
10980 
10981 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10982   match(Set dst (UDivL src1 src2));
10983 
10984   ins_cost(INSN_COST * 35);
10985   format %{ "udiv   $dst, $src1, $src2" %}
10986 
10987   ins_encode %{
10988     __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10989   %}
10990 
10991   ins_pipe(ldiv_reg_reg);
10992 %}
10993 
10994 // Unsigned Integer Remainder
10995 
10996 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10997   match(Set dst (UModI src1 src2));
10998 
10999   ins_cost(INSN_COST * 22);
11000   format %{ "udivw  rscratch1, $src1, $src2\n\t"
11001             "msubw  $dst, rscratch1, $src2, $src1" %}
11002 
11003   ins_encode %{
11004     __ udivw(rscratch1, $src1$$Register, $src2$$Register);
11005     __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
11006   %}
11007 
11008   ins_pipe(idiv_reg_reg);
11009 %}
11010 
11011 // Unsigned Long Remainder
11012 
11013 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11014   match(Set dst (UModL src1 src2));
11015 
11016   ins_cost(INSN_COST * 38);
11017   format %{ "udiv   rscratch1, $src1, $src2\n"
11018             "msub   $dst, rscratch1, $src2, $src1" %}
11019 
11020   ins_encode %{
11021     __ udiv(rscratch1, $src1$$Register, $src2$$Register);
11022     __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
11023   %}
11024 
11025   ins_pipe(ldiv_reg_reg);
11026 %}
11027 
11028 // Integer Shifts
11029 
11030 // Shift Left Register
11031 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11032   match(Set dst (LShiftI src1 src2));
11033 
11034   ins_cost(INSN_COST * 2);
11035   format %{ "lslvw  $dst, $src1, $src2" %}
11036 
11037   ins_encode %{
11038     __ lslvw(as_Register($dst$$reg),
11039              as_Register($src1$$reg),
11040              as_Register($src2$$reg));
11041   %}
11042 
11043   ins_pipe(ialu_reg_reg_vshift);
11044 %}
11045 
11046 // Shift Left Immediate
11047 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11048   match(Set dst (LShiftI src1 src2));
11049 
11050   ins_cost(INSN_COST);
11051   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11052 
11053   ins_encode %{
11054     __ lslw(as_Register($dst$$reg),
11055             as_Register($src1$$reg),
11056             $src2$$constant & 0x1f);
11057   %}
11058 
11059   ins_pipe(ialu_reg_shift);
11060 %}
11061 
11062 // Shift Right Logical Register
11063 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11064   match(Set dst (URShiftI src1 src2));
11065 
11066   ins_cost(INSN_COST * 2);
11067   format %{ "lsrvw  $dst, $src1, $src2" %}
11068 
11069   ins_encode %{
11070     __ lsrvw(as_Register($dst$$reg),
11071              as_Register($src1$$reg),
11072              as_Register($src2$$reg));
11073   %}
11074 
11075   ins_pipe(ialu_reg_reg_vshift);
11076 %}
11077 
11078 // Shift Right Logical Immediate
11079 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11080   match(Set dst (URShiftI src1 src2));
11081 
11082   ins_cost(INSN_COST);
11083   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11084 
11085   ins_encode %{
11086     __ lsrw(as_Register($dst$$reg),
11087             as_Register($src1$$reg),
11088             $src2$$constant & 0x1f);
11089   %}
11090 
11091   ins_pipe(ialu_reg_shift);
11092 %}
11093 
11094 // Shift Right Arithmetic Register
11095 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11096   match(Set dst (RShiftI src1 src2));
11097 
11098   ins_cost(INSN_COST * 2);
11099   format %{ "asrvw  $dst, $src1, $src2" %}
11100 
11101   ins_encode %{
11102     __ asrvw(as_Register($dst$$reg),
11103              as_Register($src1$$reg),
11104              as_Register($src2$$reg));
11105   %}
11106 
11107   ins_pipe(ialu_reg_reg_vshift);
11108 %}
11109 
11110 // Shift Right Arithmetic Immediate
11111 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11112   match(Set dst (RShiftI src1 src2));
11113 
11114   ins_cost(INSN_COST);
11115   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11116 
11117   ins_encode %{
11118     __ asrw(as_Register($dst$$reg),
11119             as_Register($src1$$reg),
11120             $src2$$constant & 0x1f);
11121   %}
11122 
11123   ins_pipe(ialu_reg_shift);
11124 %}
11125 
11126 // Combined Int Mask and Right Shift (using UBFM)
11127 // TODO
11128 
11129 // Long Shifts
11130 
11131 // Shift Left Register
11132 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11133   match(Set dst (LShiftL src1 src2));
11134 
11135   ins_cost(INSN_COST * 2);
11136   format %{ "lslv  $dst, $src1, $src2" %}
11137 
11138   ins_encode %{
11139     __ lslv(as_Register($dst$$reg),
11140             as_Register($src1$$reg),
11141             as_Register($src2$$reg));
11142   %}
11143 
11144   ins_pipe(ialu_reg_reg_vshift);
11145 %}
11146 
11147 // Shift Left Immediate
11148 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11149   match(Set dst (LShiftL src1 src2));
11150 
11151   ins_cost(INSN_COST);
11152   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11153 
11154   ins_encode %{
11155     __ lsl(as_Register($dst$$reg),
11156             as_Register($src1$$reg),
11157             $src2$$constant & 0x3f);
11158   %}
11159 
11160   ins_pipe(ialu_reg_shift);
11161 %}
11162 
11163 // Shift Right Logical Register
11164 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11165   match(Set dst (URShiftL src1 src2));
11166 
11167   ins_cost(INSN_COST * 2);
11168   format %{ "lsrv  $dst, $src1, $src2" %}
11169 
11170   ins_encode %{
11171     __ lsrv(as_Register($dst$$reg),
11172             as_Register($src1$$reg),
11173             as_Register($src2$$reg));
11174   %}
11175 
11176   ins_pipe(ialu_reg_reg_vshift);
11177 %}
11178 
11179 // Shift Right Logical Immediate
11180 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11181   match(Set dst (URShiftL src1 src2));
11182 
11183   ins_cost(INSN_COST);
11184   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11185 
11186   ins_encode %{
11187     __ lsr(as_Register($dst$$reg),
11188            as_Register($src1$$reg),
11189            $src2$$constant & 0x3f);
11190   %}
11191 
11192   ins_pipe(ialu_reg_shift);
11193 %}
11194 
11195 // A special-case pattern for card table stores.
11196 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11197   match(Set dst (URShiftL (CastP2X src1) src2));
11198 
11199   ins_cost(INSN_COST);
11200   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11201 
11202   ins_encode %{
11203     __ lsr(as_Register($dst$$reg),
11204            as_Register($src1$$reg),
11205            $src2$$constant & 0x3f);
11206   %}
11207 
11208   ins_pipe(ialu_reg_shift);
11209 %}
11210 
11211 // Shift Right Arithmetic Register
11212 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11213   match(Set dst (RShiftL src1 src2));
11214 
11215   ins_cost(INSN_COST * 2);
11216   format %{ "asrv  $dst, $src1, $src2" %}
11217 
11218   ins_encode %{
11219     __ asrv(as_Register($dst$$reg),
11220             as_Register($src1$$reg),
11221             as_Register($src2$$reg));
11222   %}
11223 
11224   ins_pipe(ialu_reg_reg_vshift);
11225 %}
11226 
11227 // Shift Right Arithmetic Immediate
11228 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11229   match(Set dst (RShiftL src1 src2));
11230 
11231   ins_cost(INSN_COST);
11232   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11233 
11234   ins_encode %{
11235     __ asr(as_Register($dst$$reg),
11236            as_Register($src1$$reg),
11237            $src2$$constant & 0x3f);
11238   %}
11239 
11240   ins_pipe(ialu_reg_shift);
11241 %}
11242 
11243 // BEGIN This section of the file is automatically generated. Do not edit --------------
11244 // This section is generated from aarch64_ad.m4
11245 
11246 // This pattern is automatically generated from aarch64_ad.m4
11247 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11248 instruct regL_not_reg(iRegLNoSp dst,
11249                          iRegL src1, immL_M1 m1,
11250                          rFlagsReg cr) %{
11251   match(Set dst (XorL src1 m1));
11252   ins_cost(INSN_COST);
11253   format %{ "eon  $dst, $src1, zr" %}
11254 
11255   ins_encode %{
11256     __ eon(as_Register($dst$$reg),
11257               as_Register($src1$$reg),
11258               zr,
11259               Assembler::LSL, 0);
11260   %}
11261 
11262   ins_pipe(ialu_reg);
11263 %}
11264 
11265 // This pattern is automatically generated from aarch64_ad.m4
11266 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11267 instruct regI_not_reg(iRegINoSp dst,
11268                          iRegIorL2I src1, immI_M1 m1,
11269                          rFlagsReg cr) %{
11270   match(Set dst (XorI src1 m1));
11271   ins_cost(INSN_COST);
11272   format %{ "eonw  $dst, $src1, zr" %}
11273 
11274   ins_encode %{
11275     __ eonw(as_Register($dst$$reg),
11276               as_Register($src1$$reg),
11277               zr,
11278               Assembler::LSL, 0);
11279   %}
11280 
11281   ins_pipe(ialu_reg);
11282 %}
11283 
11284 // This pattern is automatically generated from aarch64_ad.m4
11285 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11286 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11287                               immI0 zero, iRegIorL2I src1, immI src2) %{
11288   match(Set dst (SubI zero (URShiftI src1 src2)));
11289 
11290   ins_cost(1.9 * INSN_COST);
11291   format %{ "negw  $dst, $src1, LSR $src2" %}
11292 
11293   ins_encode %{
11294     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11295             Assembler::LSR, $src2$$constant & 0x1f);
11296   %}
11297 
11298   ins_pipe(ialu_reg_shift);
11299 %}
11300 
11301 // This pattern is automatically generated from aarch64_ad.m4
11302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11303 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11304                               immI0 zero, iRegIorL2I src1, immI src2) %{
11305   match(Set dst (SubI zero (RShiftI src1 src2)));
11306 
11307   ins_cost(1.9 * INSN_COST);
11308   format %{ "negw  $dst, $src1, ASR $src2" %}
11309 
11310   ins_encode %{
11311     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11312             Assembler::ASR, $src2$$constant & 0x1f);
11313   %}
11314 
11315   ins_pipe(ialu_reg_shift);
11316 %}
11317 
11318 // This pattern is automatically generated from aarch64_ad.m4
11319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11320 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11321                               immI0 zero, iRegIorL2I src1, immI src2) %{
11322   match(Set dst (SubI zero (LShiftI src1 src2)));
11323 
11324   ins_cost(1.9 * INSN_COST);
11325   format %{ "negw  $dst, $src1, LSL $src2" %}
11326 
11327   ins_encode %{
11328     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11329             Assembler::LSL, $src2$$constant & 0x1f);
11330   %}
11331 
11332   ins_pipe(ialu_reg_shift);
11333 %}
11334 
11335 // This pattern is automatically generated from aarch64_ad.m4
11336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11337 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11338                               immL0 zero, iRegL src1, immI src2) %{
11339   match(Set dst (SubL zero (URShiftL src1 src2)));
11340 
11341   ins_cost(1.9 * INSN_COST);
11342   format %{ "neg  $dst, $src1, LSR $src2" %}
11343 
11344   ins_encode %{
11345     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11346             Assembler::LSR, $src2$$constant & 0x3f);
11347   %}
11348 
11349   ins_pipe(ialu_reg_shift);
11350 %}
11351 
11352 // This pattern is automatically generated from aarch64_ad.m4
11353 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11354 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11355                               immL0 zero, iRegL src1, immI src2) %{
11356   match(Set dst (SubL zero (RShiftL src1 src2)));
11357 
11358   ins_cost(1.9 * INSN_COST);
11359   format %{ "neg  $dst, $src1, ASR $src2" %}
11360 
11361   ins_encode %{
11362     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11363             Assembler::ASR, $src2$$constant & 0x3f);
11364   %}
11365 
11366   ins_pipe(ialu_reg_shift);
11367 %}
11368 
11369 // This pattern is automatically generated from aarch64_ad.m4
11370 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11371 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11372                               immL0 zero, iRegL src1, immI src2) %{
11373   match(Set dst (SubL zero (LShiftL src1 src2)));
11374 
11375   ins_cost(1.9 * INSN_COST);
11376   format %{ "neg  $dst, $src1, LSL $src2" %}
11377 
11378   ins_encode %{
11379     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11380             Assembler::LSL, $src2$$constant & 0x3f);
11381   %}
11382 
11383   ins_pipe(ialu_reg_shift);
11384 %}
11385 
11386 // This pattern is automatically generated from aarch64_ad.m4
11387 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11388 instruct AndI_reg_not_reg(iRegINoSp dst,
11389                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11390   match(Set dst (AndI src1 (XorI src2 m1)));
11391   ins_cost(INSN_COST);
11392   format %{ "bicw  $dst, $src1, $src2" %}
11393 
11394   ins_encode %{
11395     __ bicw(as_Register($dst$$reg),
11396               as_Register($src1$$reg),
11397               as_Register($src2$$reg),
11398               Assembler::LSL, 0);
11399   %}
11400 
11401   ins_pipe(ialu_reg_reg);
11402 %}
11403 
11404 // This pattern is automatically generated from aarch64_ad.m4
11405 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11406 instruct AndL_reg_not_reg(iRegLNoSp dst,
11407                          iRegL src1, iRegL src2, immL_M1 m1) %{
11408   match(Set dst (AndL src1 (XorL src2 m1)));
11409   ins_cost(INSN_COST);
11410   format %{ "bic  $dst, $src1, $src2" %}
11411 
11412   ins_encode %{
11413     __ bic(as_Register($dst$$reg),
11414               as_Register($src1$$reg),
11415               as_Register($src2$$reg),
11416               Assembler::LSL, 0);
11417   %}
11418 
11419   ins_pipe(ialu_reg_reg);
11420 %}
11421 
11422 // This pattern is automatically generated from aarch64_ad.m4
11423 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11424 instruct OrI_reg_not_reg(iRegINoSp dst,
11425                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11426   match(Set dst (OrI src1 (XorI src2 m1)));
11427   ins_cost(INSN_COST);
11428   format %{ "ornw  $dst, $src1, $src2" %}
11429 
11430   ins_encode %{
11431     __ ornw(as_Register($dst$$reg),
11432               as_Register($src1$$reg),
11433               as_Register($src2$$reg),
11434               Assembler::LSL, 0);
11435   %}
11436 
11437   ins_pipe(ialu_reg_reg);
11438 %}
11439 
11440 // This pattern is automatically generated from aarch64_ad.m4
11441 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11442 instruct OrL_reg_not_reg(iRegLNoSp dst,
11443                          iRegL src1, iRegL src2, immL_M1 m1) %{
11444   match(Set dst (OrL src1 (XorL src2 m1)));
11445   ins_cost(INSN_COST);
11446   format %{ "orn  $dst, $src1, $src2" %}
11447 
11448   ins_encode %{
11449     __ orn(as_Register($dst$$reg),
11450               as_Register($src1$$reg),
11451               as_Register($src2$$reg),
11452               Assembler::LSL, 0);
11453   %}
11454 
11455   ins_pipe(ialu_reg_reg);
11456 %}
11457 
11458 // This pattern is automatically generated from aarch64_ad.m4
11459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11460 instruct XorI_reg_not_reg(iRegINoSp dst,
11461                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11462   match(Set dst (XorI m1 (XorI src2 src1)));
11463   ins_cost(INSN_COST);
11464   format %{ "eonw  $dst, $src1, $src2" %}
11465 
11466   ins_encode %{
11467     __ eonw(as_Register($dst$$reg),
11468               as_Register($src1$$reg),
11469               as_Register($src2$$reg),
11470               Assembler::LSL, 0);
11471   %}
11472 
11473   ins_pipe(ialu_reg_reg);
11474 %}
11475 
11476 // This pattern is automatically generated from aarch64_ad.m4
11477 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11478 instruct XorL_reg_not_reg(iRegLNoSp dst,
11479                          iRegL src1, iRegL src2, immL_M1 m1) %{
11480   match(Set dst (XorL m1 (XorL src2 src1)));
11481   ins_cost(INSN_COST);
11482   format %{ "eon  $dst, $src1, $src2" %}
11483 
11484   ins_encode %{
11485     __ eon(as_Register($dst$$reg),
11486               as_Register($src1$$reg),
11487               as_Register($src2$$reg),
11488               Assembler::LSL, 0);
11489   %}
11490 
11491   ins_pipe(ialu_reg_reg);
11492 %}
11493 
11494 // This pattern is automatically generated from aarch64_ad.m4
11495 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11496 // val & (-1 ^ (val >>> shift)) ==> bicw
11497 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11498                          iRegIorL2I src1, iRegIorL2I src2,
11499                          immI src3, immI_M1 src4) %{
11500   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11501   ins_cost(1.9 * INSN_COST);
11502   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11503 
11504   ins_encode %{
11505     __ bicw(as_Register($dst$$reg),
11506               as_Register($src1$$reg),
11507               as_Register($src2$$reg),
11508               Assembler::LSR,
11509               $src3$$constant & 0x1f);
11510   %}
11511 
11512   ins_pipe(ialu_reg_reg_shift);
11513 %}
11514 
11515 // This pattern is automatically generated from aarch64_ad.m4
11516 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11517 // val & (-1 ^ (val >>> shift)) ==> bic
11518 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11519                          iRegL src1, iRegL src2,
11520                          immI src3, immL_M1 src4) %{
11521   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11522   ins_cost(1.9 * INSN_COST);
11523   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11524 
11525   ins_encode %{
11526     __ bic(as_Register($dst$$reg),
11527               as_Register($src1$$reg),
11528               as_Register($src2$$reg),
11529               Assembler::LSR,
11530               $src3$$constant & 0x3f);
11531   %}
11532 
11533   ins_pipe(ialu_reg_reg_shift);
11534 %}
11535 
11536 // This pattern is automatically generated from aarch64_ad.m4
11537 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11538 // val & (-1 ^ (val >> shift)) ==> bicw
11539 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11540                          iRegIorL2I src1, iRegIorL2I src2,
11541                          immI src3, immI_M1 src4) %{
11542   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11543   ins_cost(1.9 * INSN_COST);
11544   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11545 
11546   ins_encode %{
11547     __ bicw(as_Register($dst$$reg),
11548               as_Register($src1$$reg),
11549               as_Register($src2$$reg),
11550               Assembler::ASR,
11551               $src3$$constant & 0x1f);
11552   %}
11553 
11554   ins_pipe(ialu_reg_reg_shift);
11555 %}
11556 
11557 // This pattern is automatically generated from aarch64_ad.m4
11558 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11559 // val & (-1 ^ (val >> shift)) ==> bic
11560 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11561                          iRegL src1, iRegL src2,
11562                          immI src3, immL_M1 src4) %{
11563   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11564   ins_cost(1.9 * INSN_COST);
11565   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11566 
11567   ins_encode %{
11568     __ bic(as_Register($dst$$reg),
11569               as_Register($src1$$reg),
11570               as_Register($src2$$reg),
11571               Assembler::ASR,
11572               $src3$$constant & 0x3f);
11573   %}
11574 
11575   ins_pipe(ialu_reg_reg_shift);
11576 %}
11577 
11578 // This pattern is automatically generated from aarch64_ad.m4
11579 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11580 // val & (-1 ^ (val ror shift)) ==> bicw
11581 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11582                          iRegIorL2I src1, iRegIorL2I src2,
11583                          immI src3, immI_M1 src4) %{
11584   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11585   ins_cost(1.9 * INSN_COST);
11586   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11587 
11588   ins_encode %{
11589     __ bicw(as_Register($dst$$reg),
11590               as_Register($src1$$reg),
11591               as_Register($src2$$reg),
11592               Assembler::ROR,
11593               $src3$$constant & 0x1f);
11594   %}
11595 
11596   ins_pipe(ialu_reg_reg_shift);
11597 %}
11598 
11599 // This pattern is automatically generated from aarch64_ad.m4
11600 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11601 // val & (-1 ^ (val ror shift)) ==> bic
11602 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11603                          iRegL src1, iRegL src2,
11604                          immI src3, immL_M1 src4) %{
11605   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11606   ins_cost(1.9 * INSN_COST);
11607   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11608 
11609   ins_encode %{
11610     __ bic(as_Register($dst$$reg),
11611               as_Register($src1$$reg),
11612               as_Register($src2$$reg),
11613               Assembler::ROR,
11614               $src3$$constant & 0x3f);
11615   %}
11616 
11617   ins_pipe(ialu_reg_reg_shift);
11618 %}
11619 
11620 // This pattern is automatically generated from aarch64_ad.m4
11621 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11622 // val & (-1 ^ (val << shift)) ==> bicw
11623 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11624                          iRegIorL2I src1, iRegIorL2I src2,
11625                          immI src3, immI_M1 src4) %{
11626   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11627   ins_cost(1.9 * INSN_COST);
11628   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11629 
11630   ins_encode %{
11631     __ bicw(as_Register($dst$$reg),
11632               as_Register($src1$$reg),
11633               as_Register($src2$$reg),
11634               Assembler::LSL,
11635               $src3$$constant & 0x1f);
11636   %}
11637 
11638   ins_pipe(ialu_reg_reg_shift);
11639 %}
11640 
11641 // This pattern is automatically generated from aarch64_ad.m4
11642 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11643 // val & (-1 ^ (val << shift)) ==> bic
11644 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11645                          iRegL src1, iRegL src2,
11646                          immI src3, immL_M1 src4) %{
11647   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11648   ins_cost(1.9 * INSN_COST);
11649   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11650 
11651   ins_encode %{
11652     __ bic(as_Register($dst$$reg),
11653               as_Register($src1$$reg),
11654               as_Register($src2$$reg),
11655               Assembler::LSL,
11656               $src3$$constant & 0x3f);
11657   %}
11658 
11659   ins_pipe(ialu_reg_reg_shift);
11660 %}
11661 
11662 // This pattern is automatically generated from aarch64_ad.m4
11663 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11664 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11665 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11666                          iRegIorL2I src1, iRegIorL2I src2,
11667                          immI src3, immI_M1 src4) %{
11668   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11669   ins_cost(1.9 * INSN_COST);
11670   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11671 
11672   ins_encode %{
11673     __ eonw(as_Register($dst$$reg),
11674               as_Register($src1$$reg),
11675               as_Register($src2$$reg),
11676               Assembler::LSR,
11677               $src3$$constant & 0x1f);
11678   %}
11679 
11680   ins_pipe(ialu_reg_reg_shift);
11681 %}
11682 
11683 // This pattern is automatically generated from aarch64_ad.m4
11684 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11685 // val ^ (-1 ^ (val >>> shift)) ==> eon
11686 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11687                          iRegL src1, iRegL src2,
11688                          immI src3, immL_M1 src4) %{
11689   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11690   ins_cost(1.9 * INSN_COST);
11691   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11692 
11693   ins_encode %{
11694     __ eon(as_Register($dst$$reg),
11695               as_Register($src1$$reg),
11696               as_Register($src2$$reg),
11697               Assembler::LSR,
11698               $src3$$constant & 0x3f);
11699   %}
11700 
11701   ins_pipe(ialu_reg_reg_shift);
11702 %}
11703 
11704 // This pattern is automatically generated from aarch64_ad.m4
11705 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11706 // val ^ (-1 ^ (val >> shift)) ==> eonw
11707 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11708                          iRegIorL2I src1, iRegIorL2I src2,
11709                          immI src3, immI_M1 src4) %{
11710   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11711   ins_cost(1.9 * INSN_COST);
11712   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11713 
11714   ins_encode %{
11715     __ eonw(as_Register($dst$$reg),
11716               as_Register($src1$$reg),
11717               as_Register($src2$$reg),
11718               Assembler::ASR,
11719               $src3$$constant & 0x1f);
11720   %}
11721 
11722   ins_pipe(ialu_reg_reg_shift);
11723 %}
11724 
11725 // This pattern is automatically generated from aarch64_ad.m4
11726 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11727 // val ^ (-1 ^ (val >> shift)) ==> eon
11728 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11729                          iRegL src1, iRegL src2,
11730                          immI src3, immL_M1 src4) %{
11731   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11732   ins_cost(1.9 * INSN_COST);
11733   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11734 
11735   ins_encode %{
11736     __ eon(as_Register($dst$$reg),
11737               as_Register($src1$$reg),
11738               as_Register($src2$$reg),
11739               Assembler::ASR,
11740               $src3$$constant & 0x3f);
11741   %}
11742 
11743   ins_pipe(ialu_reg_reg_shift);
11744 %}
11745 
11746 // This pattern is automatically generated from aarch64_ad.m4
11747 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11748 // val ^ (-1 ^ (val ror shift)) ==> eonw
11749 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11750                          iRegIorL2I src1, iRegIorL2I src2,
11751                          immI src3, immI_M1 src4) %{
11752   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11753   ins_cost(1.9 * INSN_COST);
11754   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11755 
11756   ins_encode %{
11757     __ eonw(as_Register($dst$$reg),
11758               as_Register($src1$$reg),
11759               as_Register($src2$$reg),
11760               Assembler::ROR,
11761               $src3$$constant & 0x1f);
11762   %}
11763 
11764   ins_pipe(ialu_reg_reg_shift);
11765 %}
11766 
11767 // This pattern is automatically generated from aarch64_ad.m4
11768 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11769 // val ^ (-1 ^ (val ror shift)) ==> eon
11770 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11771                          iRegL src1, iRegL src2,
11772                          immI src3, immL_M1 src4) %{
11773   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11774   ins_cost(1.9 * INSN_COST);
11775   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11776 
11777   ins_encode %{
11778     __ eon(as_Register($dst$$reg),
11779               as_Register($src1$$reg),
11780               as_Register($src2$$reg),
11781               Assembler::ROR,
11782               $src3$$constant & 0x3f);
11783   %}
11784 
11785   ins_pipe(ialu_reg_reg_shift);
11786 %}
11787 
11788 // This pattern is automatically generated from aarch64_ad.m4
11789 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11790 // val ^ (-1 ^ (val << shift)) ==> eonw
11791 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11792                          iRegIorL2I src1, iRegIorL2I src2,
11793                          immI src3, immI_M1 src4) %{
11794   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11795   ins_cost(1.9 * INSN_COST);
11796   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11797 
11798   ins_encode %{
11799     __ eonw(as_Register($dst$$reg),
11800               as_Register($src1$$reg),
11801               as_Register($src2$$reg),
11802               Assembler::LSL,
11803               $src3$$constant & 0x1f);
11804   %}
11805 
11806   ins_pipe(ialu_reg_reg_shift);
11807 %}
11808 
11809 // This pattern is automatically generated from aarch64_ad.m4
11810 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11811 // val ^ (-1 ^ (val << shift)) ==> eon
11812 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11813                          iRegL src1, iRegL src2,
11814                          immI src3, immL_M1 src4) %{
11815   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11816   ins_cost(1.9 * INSN_COST);
11817   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11818 
11819   ins_encode %{
11820     __ eon(as_Register($dst$$reg),
11821               as_Register($src1$$reg),
11822               as_Register($src2$$reg),
11823               Assembler::LSL,
11824               $src3$$constant & 0x3f);
11825   %}
11826 
11827   ins_pipe(ialu_reg_reg_shift);
11828 %}
11829 
11830 // This pattern is automatically generated from aarch64_ad.m4
11831 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11832 // val | (-1 ^ (val >>> shift)) ==> ornw
11833 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11834                          iRegIorL2I src1, iRegIorL2I src2,
11835                          immI src3, immI_M1 src4) %{
11836   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11837   ins_cost(1.9 * INSN_COST);
11838   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11839 
11840   ins_encode %{
11841     __ ornw(as_Register($dst$$reg),
11842               as_Register($src1$$reg),
11843               as_Register($src2$$reg),
11844               Assembler::LSR,
11845               $src3$$constant & 0x1f);
11846   %}
11847 
11848   ins_pipe(ialu_reg_reg_shift);
11849 %}
11850 
11851 // This pattern is automatically generated from aarch64_ad.m4
11852 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11853 // val | (-1 ^ (val >>> shift)) ==> orn
11854 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11855                          iRegL src1, iRegL src2,
11856                          immI src3, immL_M1 src4) %{
11857   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11858   ins_cost(1.9 * INSN_COST);
11859   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11860 
11861   ins_encode %{
11862     __ orn(as_Register($dst$$reg),
11863               as_Register($src1$$reg),
11864               as_Register($src2$$reg),
11865               Assembler::LSR,
11866               $src3$$constant & 0x3f);
11867   %}
11868 
11869   ins_pipe(ialu_reg_reg_shift);
11870 %}
11871 
11872 // This pattern is automatically generated from aarch64_ad.m4
11873 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11874 // val | (-1 ^ (val >> shift)) ==> ornw
11875 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11876                          iRegIorL2I src1, iRegIorL2I src2,
11877                          immI src3, immI_M1 src4) %{
11878   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11879   ins_cost(1.9 * INSN_COST);
11880   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11881 
11882   ins_encode %{
11883     __ ornw(as_Register($dst$$reg),
11884               as_Register($src1$$reg),
11885               as_Register($src2$$reg),
11886               Assembler::ASR,
11887               $src3$$constant & 0x1f);
11888   %}
11889 
11890   ins_pipe(ialu_reg_reg_shift);
11891 %}
11892 
11893 // This pattern is automatically generated from aarch64_ad.m4
11894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11895 // val | (-1 ^ (val >> shift)) ==> orn
11896 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11897                          iRegL src1, iRegL src2,
11898                          immI src3, immL_M1 src4) %{
11899   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11900   ins_cost(1.9 * INSN_COST);
11901   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11902 
11903   ins_encode %{
11904     __ orn(as_Register($dst$$reg),
11905               as_Register($src1$$reg),
11906               as_Register($src2$$reg),
11907               Assembler::ASR,
11908               $src3$$constant & 0x3f);
11909   %}
11910 
11911   ins_pipe(ialu_reg_reg_shift);
11912 %}
11913 
11914 // This pattern is automatically generated from aarch64_ad.m4
11915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11916 // val | (-1 ^ (val ror shift)) ==> ornw
11917 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11918                          iRegIorL2I src1, iRegIorL2I src2,
11919                          immI src3, immI_M1 src4) %{
11920   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11921   ins_cost(1.9 * INSN_COST);
11922   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11923 
11924   ins_encode %{
11925     __ ornw(as_Register($dst$$reg),
11926               as_Register($src1$$reg),
11927               as_Register($src2$$reg),
11928               Assembler::ROR,
11929               $src3$$constant & 0x1f);
11930   %}
11931 
11932   ins_pipe(ialu_reg_reg_shift);
11933 %}
11934 
11935 // This pattern is automatically generated from aarch64_ad.m4
11936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11937 // val | (-1 ^ (val ror shift)) ==> orn
11938 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11939                          iRegL src1, iRegL src2,
11940                          immI src3, immL_M1 src4) %{
11941   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11942   ins_cost(1.9 * INSN_COST);
11943   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11944 
11945   ins_encode %{
11946     __ orn(as_Register($dst$$reg),
11947               as_Register($src1$$reg),
11948               as_Register($src2$$reg),
11949               Assembler::ROR,
11950               $src3$$constant & 0x3f);
11951   %}
11952 
11953   ins_pipe(ialu_reg_reg_shift);
11954 %}
11955 
11956 // This pattern is automatically generated from aarch64_ad.m4
11957 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11958 // val | (-1 ^ (val << shift)) ==> ornw
11959 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11960                          iRegIorL2I src1, iRegIorL2I src2,
11961                          immI src3, immI_M1 src4) %{
11962   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11963   ins_cost(1.9 * INSN_COST);
11964   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11965 
11966   ins_encode %{
11967     __ ornw(as_Register($dst$$reg),
11968               as_Register($src1$$reg),
11969               as_Register($src2$$reg),
11970               Assembler::LSL,
11971               $src3$$constant & 0x1f);
11972   %}
11973 
11974   ins_pipe(ialu_reg_reg_shift);
11975 %}
11976 
11977 // This pattern is automatically generated from aarch64_ad.m4
11978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11979 // val | (-1 ^ (val << shift)) ==> orn
11980 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11981                          iRegL src1, iRegL src2,
11982                          immI src3, immL_M1 src4) %{
11983   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11984   ins_cost(1.9 * INSN_COST);
11985   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11986 
11987   ins_encode %{
11988     __ orn(as_Register($dst$$reg),
11989               as_Register($src1$$reg),
11990               as_Register($src2$$reg),
11991               Assembler::LSL,
11992               $src3$$constant & 0x3f);
11993   %}
11994 
11995   ins_pipe(ialu_reg_reg_shift);
11996 %}
11997 
11998 // This pattern is automatically generated from aarch64_ad.m4
11999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12000 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12001                          iRegIorL2I src1, iRegIorL2I src2,
12002                          immI src3) %{
12003   match(Set dst (AndI src1 (URShiftI src2 src3)));
12004 
12005   ins_cost(1.9 * INSN_COST);
12006   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
12007 
12008   ins_encode %{
12009     __ andw(as_Register($dst$$reg),
12010               as_Register($src1$$reg),
12011               as_Register($src2$$reg),
12012               Assembler::LSR,
12013               $src3$$constant & 0x1f);
12014   %}
12015 
12016   ins_pipe(ialu_reg_reg_shift);
12017 %}
12018 
12019 // This pattern is automatically generated from aarch64_ad.m4
12020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12021 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12022                          iRegL src1, iRegL src2,
12023                          immI src3) %{
12024   match(Set dst (AndL src1 (URShiftL src2 src3)));
12025 
12026   ins_cost(1.9 * INSN_COST);
12027   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
12028 
12029   ins_encode %{
12030     __ andr(as_Register($dst$$reg),
12031               as_Register($src1$$reg),
12032               as_Register($src2$$reg),
12033               Assembler::LSR,
12034               $src3$$constant & 0x3f);
12035   %}
12036 
12037   ins_pipe(ialu_reg_reg_shift);
12038 %}
12039 
12040 // This pattern is automatically generated from aarch64_ad.m4
12041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12042 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12043                          iRegIorL2I src1, iRegIorL2I src2,
12044                          immI src3) %{
12045   match(Set dst (AndI src1 (RShiftI src2 src3)));
12046 
12047   ins_cost(1.9 * INSN_COST);
12048   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
12049 
12050   ins_encode %{
12051     __ andw(as_Register($dst$$reg),
12052               as_Register($src1$$reg),
12053               as_Register($src2$$reg),
12054               Assembler::ASR,
12055               $src3$$constant & 0x1f);
12056   %}
12057 
12058   ins_pipe(ialu_reg_reg_shift);
12059 %}
12060 
12061 // This pattern is automatically generated from aarch64_ad.m4
12062 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12063 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12064                          iRegL src1, iRegL src2,
12065                          immI src3) %{
12066   match(Set dst (AndL src1 (RShiftL src2 src3)));
12067 
12068   ins_cost(1.9 * INSN_COST);
12069   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
12070 
12071   ins_encode %{
12072     __ andr(as_Register($dst$$reg),
12073               as_Register($src1$$reg),
12074               as_Register($src2$$reg),
12075               Assembler::ASR,
12076               $src3$$constant & 0x3f);
12077   %}
12078 
12079   ins_pipe(ialu_reg_reg_shift);
12080 %}
12081 
12082 // This pattern is automatically generated from aarch64_ad.m4
12083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12084 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12085                          iRegIorL2I src1, iRegIorL2I src2,
12086                          immI src3) %{
12087   match(Set dst (AndI src1 (LShiftI src2 src3)));
12088 
12089   ins_cost(1.9 * INSN_COST);
12090   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12091 
12092   ins_encode %{
12093     __ andw(as_Register($dst$$reg),
12094               as_Register($src1$$reg),
12095               as_Register($src2$$reg),
12096               Assembler::LSL,
12097               $src3$$constant & 0x1f);
12098   %}
12099 
12100   ins_pipe(ialu_reg_reg_shift);
12101 %}
12102 
12103 // This pattern is automatically generated from aarch64_ad.m4
12104 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12105 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12106                          iRegL src1, iRegL src2,
12107                          immI src3) %{
12108   match(Set dst (AndL src1 (LShiftL src2 src3)));
12109 
12110   ins_cost(1.9 * INSN_COST);
12111   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12112 
12113   ins_encode %{
12114     __ andr(as_Register($dst$$reg),
12115               as_Register($src1$$reg),
12116               as_Register($src2$$reg),
12117               Assembler::LSL,
12118               $src3$$constant & 0x3f);
12119   %}
12120 
12121   ins_pipe(ialu_reg_reg_shift);
12122 %}
12123 
12124 // This pattern is automatically generated from aarch64_ad.m4
12125 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12126 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12127                          iRegIorL2I src1, iRegIorL2I src2,
12128                          immI src3) %{
12129   match(Set dst (AndI src1 (RotateRight src2 src3)));
12130 
12131   ins_cost(1.9 * INSN_COST);
12132   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12133 
12134   ins_encode %{
12135     __ andw(as_Register($dst$$reg),
12136               as_Register($src1$$reg),
12137               as_Register($src2$$reg),
12138               Assembler::ROR,
12139               $src3$$constant & 0x1f);
12140   %}
12141 
12142   ins_pipe(ialu_reg_reg_shift);
12143 %}
12144 
12145 // This pattern is automatically generated from aarch64_ad.m4
12146 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12147 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12148                          iRegL src1, iRegL src2,
12149                          immI src3) %{
12150   match(Set dst (AndL src1 (RotateRight src2 src3)));
12151 
12152   ins_cost(1.9 * INSN_COST);
12153   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12154 
12155   ins_encode %{
12156     __ andr(as_Register($dst$$reg),
12157               as_Register($src1$$reg),
12158               as_Register($src2$$reg),
12159               Assembler::ROR,
12160               $src3$$constant & 0x3f);
12161   %}
12162 
12163   ins_pipe(ialu_reg_reg_shift);
12164 %}
12165 
12166 // This pattern is automatically generated from aarch64_ad.m4
12167 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12168 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12169                          iRegIorL2I src1, iRegIorL2I src2,
12170                          immI src3) %{
12171   match(Set dst (XorI src1 (URShiftI src2 src3)));
12172 
12173   ins_cost(1.9 * INSN_COST);
12174   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12175 
12176   ins_encode %{
12177     __ eorw(as_Register($dst$$reg),
12178               as_Register($src1$$reg),
12179               as_Register($src2$$reg),
12180               Assembler::LSR,
12181               $src3$$constant & 0x1f);
12182   %}
12183 
12184   ins_pipe(ialu_reg_reg_shift);
12185 %}
12186 
12187 // This pattern is automatically generated from aarch64_ad.m4
12188 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12189 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12190                          iRegL src1, iRegL src2,
12191                          immI src3) %{
12192   match(Set dst (XorL src1 (URShiftL src2 src3)));
12193 
12194   ins_cost(1.9 * INSN_COST);
12195   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12196 
12197   ins_encode %{
12198     __ eor(as_Register($dst$$reg),
12199               as_Register($src1$$reg),
12200               as_Register($src2$$reg),
12201               Assembler::LSR,
12202               $src3$$constant & 0x3f);
12203   %}
12204 
12205   ins_pipe(ialu_reg_reg_shift);
12206 %}
12207 
12208 // This pattern is automatically generated from aarch64_ad.m4
12209 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12210 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12211                          iRegIorL2I src1, iRegIorL2I src2,
12212                          immI src3) %{
12213   match(Set dst (XorI src1 (RShiftI src2 src3)));
12214 
12215   ins_cost(1.9 * INSN_COST);
12216   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12217 
12218   ins_encode %{
12219     __ eorw(as_Register($dst$$reg),
12220               as_Register($src1$$reg),
12221               as_Register($src2$$reg),
12222               Assembler::ASR,
12223               $src3$$constant & 0x1f);
12224   %}
12225 
12226   ins_pipe(ialu_reg_reg_shift);
12227 %}
12228 
12229 // This pattern is automatically generated from aarch64_ad.m4
12230 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12231 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12232                          iRegL src1, iRegL src2,
12233                          immI src3) %{
12234   match(Set dst (XorL src1 (RShiftL src2 src3)));
12235 
12236   ins_cost(1.9 * INSN_COST);
12237   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12238 
12239   ins_encode %{
12240     __ eor(as_Register($dst$$reg),
12241               as_Register($src1$$reg),
12242               as_Register($src2$$reg),
12243               Assembler::ASR,
12244               $src3$$constant & 0x3f);
12245   %}
12246 
12247   ins_pipe(ialu_reg_reg_shift);
12248 %}
12249 
12250 // This pattern is automatically generated from aarch64_ad.m4
12251 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12252 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12253                          iRegIorL2I src1, iRegIorL2I src2,
12254                          immI src3) %{
12255   match(Set dst (XorI src1 (LShiftI src2 src3)));
12256 
12257   ins_cost(1.9 * INSN_COST);
12258   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12259 
12260   ins_encode %{
12261     __ eorw(as_Register($dst$$reg),
12262               as_Register($src1$$reg),
12263               as_Register($src2$$reg),
12264               Assembler::LSL,
12265               $src3$$constant & 0x1f);
12266   %}
12267 
12268   ins_pipe(ialu_reg_reg_shift);
12269 %}
12270 
12271 // This pattern is automatically generated from aarch64_ad.m4
12272 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12273 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12274                          iRegL src1, iRegL src2,
12275                          immI src3) %{
12276   match(Set dst (XorL src1 (LShiftL src2 src3)));
12277 
12278   ins_cost(1.9 * INSN_COST);
12279   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12280 
12281   ins_encode %{
12282     __ eor(as_Register($dst$$reg),
12283               as_Register($src1$$reg),
12284               as_Register($src2$$reg),
12285               Assembler::LSL,
12286               $src3$$constant & 0x3f);
12287   %}
12288 
12289   ins_pipe(ialu_reg_reg_shift);
12290 %}
12291 
12292 // This pattern is automatically generated from aarch64_ad.m4
12293 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12294 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12295                          iRegIorL2I src1, iRegIorL2I src2,
12296                          immI src3) %{
12297   match(Set dst (XorI src1 (RotateRight src2 src3)));
12298 
12299   ins_cost(1.9 * INSN_COST);
12300   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12301 
12302   ins_encode %{
12303     __ eorw(as_Register($dst$$reg),
12304               as_Register($src1$$reg),
12305               as_Register($src2$$reg),
12306               Assembler::ROR,
12307               $src3$$constant & 0x1f);
12308   %}
12309 
12310   ins_pipe(ialu_reg_reg_shift);
12311 %}
12312 
12313 // This pattern is automatically generated from aarch64_ad.m4
12314 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12315 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12316                          iRegL src1, iRegL src2,
12317                          immI src3) %{
12318   match(Set dst (XorL src1 (RotateRight src2 src3)));
12319 
12320   ins_cost(1.9 * INSN_COST);
12321   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12322 
12323   ins_encode %{
12324     __ eor(as_Register($dst$$reg),
12325               as_Register($src1$$reg),
12326               as_Register($src2$$reg),
12327               Assembler::ROR,
12328               $src3$$constant & 0x3f);
12329   %}
12330 
12331   ins_pipe(ialu_reg_reg_shift);
12332 %}
12333 
12334 // This pattern is automatically generated from aarch64_ad.m4
12335 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12336 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12337                          iRegIorL2I src1, iRegIorL2I src2,
12338                          immI src3) %{
12339   match(Set dst (OrI src1 (URShiftI src2 src3)));
12340 
12341   ins_cost(1.9 * INSN_COST);
12342   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12343 
12344   ins_encode %{
12345     __ orrw(as_Register($dst$$reg),
12346               as_Register($src1$$reg),
12347               as_Register($src2$$reg),
12348               Assembler::LSR,
12349               $src3$$constant & 0x1f);
12350   %}
12351 
12352   ins_pipe(ialu_reg_reg_shift);
12353 %}
12354 
12355 // This pattern is automatically generated from aarch64_ad.m4
12356 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12357 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12358                          iRegL src1, iRegL src2,
12359                          immI src3) %{
12360   match(Set dst (OrL src1 (URShiftL src2 src3)));
12361 
12362   ins_cost(1.9 * INSN_COST);
12363   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12364 
12365   ins_encode %{
12366     __ orr(as_Register($dst$$reg),
12367               as_Register($src1$$reg),
12368               as_Register($src2$$reg),
12369               Assembler::LSR,
12370               $src3$$constant & 0x3f);
12371   %}
12372 
12373   ins_pipe(ialu_reg_reg_shift);
12374 %}
12375 
12376 // This pattern is automatically generated from aarch64_ad.m4
12377 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12378 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12379                          iRegIorL2I src1, iRegIorL2I src2,
12380                          immI src3) %{
12381   match(Set dst (OrI src1 (RShiftI src2 src3)));
12382 
12383   ins_cost(1.9 * INSN_COST);
12384   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12385 
12386   ins_encode %{
12387     __ orrw(as_Register($dst$$reg),
12388               as_Register($src1$$reg),
12389               as_Register($src2$$reg),
12390               Assembler::ASR,
12391               $src3$$constant & 0x1f);
12392   %}
12393 
12394   ins_pipe(ialu_reg_reg_shift);
12395 %}
12396 
12397 // This pattern is automatically generated from aarch64_ad.m4
12398 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12399 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12400                          iRegL src1, iRegL src2,
12401                          immI src3) %{
12402   match(Set dst (OrL src1 (RShiftL src2 src3)));
12403 
12404   ins_cost(1.9 * INSN_COST);
12405   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12406 
12407   ins_encode %{
12408     __ orr(as_Register($dst$$reg),
12409               as_Register($src1$$reg),
12410               as_Register($src2$$reg),
12411               Assembler::ASR,
12412               $src3$$constant & 0x3f);
12413   %}
12414 
12415   ins_pipe(ialu_reg_reg_shift);
12416 %}
12417 
12418 // This pattern is automatically generated from aarch64_ad.m4
12419 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12420 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12421                          iRegIorL2I src1, iRegIorL2I src2,
12422                          immI src3) %{
12423   match(Set dst (OrI src1 (LShiftI src2 src3)));
12424 
12425   ins_cost(1.9 * INSN_COST);
12426   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12427 
12428   ins_encode %{
12429     __ orrw(as_Register($dst$$reg),
12430               as_Register($src1$$reg),
12431               as_Register($src2$$reg),
12432               Assembler::LSL,
12433               $src3$$constant & 0x1f);
12434   %}
12435 
12436   ins_pipe(ialu_reg_reg_shift);
12437 %}
12438 
12439 // This pattern is automatically generated from aarch64_ad.m4
12440 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12441 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12442                          iRegL src1, iRegL src2,
12443                          immI src3) %{
12444   match(Set dst (OrL src1 (LShiftL src2 src3)));
12445 
12446   ins_cost(1.9 * INSN_COST);
12447   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12448 
12449   ins_encode %{
12450     __ orr(as_Register($dst$$reg),
12451               as_Register($src1$$reg),
12452               as_Register($src2$$reg),
12453               Assembler::LSL,
12454               $src3$$constant & 0x3f);
12455   %}
12456 
12457   ins_pipe(ialu_reg_reg_shift);
12458 %}
12459 
12460 // This pattern is automatically generated from aarch64_ad.m4
12461 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12462 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12463                          iRegIorL2I src1, iRegIorL2I src2,
12464                          immI src3) %{
12465   match(Set dst (OrI src1 (RotateRight src2 src3)));
12466 
12467   ins_cost(1.9 * INSN_COST);
12468   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12469 
12470   ins_encode %{
12471     __ orrw(as_Register($dst$$reg),
12472               as_Register($src1$$reg),
12473               as_Register($src2$$reg),
12474               Assembler::ROR,
12475               $src3$$constant & 0x1f);
12476   %}
12477 
12478   ins_pipe(ialu_reg_reg_shift);
12479 %}
12480 
12481 // This pattern is automatically generated from aarch64_ad.m4
12482 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12483 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12484                          iRegL src1, iRegL src2,
12485                          immI src3) %{
12486   match(Set dst (OrL src1 (RotateRight src2 src3)));
12487 
12488   ins_cost(1.9 * INSN_COST);
12489   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12490 
12491   ins_encode %{
12492     __ orr(as_Register($dst$$reg),
12493               as_Register($src1$$reg),
12494               as_Register($src2$$reg),
12495               Assembler::ROR,
12496               $src3$$constant & 0x3f);
12497   %}
12498 
12499   ins_pipe(ialu_reg_reg_shift);
12500 %}
12501 
12502 // This pattern is automatically generated from aarch64_ad.m4
12503 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12504 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12505                          iRegIorL2I src1, iRegIorL2I src2,
12506                          immI src3) %{
12507   match(Set dst (AddI src1 (URShiftI src2 src3)));
12508 
12509   ins_cost(1.9 * INSN_COST);
12510   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12511 
12512   ins_encode %{
12513     __ addw(as_Register($dst$$reg),
12514               as_Register($src1$$reg),
12515               as_Register($src2$$reg),
12516               Assembler::LSR,
12517               $src3$$constant & 0x1f);
12518   %}
12519 
12520   ins_pipe(ialu_reg_reg_shift);
12521 %}
12522 
12523 // This pattern is automatically generated from aarch64_ad.m4
12524 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12525 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12526                          iRegL src1, iRegL src2,
12527                          immI src3) %{
12528   match(Set dst (AddL src1 (URShiftL src2 src3)));
12529 
12530   ins_cost(1.9 * INSN_COST);
12531   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12532 
12533   ins_encode %{
12534     __ add(as_Register($dst$$reg),
12535               as_Register($src1$$reg),
12536               as_Register($src2$$reg),
12537               Assembler::LSR,
12538               $src3$$constant & 0x3f);
12539   %}
12540 
12541   ins_pipe(ialu_reg_reg_shift);
12542 %}
12543 
12544 // This pattern is automatically generated from aarch64_ad.m4
12545 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12546 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12547                          iRegIorL2I src1, iRegIorL2I src2,
12548                          immI src3) %{
12549   match(Set dst (AddI src1 (RShiftI src2 src3)));
12550 
12551   ins_cost(1.9 * INSN_COST);
12552   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12553 
12554   ins_encode %{
12555     __ addw(as_Register($dst$$reg),
12556               as_Register($src1$$reg),
12557               as_Register($src2$$reg),
12558               Assembler::ASR,
12559               $src3$$constant & 0x1f);
12560   %}
12561 
12562   ins_pipe(ialu_reg_reg_shift);
12563 %}
12564 
12565 // This pattern is automatically generated from aarch64_ad.m4
12566 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12567 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12568                          iRegL src1, iRegL src2,
12569                          immI src3) %{
12570   match(Set dst (AddL src1 (RShiftL src2 src3)));
12571 
12572   ins_cost(1.9 * INSN_COST);
12573   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12574 
12575   ins_encode %{
12576     __ add(as_Register($dst$$reg),
12577               as_Register($src1$$reg),
12578               as_Register($src2$$reg),
12579               Assembler::ASR,
12580               $src3$$constant & 0x3f);
12581   %}
12582 
12583   ins_pipe(ialu_reg_reg_shift);
12584 %}
12585 
12586 // This pattern is automatically generated from aarch64_ad.m4
12587 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12588 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12589                          iRegIorL2I src1, iRegIorL2I src2,
12590                          immI src3) %{
12591   match(Set dst (AddI src1 (LShiftI src2 src3)));
12592 
12593   ins_cost(1.9 * INSN_COST);
12594   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12595 
12596   ins_encode %{
12597     __ addw(as_Register($dst$$reg),
12598               as_Register($src1$$reg),
12599               as_Register($src2$$reg),
12600               Assembler::LSL,
12601               $src3$$constant & 0x1f);
12602   %}
12603 
12604   ins_pipe(ialu_reg_reg_shift);
12605 %}
12606 
12607 // This pattern is automatically generated from aarch64_ad.m4
12608 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12609 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12610                          iRegL src1, iRegL src2,
12611                          immI src3) %{
12612   match(Set dst (AddL src1 (LShiftL src2 src3)));
12613 
12614   ins_cost(1.9 * INSN_COST);
12615   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12616 
12617   ins_encode %{
12618     __ add(as_Register($dst$$reg),
12619               as_Register($src1$$reg),
12620               as_Register($src2$$reg),
12621               Assembler::LSL,
12622               $src3$$constant & 0x3f);
12623   %}
12624 
12625   ins_pipe(ialu_reg_reg_shift);
12626 %}
12627 
12628 // This pattern is automatically generated from aarch64_ad.m4
12629 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12630 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12631                          iRegIorL2I src1, iRegIorL2I src2,
12632                          immI src3) %{
12633   match(Set dst (SubI src1 (URShiftI src2 src3)));
12634 
12635   ins_cost(1.9 * INSN_COST);
12636   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12637 
12638   ins_encode %{
12639     __ subw(as_Register($dst$$reg),
12640               as_Register($src1$$reg),
12641               as_Register($src2$$reg),
12642               Assembler::LSR,
12643               $src3$$constant & 0x1f);
12644   %}
12645 
12646   ins_pipe(ialu_reg_reg_shift);
12647 %}
12648 
12649 // This pattern is automatically generated from aarch64_ad.m4
12650 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12651 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12652                          iRegL src1, iRegL src2,
12653                          immI src3) %{
12654   match(Set dst (SubL src1 (URShiftL src2 src3)));
12655 
12656   ins_cost(1.9 * INSN_COST);
12657   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12658 
12659   ins_encode %{
12660     __ sub(as_Register($dst$$reg),
12661               as_Register($src1$$reg),
12662               as_Register($src2$$reg),
12663               Assembler::LSR,
12664               $src3$$constant & 0x3f);
12665   %}
12666 
12667   ins_pipe(ialu_reg_reg_shift);
12668 %}
12669 
12670 // This pattern is automatically generated from aarch64_ad.m4
12671 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12672 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12673                          iRegIorL2I src1, iRegIorL2I src2,
12674                          immI src3) %{
12675   match(Set dst (SubI src1 (RShiftI src2 src3)));
12676 
12677   ins_cost(1.9 * INSN_COST);
12678   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12679 
12680   ins_encode %{
12681     __ subw(as_Register($dst$$reg),
12682               as_Register($src1$$reg),
12683               as_Register($src2$$reg),
12684               Assembler::ASR,
12685               $src3$$constant & 0x1f);
12686   %}
12687 
12688   ins_pipe(ialu_reg_reg_shift);
12689 %}
12690 
12691 // This pattern is automatically generated from aarch64_ad.m4
12692 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12693 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12694                          iRegL src1, iRegL src2,
12695                          immI src3) %{
12696   match(Set dst (SubL src1 (RShiftL src2 src3)));
12697 
12698   ins_cost(1.9 * INSN_COST);
12699   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12700 
12701   ins_encode %{
12702     __ sub(as_Register($dst$$reg),
12703               as_Register($src1$$reg),
12704               as_Register($src2$$reg),
12705               Assembler::ASR,
12706               $src3$$constant & 0x3f);
12707   %}
12708 
12709   ins_pipe(ialu_reg_reg_shift);
12710 %}
12711 
12712 // This pattern is automatically generated from aarch64_ad.m4
12713 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12714 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12715                          iRegIorL2I src1, iRegIorL2I src2,
12716                          immI src3) %{
12717   match(Set dst (SubI src1 (LShiftI src2 src3)));
12718 
12719   ins_cost(1.9 * INSN_COST);
12720   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12721 
12722   ins_encode %{
12723     __ subw(as_Register($dst$$reg),
12724               as_Register($src1$$reg),
12725               as_Register($src2$$reg),
12726               Assembler::LSL,
12727               $src3$$constant & 0x1f);
12728   %}
12729 
12730   ins_pipe(ialu_reg_reg_shift);
12731 %}
12732 
12733 // This pattern is automatically generated from aarch64_ad.m4
12734 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12735 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12736                          iRegL src1, iRegL src2,
12737                          immI src3) %{
12738   match(Set dst (SubL src1 (LShiftL src2 src3)));
12739 
12740   ins_cost(1.9 * INSN_COST);
12741   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12742 
12743   ins_encode %{
12744     __ sub(as_Register($dst$$reg),
12745               as_Register($src1$$reg),
12746               as_Register($src2$$reg),
12747               Assembler::LSL,
12748               $src3$$constant & 0x3f);
12749   %}
12750 
12751   ins_pipe(ialu_reg_reg_shift);
12752 %}
12753 
12754 // This pattern is automatically generated from aarch64_ad.m4
12755 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12756 
12757 // Shift Left followed by Shift Right.
12758 // This idiom is used by the compiler for the i2b bytecode etc.
12759 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12760 %{
12761   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12762   ins_cost(INSN_COST * 2);
12763   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12764   ins_encode %{
12765     int lshift = $lshift_count$$constant & 63;
12766     int rshift = $rshift_count$$constant & 63;
12767     int s = 63 - lshift;
12768     int r = (rshift - lshift) & 63;
12769     __ sbfm(as_Register($dst$$reg),
12770             as_Register($src$$reg),
12771             r, s);
12772   %}
12773 
12774   ins_pipe(ialu_reg_shift);
12775 %}
12776 
12777 // This pattern is automatically generated from aarch64_ad.m4
12778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12779 
12780 // Shift Left followed by Shift Right.
12781 // This idiom is used by the compiler for the i2b bytecode etc.
12782 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12783 %{
12784   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12785   ins_cost(INSN_COST * 2);
12786   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12787   ins_encode %{
12788     int lshift = $lshift_count$$constant & 31;
12789     int rshift = $rshift_count$$constant & 31;
12790     int s = 31 - lshift;
12791     int r = (rshift - lshift) & 31;
12792     __ sbfmw(as_Register($dst$$reg),
12793             as_Register($src$$reg),
12794             r, s);
12795   %}
12796 
12797   ins_pipe(ialu_reg_shift);
12798 %}
12799 
12800 // This pattern is automatically generated from aarch64_ad.m4
12801 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12802 
12803 // Shift Left followed by Shift Right.
12804 // This idiom is used by the compiler for the i2b bytecode etc.
12805 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12806 %{
12807   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12808   ins_cost(INSN_COST * 2);
12809   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12810   ins_encode %{
12811     int lshift = $lshift_count$$constant & 63;
12812     int rshift = $rshift_count$$constant & 63;
12813     int s = 63 - lshift;
12814     int r = (rshift - lshift) & 63;
12815     __ ubfm(as_Register($dst$$reg),
12816             as_Register($src$$reg),
12817             r, s);
12818   %}
12819 
12820   ins_pipe(ialu_reg_shift);
12821 %}
12822 
12823 // This pattern is automatically generated from aarch64_ad.m4
12824 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12825 
12826 // Shift Left followed by Shift Right.
12827 // This idiom is used by the compiler for the i2b bytecode etc.
12828 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12829 %{
12830   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12831   ins_cost(INSN_COST * 2);
12832   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12833   ins_encode %{
12834     int lshift = $lshift_count$$constant & 31;
12835     int rshift = $rshift_count$$constant & 31;
12836     int s = 31 - lshift;
12837     int r = (rshift - lshift) & 31;
12838     __ ubfmw(as_Register($dst$$reg),
12839             as_Register($src$$reg),
12840             r, s);
12841   %}
12842 
12843   ins_pipe(ialu_reg_shift);
12844 %}
12845 
12846 // Bitfield extract with shift & mask
12847 
12848 // This pattern is automatically generated from aarch64_ad.m4
12849 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12850 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12851 %{
12852   match(Set dst (AndI (URShiftI src rshift) mask));
12853   // Make sure we are not going to exceed what ubfxw can do.
12854   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12855 
12856   ins_cost(INSN_COST);
12857   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12858   ins_encode %{
12859     int rshift = $rshift$$constant & 31;
12860     intptr_t mask = $mask$$constant;
12861     int width = exact_log2(mask+1);
12862     __ ubfxw(as_Register($dst$$reg),
12863             as_Register($src$$reg), rshift, width);
12864   %}
12865   ins_pipe(ialu_reg_shift);
12866 %}
12867 
12868 // This pattern is automatically generated from aarch64_ad.m4
12869 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12870 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12871 %{
12872   match(Set dst (AndL (URShiftL src rshift) mask));
12873   // Make sure we are not going to exceed what ubfx can do.
12874   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12875 
12876   ins_cost(INSN_COST);
12877   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12878   ins_encode %{
12879     int rshift = $rshift$$constant & 63;
12880     intptr_t mask = $mask$$constant;
12881     int width = exact_log2_long(mask+1);
12882     __ ubfx(as_Register($dst$$reg),
12883             as_Register($src$$reg), rshift, width);
12884   %}
12885   ins_pipe(ialu_reg_shift);
12886 %}
12887 
12888 
12889 // This pattern is automatically generated from aarch64_ad.m4
12890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12891 
12892 // We can use ubfx when extending an And with a mask when we know mask
12893 // is positive.  We know that because immI_bitmask guarantees it.
12894 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12895 %{
12896   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12897   // Make sure we are not going to exceed what ubfxw can do.
12898   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12899 
12900   ins_cost(INSN_COST * 2);
12901   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12902   ins_encode %{
12903     int rshift = $rshift$$constant & 31;
12904     intptr_t mask = $mask$$constant;
12905     int width = exact_log2(mask+1);
12906     __ ubfx(as_Register($dst$$reg),
12907             as_Register($src$$reg), rshift, width);
12908   %}
12909   ins_pipe(ialu_reg_shift);
12910 %}
12911 
12912 
12913 // This pattern is automatically generated from aarch64_ad.m4
12914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12915 
12916 // We can use ubfiz when masking by a positive number and then left shifting the result.
12917 // We know that the mask is positive because immI_bitmask guarantees it.
12918 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12919 %{
12920   match(Set dst (LShiftI (AndI src mask) lshift));
12921   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12922 
12923   ins_cost(INSN_COST);
12924   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12925   ins_encode %{
12926     int lshift = $lshift$$constant & 31;
12927     intptr_t mask = $mask$$constant;
12928     int width = exact_log2(mask+1);
12929     __ ubfizw(as_Register($dst$$reg),
12930           as_Register($src$$reg), lshift, width);
12931   %}
12932   ins_pipe(ialu_reg_shift);
12933 %}
12934 
12935 // This pattern is automatically generated from aarch64_ad.m4
12936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12937 
12938 // We can use ubfiz when masking by a positive number and then left shifting the result.
12939 // We know that the mask is positive because immL_bitmask guarantees it.
12940 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12941 %{
12942   match(Set dst (LShiftL (AndL src mask) lshift));
12943   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12944 
12945   ins_cost(INSN_COST);
12946   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12947   ins_encode %{
12948     int lshift = $lshift$$constant & 63;
12949     intptr_t mask = $mask$$constant;
12950     int width = exact_log2_long(mask+1);
12951     __ ubfiz(as_Register($dst$$reg),
12952           as_Register($src$$reg), lshift, width);
12953   %}
12954   ins_pipe(ialu_reg_shift);
12955 %}
12956 
12957 // This pattern is automatically generated from aarch64_ad.m4
12958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12959 
12960 // We can use ubfiz when masking by a positive number and then left shifting the result.
12961 // We know that the mask is positive because immI_bitmask guarantees it.
12962 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12963 %{
12964   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12965   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12966 
12967   ins_cost(INSN_COST);
12968   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12969   ins_encode %{
12970     int lshift = $lshift$$constant & 31;
12971     intptr_t mask = $mask$$constant;
12972     int width = exact_log2(mask+1);
12973     __ ubfizw(as_Register($dst$$reg),
12974           as_Register($src$$reg), lshift, width);
12975   %}
12976   ins_pipe(ialu_reg_shift);
12977 %}
12978 
12979 // This pattern is automatically generated from aarch64_ad.m4
12980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12981 
12982 // We can use ubfiz when masking by a positive number and then left shifting the result.
12983 // We know that the mask is positive because immL_bitmask guarantees it.
12984 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12985 %{
12986   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12987   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12988 
12989   ins_cost(INSN_COST);
12990   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12991   ins_encode %{
12992     int lshift = $lshift$$constant & 63;
12993     intptr_t mask = $mask$$constant;
12994     int width = exact_log2_long(mask+1);
12995     __ ubfiz(as_Register($dst$$reg),
12996           as_Register($src$$reg), lshift, width);
12997   %}
12998   ins_pipe(ialu_reg_shift);
12999 %}
13000 
13001 
13002 // This pattern is automatically generated from aarch64_ad.m4
13003 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13004 
13005 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13006 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13007 %{
13008   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13009   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13010 
13011   ins_cost(INSN_COST);
13012   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13013   ins_encode %{
13014     int lshift = $lshift$$constant & 63;
13015     intptr_t mask = $mask$$constant;
13016     int width = exact_log2(mask+1);
13017     __ ubfiz(as_Register($dst$$reg),
13018              as_Register($src$$reg), lshift, width);
13019   %}
13020   ins_pipe(ialu_reg_shift);
13021 %}
13022 
13023 // This pattern is automatically generated from aarch64_ad.m4
13024 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13025 
13026 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13027 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13028 %{
13029   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13030   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13031 
13032   ins_cost(INSN_COST);
13033   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13034   ins_encode %{
13035     int lshift = $lshift$$constant & 31;
13036     intptr_t mask = $mask$$constant;
13037     int width = exact_log2(mask+1);
13038     __ ubfiz(as_Register($dst$$reg),
13039              as_Register($src$$reg), lshift, width);
13040   %}
13041   ins_pipe(ialu_reg_shift);
13042 %}
13043 
13044 // This pattern is automatically generated from aarch64_ad.m4
13045 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13046 
13047 // Can skip int2long conversions after AND with small bitmask
13048 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13049 %{
13050   match(Set dst (ConvI2L (AndI src msk)));
13051   ins_cost(INSN_COST);
13052   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13053   ins_encode %{
13054     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13055   %}
13056   ins_pipe(ialu_reg_shift);
13057 %}
13058 
13059 
13060 // Rotations
13061 
13062 // This pattern is automatically generated from aarch64_ad.m4
13063 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13064 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13065 %{
13066   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13067   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13068 
13069   ins_cost(INSN_COST);
13070   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13071 
13072   ins_encode %{
13073     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13074             $rshift$$constant & 63);
13075   %}
13076   ins_pipe(ialu_reg_reg_extr);
13077 %}
13078 
13079 
13080 // This pattern is automatically generated from aarch64_ad.m4
13081 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13082 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13083 %{
13084   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13085   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13086 
13087   ins_cost(INSN_COST);
13088   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13089 
13090   ins_encode %{
13091     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13092             $rshift$$constant & 31);
13093   %}
13094   ins_pipe(ialu_reg_reg_extr);
13095 %}
13096 
13097 
13098 // This pattern is automatically generated from aarch64_ad.m4
13099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13100 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13101 %{
13102   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13103   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13104 
13105   ins_cost(INSN_COST);
13106   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13107 
13108   ins_encode %{
13109     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13110             $rshift$$constant & 63);
13111   %}
13112   ins_pipe(ialu_reg_reg_extr);
13113 %}
13114 
13115 
13116 // This pattern is automatically generated from aarch64_ad.m4
13117 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13118 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13119 %{
13120   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13121   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13122 
13123   ins_cost(INSN_COST);
13124   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13125 
13126   ins_encode %{
13127     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13128             $rshift$$constant & 31);
13129   %}
13130   ins_pipe(ialu_reg_reg_extr);
13131 %}
13132 
13133 // This pattern is automatically generated from aarch64_ad.m4
13134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13135 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13136 %{
13137   match(Set dst (RotateRight src shift));
13138 
13139   ins_cost(INSN_COST);
13140   format %{ "ror    $dst, $src, $shift" %}
13141 
13142   ins_encode %{
13143      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13144                $shift$$constant & 0x1f);
13145   %}
13146   ins_pipe(ialu_reg_reg_vshift);
13147 %}
13148 
13149 // This pattern is automatically generated from aarch64_ad.m4
13150 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13151 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13152 %{
13153   match(Set dst (RotateRight src shift));
13154 
13155   ins_cost(INSN_COST);
13156   format %{ "ror    $dst, $src, $shift" %}
13157 
13158   ins_encode %{
13159      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13160                $shift$$constant & 0x3f);
13161   %}
13162   ins_pipe(ialu_reg_reg_vshift);
13163 %}
13164 
13165 // This pattern is automatically generated from aarch64_ad.m4
13166 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13167 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13168 %{
13169   match(Set dst (RotateRight src shift));
13170 
13171   ins_cost(INSN_COST);
13172   format %{ "ror    $dst, $src, $shift" %}
13173 
13174   ins_encode %{
13175      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13176   %}
13177   ins_pipe(ialu_reg_reg_vshift);
13178 %}
13179 
13180 // This pattern is automatically generated from aarch64_ad.m4
13181 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13182 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13183 %{
13184   match(Set dst (RotateRight src shift));
13185 
13186   ins_cost(INSN_COST);
13187   format %{ "ror    $dst, $src, $shift" %}
13188 
13189   ins_encode %{
13190      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13191   %}
13192   ins_pipe(ialu_reg_reg_vshift);
13193 %}
13194 
13195 // This pattern is automatically generated from aarch64_ad.m4
13196 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13197 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13198 %{
13199   match(Set dst (RotateLeft src shift));
13200 
13201   ins_cost(INSN_COST);
13202   format %{ "rol    $dst, $src, $shift" %}
13203 
13204   ins_encode %{
13205      __ subw(rscratch1, zr, as_Register($shift$$reg));
13206      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13207   %}
13208   ins_pipe(ialu_reg_reg_vshift);
13209 %}
13210 
13211 // This pattern is automatically generated from aarch64_ad.m4
13212 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13213 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13214 %{
13215   match(Set dst (RotateLeft src shift));
13216 
13217   ins_cost(INSN_COST);
13218   format %{ "rol    $dst, $src, $shift" %}
13219 
13220   ins_encode %{
13221      __ subw(rscratch1, zr, as_Register($shift$$reg));
13222      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13223   %}
13224   ins_pipe(ialu_reg_reg_vshift);
13225 %}
13226 
13227 
13228 // Add/subtract (extended)
13229 
13230 // This pattern is automatically generated from aarch64_ad.m4
13231 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13232 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13233 %{
13234   match(Set dst (AddL src1 (ConvI2L src2)));
13235   ins_cost(INSN_COST);
13236   format %{ "add  $dst, $src1, $src2, sxtw" %}
13237 
13238    ins_encode %{
13239      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13240             as_Register($src2$$reg), ext::sxtw);
13241    %}
13242   ins_pipe(ialu_reg_reg);
13243 %}
13244 
13245 // This pattern is automatically generated from aarch64_ad.m4
13246 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13247 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13248 %{
13249   match(Set dst (SubL src1 (ConvI2L src2)));
13250   ins_cost(INSN_COST);
13251   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13252 
13253    ins_encode %{
13254      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13255             as_Register($src2$$reg), ext::sxtw);
13256    %}
13257   ins_pipe(ialu_reg_reg);
13258 %}
13259 
13260 // This pattern is automatically generated from aarch64_ad.m4
13261 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13262 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13263 %{
13264   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13265   ins_cost(INSN_COST);
13266   format %{ "add  $dst, $src1, $src2, sxth" %}
13267 
13268    ins_encode %{
13269      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13270             as_Register($src2$$reg), ext::sxth);
13271    %}
13272   ins_pipe(ialu_reg_reg);
13273 %}
13274 
13275 // This pattern is automatically generated from aarch64_ad.m4
13276 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13277 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13278 %{
13279   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13280   ins_cost(INSN_COST);
13281   format %{ "add  $dst, $src1, $src2, sxtb" %}
13282 
13283    ins_encode %{
13284      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13285             as_Register($src2$$reg), ext::sxtb);
13286    %}
13287   ins_pipe(ialu_reg_reg);
13288 %}
13289 
13290 // This pattern is automatically generated from aarch64_ad.m4
13291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13292 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13293 %{
13294   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13295   ins_cost(INSN_COST);
13296   format %{ "add  $dst, $src1, $src2, uxtb" %}
13297 
13298    ins_encode %{
13299      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13300             as_Register($src2$$reg), ext::uxtb);
13301    %}
13302   ins_pipe(ialu_reg_reg);
13303 %}
13304 
13305 // This pattern is automatically generated from aarch64_ad.m4
13306 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13307 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13308 %{
13309   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13310   ins_cost(INSN_COST);
13311   format %{ "add  $dst, $src1, $src2, sxth" %}
13312 
13313    ins_encode %{
13314      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13315             as_Register($src2$$reg), ext::sxth);
13316    %}
13317   ins_pipe(ialu_reg_reg);
13318 %}
13319 
13320 // This pattern is automatically generated from aarch64_ad.m4
13321 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13322 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13323 %{
13324   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13325   ins_cost(INSN_COST);
13326   format %{ "add  $dst, $src1, $src2, sxtw" %}
13327 
13328    ins_encode %{
13329      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13330             as_Register($src2$$reg), ext::sxtw);
13331    %}
13332   ins_pipe(ialu_reg_reg);
13333 %}
13334 
13335 // This pattern is automatically generated from aarch64_ad.m4
13336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13337 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13338 %{
13339   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13340   ins_cost(INSN_COST);
13341   format %{ "add  $dst, $src1, $src2, sxtb" %}
13342 
13343    ins_encode %{
13344      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13345             as_Register($src2$$reg), ext::sxtb);
13346    %}
13347   ins_pipe(ialu_reg_reg);
13348 %}
13349 
13350 // This pattern is automatically generated from aarch64_ad.m4
13351 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13352 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13353 %{
13354   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13355   ins_cost(INSN_COST);
13356   format %{ "add  $dst, $src1, $src2, uxtb" %}
13357 
13358    ins_encode %{
13359      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13360             as_Register($src2$$reg), ext::uxtb);
13361    %}
13362   ins_pipe(ialu_reg_reg);
13363 %}
13364 
13365 // This pattern is automatically generated from aarch64_ad.m4
13366 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13367 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13368 %{
13369   match(Set dst (AddI src1 (AndI src2 mask)));
13370   ins_cost(INSN_COST);
13371   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13372 
13373    ins_encode %{
13374      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13375             as_Register($src2$$reg), ext::uxtb);
13376    %}
13377   ins_pipe(ialu_reg_reg);
13378 %}
13379 
13380 // This pattern is automatically generated from aarch64_ad.m4
13381 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13382 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13383 %{
13384   match(Set dst (AddI src1 (AndI src2 mask)));
13385   ins_cost(INSN_COST);
13386   format %{ "addw  $dst, $src1, $src2, uxth" %}
13387 
13388    ins_encode %{
13389      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13390             as_Register($src2$$reg), ext::uxth);
13391    %}
13392   ins_pipe(ialu_reg_reg);
13393 %}
13394 
13395 // This pattern is automatically generated from aarch64_ad.m4
13396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13397 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13398 %{
13399   match(Set dst (AddL src1 (AndL src2 mask)));
13400   ins_cost(INSN_COST);
13401   format %{ "add  $dst, $src1, $src2, uxtb" %}
13402 
13403    ins_encode %{
13404      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13405             as_Register($src2$$reg), ext::uxtb);
13406    %}
13407   ins_pipe(ialu_reg_reg);
13408 %}
13409 
13410 // This pattern is automatically generated from aarch64_ad.m4
13411 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13412 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13413 %{
13414   match(Set dst (AddL src1 (AndL src2 mask)));
13415   ins_cost(INSN_COST);
13416   format %{ "add  $dst, $src1, $src2, uxth" %}
13417 
13418    ins_encode %{
13419      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13420             as_Register($src2$$reg), ext::uxth);
13421    %}
13422   ins_pipe(ialu_reg_reg);
13423 %}
13424 
13425 // This pattern is automatically generated from aarch64_ad.m4
13426 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13427 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13428 %{
13429   match(Set dst (AddL src1 (AndL src2 mask)));
13430   ins_cost(INSN_COST);
13431   format %{ "add  $dst, $src1, $src2, uxtw" %}
13432 
13433    ins_encode %{
13434      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13435             as_Register($src2$$reg), ext::uxtw);
13436    %}
13437   ins_pipe(ialu_reg_reg);
13438 %}
13439 
13440 // This pattern is automatically generated from aarch64_ad.m4
13441 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13442 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13443 %{
13444   match(Set dst (SubI src1 (AndI src2 mask)));
13445   ins_cost(INSN_COST);
13446   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13447 
13448    ins_encode %{
13449      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13450             as_Register($src2$$reg), ext::uxtb);
13451    %}
13452   ins_pipe(ialu_reg_reg);
13453 %}
13454 
13455 // This pattern is automatically generated from aarch64_ad.m4
13456 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13457 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13458 %{
13459   match(Set dst (SubI src1 (AndI src2 mask)));
13460   ins_cost(INSN_COST);
13461   format %{ "subw  $dst, $src1, $src2, uxth" %}
13462 
13463    ins_encode %{
13464      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13465             as_Register($src2$$reg), ext::uxth);
13466    %}
13467   ins_pipe(ialu_reg_reg);
13468 %}
13469 
13470 // This pattern is automatically generated from aarch64_ad.m4
13471 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13472 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13473 %{
13474   match(Set dst (SubL src1 (AndL src2 mask)));
13475   ins_cost(INSN_COST);
13476   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13477 
13478    ins_encode %{
13479      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13480             as_Register($src2$$reg), ext::uxtb);
13481    %}
13482   ins_pipe(ialu_reg_reg);
13483 %}
13484 
13485 // This pattern is automatically generated from aarch64_ad.m4
13486 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13487 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13488 %{
13489   match(Set dst (SubL src1 (AndL src2 mask)));
13490   ins_cost(INSN_COST);
13491   format %{ "sub  $dst, $src1, $src2, uxth" %}
13492 
13493    ins_encode %{
13494      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13495             as_Register($src2$$reg), ext::uxth);
13496    %}
13497   ins_pipe(ialu_reg_reg);
13498 %}
13499 
13500 // This pattern is automatically generated from aarch64_ad.m4
13501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13502 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13503 %{
13504   match(Set dst (SubL src1 (AndL src2 mask)));
13505   ins_cost(INSN_COST);
13506   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13507 
13508    ins_encode %{
13509      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13510             as_Register($src2$$reg), ext::uxtw);
13511    %}
13512   ins_pipe(ialu_reg_reg);
13513 %}
13514 
13515 
13516 // This pattern is automatically generated from aarch64_ad.m4
13517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13518 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13519 %{
13520   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13521   ins_cost(1.9 * INSN_COST);
13522   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13523 
13524    ins_encode %{
13525      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13526             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13527    %}
13528   ins_pipe(ialu_reg_reg_shift);
13529 %}
13530 
13531 // This pattern is automatically generated from aarch64_ad.m4
13532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13533 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13534 %{
13535   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13536   ins_cost(1.9 * INSN_COST);
13537   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13538 
13539    ins_encode %{
13540      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13541             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13542    %}
13543   ins_pipe(ialu_reg_reg_shift);
13544 %}
13545 
13546 // This pattern is automatically generated from aarch64_ad.m4
13547 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13548 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13549 %{
13550   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13551   ins_cost(1.9 * INSN_COST);
13552   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13553 
13554    ins_encode %{
13555      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13556             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13557    %}
13558   ins_pipe(ialu_reg_reg_shift);
13559 %}
13560 
13561 // This pattern is automatically generated from aarch64_ad.m4
13562 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13563 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13564 %{
13565   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13566   ins_cost(1.9 * INSN_COST);
13567   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13568 
13569    ins_encode %{
13570      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13571             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13572    %}
13573   ins_pipe(ialu_reg_reg_shift);
13574 %}
13575 
13576 // This pattern is automatically generated from aarch64_ad.m4
13577 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13578 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13579 %{
13580   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13581   ins_cost(1.9 * INSN_COST);
13582   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13583 
13584    ins_encode %{
13585      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13586             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13587    %}
13588   ins_pipe(ialu_reg_reg_shift);
13589 %}
13590 
13591 // This pattern is automatically generated from aarch64_ad.m4
13592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13593 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13594 %{
13595   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13596   ins_cost(1.9 * INSN_COST);
13597   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13598 
13599    ins_encode %{
13600      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13601             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13602    %}
13603   ins_pipe(ialu_reg_reg_shift);
13604 %}
13605 
13606 // This pattern is automatically generated from aarch64_ad.m4
13607 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13608 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13609 %{
13610   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13611   ins_cost(1.9 * INSN_COST);
13612   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13613 
13614    ins_encode %{
13615      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13616             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13617    %}
13618   ins_pipe(ialu_reg_reg_shift);
13619 %}
13620 
13621 // This pattern is automatically generated from aarch64_ad.m4
13622 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13623 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13624 %{
13625   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13626   ins_cost(1.9 * INSN_COST);
13627   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13628 
13629    ins_encode %{
13630      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13631             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13632    %}
13633   ins_pipe(ialu_reg_reg_shift);
13634 %}
13635 
13636 // This pattern is automatically generated from aarch64_ad.m4
13637 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13638 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13639 %{
13640   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13641   ins_cost(1.9 * INSN_COST);
13642   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13643 
13644    ins_encode %{
13645      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13646             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13647    %}
13648   ins_pipe(ialu_reg_reg_shift);
13649 %}
13650 
13651 // This pattern is automatically generated from aarch64_ad.m4
13652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13653 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13654 %{
13655   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13656   ins_cost(1.9 * INSN_COST);
13657   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13658 
13659    ins_encode %{
13660      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13661             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13662    %}
13663   ins_pipe(ialu_reg_reg_shift);
13664 %}
13665 
13666 // This pattern is automatically generated from aarch64_ad.m4
13667 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13668 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13669 %{
13670   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13671   ins_cost(1.9 * INSN_COST);
13672   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13673 
13674    ins_encode %{
13675      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13676             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13677    %}
13678   ins_pipe(ialu_reg_reg_shift);
13679 %}
13680 
13681 // This pattern is automatically generated from aarch64_ad.m4
13682 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13683 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13684 %{
13685   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13686   ins_cost(1.9 * INSN_COST);
13687   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13688 
13689    ins_encode %{
13690      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13691             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13692    %}
13693   ins_pipe(ialu_reg_reg_shift);
13694 %}
13695 
13696 // This pattern is automatically generated from aarch64_ad.m4
13697 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13698 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13699 %{
13700   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13701   ins_cost(1.9 * INSN_COST);
13702   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13703 
13704    ins_encode %{
13705      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13706             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13707    %}
13708   ins_pipe(ialu_reg_reg_shift);
13709 %}
13710 
13711 // This pattern is automatically generated from aarch64_ad.m4
13712 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13713 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13714 %{
13715   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13716   ins_cost(1.9 * INSN_COST);
13717   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13718 
13719    ins_encode %{
13720      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13721             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13722    %}
13723   ins_pipe(ialu_reg_reg_shift);
13724 %}
13725 
13726 // This pattern is automatically generated from aarch64_ad.m4
13727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13728 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13729 %{
13730   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13731   ins_cost(1.9 * INSN_COST);
13732   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13733 
13734    ins_encode %{
13735      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13736             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13737    %}
13738   ins_pipe(ialu_reg_reg_shift);
13739 %}
13740 
13741 // This pattern is automatically generated from aarch64_ad.m4
13742 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13743 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13744 %{
13745   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13746   ins_cost(1.9 * INSN_COST);
13747   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13748 
13749    ins_encode %{
13750      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13751             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13752    %}
13753   ins_pipe(ialu_reg_reg_shift);
13754 %}
13755 
13756 // This pattern is automatically generated from aarch64_ad.m4
13757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13758 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13759 %{
13760   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13761   ins_cost(1.9 * INSN_COST);
13762   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13763 
13764    ins_encode %{
13765      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13766             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13767    %}
13768   ins_pipe(ialu_reg_reg_shift);
13769 %}
13770 
13771 // This pattern is automatically generated from aarch64_ad.m4
13772 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13773 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13774 %{
13775   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13776   ins_cost(1.9 * INSN_COST);
13777   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13778 
13779    ins_encode %{
13780      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13781             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13782    %}
13783   ins_pipe(ialu_reg_reg_shift);
13784 %}
13785 
13786 // This pattern is automatically generated from aarch64_ad.m4
13787 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13788 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13789 %{
13790   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13791   ins_cost(1.9 * INSN_COST);
13792   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13793 
13794    ins_encode %{
13795      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13796             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13797    %}
13798   ins_pipe(ialu_reg_reg_shift);
13799 %}
13800 
13801 // This pattern is automatically generated from aarch64_ad.m4
13802 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13803 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13804 %{
13805   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13806   ins_cost(1.9 * INSN_COST);
13807   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13808 
13809    ins_encode %{
13810      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13811             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13812    %}
13813   ins_pipe(ialu_reg_reg_shift);
13814 %}
13815 
13816 // This pattern is automatically generated from aarch64_ad.m4
13817 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13818 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13819 %{
13820   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13821   ins_cost(1.9 * INSN_COST);
13822   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13823 
13824    ins_encode %{
13825      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13826             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13827    %}
13828   ins_pipe(ialu_reg_reg_shift);
13829 %}
13830 
13831 // This pattern is automatically generated from aarch64_ad.m4
13832 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13833 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13834 %{
13835   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13836   ins_cost(1.9 * INSN_COST);
13837   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13838 
13839    ins_encode %{
13840      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13841             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13842    %}
13843   ins_pipe(ialu_reg_reg_shift);
13844 %}
13845 
13846 // This pattern is automatically generated from aarch64_ad.m4
13847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13848 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13849 %{
13850   effect(DEF dst, USE src1, USE src2, USE cr);
13851   ins_cost(INSN_COST * 2);
13852   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
13853 
13854   ins_encode %{
13855     __ cselw($dst$$Register,
13856              $src1$$Register,
13857              $src2$$Register,
13858              Assembler::LT);
13859   %}
13860   ins_pipe(icond_reg_reg);
13861 %}
13862 
13863 // This pattern is automatically generated from aarch64_ad.m4
13864 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13865 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13866 %{
13867   effect(DEF dst, USE src1, USE src2, USE cr);
13868   ins_cost(INSN_COST * 2);
13869   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
13870 
13871   ins_encode %{
13872     __ cselw($dst$$Register,
13873              $src1$$Register,
13874              $src2$$Register,
13875              Assembler::GT);
13876   %}
13877   ins_pipe(icond_reg_reg);
13878 %}
13879 
13880 // This pattern is automatically generated from aarch64_ad.m4
13881 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13882 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13883 %{
13884   effect(DEF dst, USE src1, USE cr);
13885   ins_cost(INSN_COST * 2);
13886   format %{ "cselw $dst, $src1, zr lt\t"  %}
13887 
13888   ins_encode %{
13889     __ cselw($dst$$Register,
13890              $src1$$Register,
13891              zr,
13892              Assembler::LT);
13893   %}
13894   ins_pipe(icond_reg);
13895 %}
13896 
13897 // This pattern is automatically generated from aarch64_ad.m4
13898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13899 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13900 %{
13901   effect(DEF dst, USE src1, USE cr);
13902   ins_cost(INSN_COST * 2);
13903   format %{ "cselw $dst, $src1, zr gt\t"  %}
13904 
13905   ins_encode %{
13906     __ cselw($dst$$Register,
13907              $src1$$Register,
13908              zr,
13909              Assembler::GT);
13910   %}
13911   ins_pipe(icond_reg);
13912 %}
13913 
13914 // This pattern is automatically generated from aarch64_ad.m4
13915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13916 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13917 %{
13918   effect(DEF dst, USE src1, USE cr);
13919   ins_cost(INSN_COST * 2);
13920   format %{ "csincw $dst, $src1, zr le\t"  %}
13921 
13922   ins_encode %{
13923     __ csincw($dst$$Register,
13924              $src1$$Register,
13925              zr,
13926              Assembler::LE);
13927   %}
13928   ins_pipe(icond_reg);
13929 %}
13930 
13931 // This pattern is automatically generated from aarch64_ad.m4
13932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13933 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13934 %{
13935   effect(DEF dst, USE src1, USE cr);
13936   ins_cost(INSN_COST * 2);
13937   format %{ "csincw $dst, $src1, zr gt\t"  %}
13938 
13939   ins_encode %{
13940     __ csincw($dst$$Register,
13941              $src1$$Register,
13942              zr,
13943              Assembler::GT);
13944   %}
13945   ins_pipe(icond_reg);
13946 %}
13947 
13948 // This pattern is automatically generated from aarch64_ad.m4
13949 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13950 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13951 %{
13952   effect(DEF dst, USE src1, USE cr);
13953   ins_cost(INSN_COST * 2);
13954   format %{ "csinvw $dst, $src1, zr lt\t"  %}
13955 
13956   ins_encode %{
13957     __ csinvw($dst$$Register,
13958              $src1$$Register,
13959              zr,
13960              Assembler::LT);
13961   %}
13962   ins_pipe(icond_reg);
13963 %}
13964 
13965 // This pattern is automatically generated from aarch64_ad.m4
13966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13967 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13968 %{
13969   effect(DEF dst, USE src1, USE cr);
13970   ins_cost(INSN_COST * 2);
13971   format %{ "csinvw $dst, $src1, zr ge\t"  %}
13972 
13973   ins_encode %{
13974     __ csinvw($dst$$Register,
13975              $src1$$Register,
13976              zr,
13977              Assembler::GE);
13978   %}
13979   ins_pipe(icond_reg);
13980 %}
13981 
13982 // This pattern is automatically generated from aarch64_ad.m4
13983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13984 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13985 %{
13986   match(Set dst (MinI src imm));
13987   ins_cost(INSN_COST * 3);
13988   expand %{
13989     rFlagsReg cr;
13990     compI_reg_imm0(cr, src);
13991     cmovI_reg_imm0_lt(dst, src, cr);
13992   %}
13993 %}
13994 
13995 // This pattern is automatically generated from aarch64_ad.m4
13996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13997 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13998 %{
13999   match(Set dst (MinI imm src));
14000   ins_cost(INSN_COST * 3);
14001   expand %{
14002     rFlagsReg cr;
14003     compI_reg_imm0(cr, src);
14004     cmovI_reg_imm0_lt(dst, src, cr);
14005   %}
14006 %}
14007 
14008 // This pattern is automatically generated from aarch64_ad.m4
14009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14010 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
14011 %{
14012   match(Set dst (MinI src imm));
14013   ins_cost(INSN_COST * 3);
14014   expand %{
14015     rFlagsReg cr;
14016     compI_reg_imm0(cr, src);
14017     cmovI_reg_imm1_le(dst, src, cr);
14018   %}
14019 %}
14020 
14021 // This pattern is automatically generated from aarch64_ad.m4
14022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14023 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14024 %{
14025   match(Set dst (MinI imm src));
14026   ins_cost(INSN_COST * 3);
14027   expand %{
14028     rFlagsReg cr;
14029     compI_reg_imm0(cr, src);
14030     cmovI_reg_imm1_le(dst, src, cr);
14031   %}
14032 %}
14033 
14034 // This pattern is automatically generated from aarch64_ad.m4
14035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14036 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14037 %{
14038   match(Set dst (MinI src imm));
14039   ins_cost(INSN_COST * 3);
14040   expand %{
14041     rFlagsReg cr;
14042     compI_reg_imm0(cr, src);
14043     cmovI_reg_immM1_lt(dst, src, cr);
14044   %}
14045 %}
14046 
14047 // This pattern is automatically generated from aarch64_ad.m4
14048 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14049 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14050 %{
14051   match(Set dst (MinI imm src));
14052   ins_cost(INSN_COST * 3);
14053   expand %{
14054     rFlagsReg cr;
14055     compI_reg_imm0(cr, src);
14056     cmovI_reg_immM1_lt(dst, src, cr);
14057   %}
14058 %}
14059 
14060 // This pattern is automatically generated from aarch64_ad.m4
14061 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14062 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
14063 %{
14064   match(Set dst (MaxI src imm));
14065   ins_cost(INSN_COST * 3);
14066   expand %{
14067     rFlagsReg cr;
14068     compI_reg_imm0(cr, src);
14069     cmovI_reg_imm0_gt(dst, src, cr);
14070   %}
14071 %}
14072 
14073 // This pattern is automatically generated from aarch64_ad.m4
14074 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14075 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
14076 %{
14077   match(Set dst (MaxI imm src));
14078   ins_cost(INSN_COST * 3);
14079   expand %{
14080     rFlagsReg cr;
14081     compI_reg_imm0(cr, src);
14082     cmovI_reg_imm0_gt(dst, src, cr);
14083   %}
14084 %}
14085 
14086 // This pattern is automatically generated from aarch64_ad.m4
14087 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14088 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
14089 %{
14090   match(Set dst (MaxI src imm));
14091   ins_cost(INSN_COST * 3);
14092   expand %{
14093     rFlagsReg cr;
14094     compI_reg_imm0(cr, src);
14095     cmovI_reg_imm1_gt(dst, src, cr);
14096   %}
14097 %}
14098 
14099 // This pattern is automatically generated from aarch64_ad.m4
14100 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14101 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14102 %{
14103   match(Set dst (MaxI imm src));
14104   ins_cost(INSN_COST * 3);
14105   expand %{
14106     rFlagsReg cr;
14107     compI_reg_imm0(cr, src);
14108     cmovI_reg_imm1_gt(dst, src, cr);
14109   %}
14110 %}
14111 
14112 // This pattern is automatically generated from aarch64_ad.m4
14113 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14114 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14115 %{
14116   match(Set dst (MaxI src imm));
14117   ins_cost(INSN_COST * 3);
14118   expand %{
14119     rFlagsReg cr;
14120     compI_reg_imm0(cr, src);
14121     cmovI_reg_immM1_ge(dst, src, cr);
14122   %}
14123 %}
14124 
14125 // This pattern is automatically generated from aarch64_ad.m4
14126 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14127 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14128 %{
14129   match(Set dst (MaxI imm src));
14130   ins_cost(INSN_COST * 3);
14131   expand %{
14132     rFlagsReg cr;
14133     compI_reg_imm0(cr, src);
14134     cmovI_reg_immM1_ge(dst, src, cr);
14135   %}
14136 %}
14137 
14138 // This pattern is automatically generated from aarch64_ad.m4
14139 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14140 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
14141 %{
14142   match(Set dst (ReverseI src));
14143   ins_cost(INSN_COST);
14144   format %{ "rbitw  $dst, $src" %}
14145   ins_encode %{
14146     __ rbitw($dst$$Register, $src$$Register);
14147   %}
14148   ins_pipe(ialu_reg);
14149 %}
14150 
14151 // This pattern is automatically generated from aarch64_ad.m4
14152 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14153 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
14154 %{
14155   match(Set dst (ReverseL src));
14156   ins_cost(INSN_COST);
14157   format %{ "rbit  $dst, $src" %}
14158   ins_encode %{
14159     __ rbit($dst$$Register, $src$$Register);
14160   %}
14161   ins_pipe(ialu_reg);
14162 %}
14163 
14164 
14165 // END This section of the file is automatically generated. Do not edit --------------
14166 
14167 
14168 // ============================================================================
14169 // Floating Point Arithmetic Instructions
14170 
14171 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14172   match(Set dst (AddF src1 src2));
14173 
14174   ins_cost(INSN_COST * 5);
14175   format %{ "fadds   $dst, $src1, $src2" %}
14176 
14177   ins_encode %{
14178     __ fadds(as_FloatRegister($dst$$reg),
14179              as_FloatRegister($src1$$reg),
14180              as_FloatRegister($src2$$reg));
14181   %}
14182 
14183   ins_pipe(fp_dop_reg_reg_s);
14184 %}
14185 
14186 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14187   match(Set dst (AddD src1 src2));
14188 
14189   ins_cost(INSN_COST * 5);
14190   format %{ "faddd   $dst, $src1, $src2" %}
14191 
14192   ins_encode %{
14193     __ faddd(as_FloatRegister($dst$$reg),
14194              as_FloatRegister($src1$$reg),
14195              as_FloatRegister($src2$$reg));
14196   %}
14197 
14198   ins_pipe(fp_dop_reg_reg_d);
14199 %}
14200 
14201 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14202   match(Set dst (SubF src1 src2));
14203 
14204   ins_cost(INSN_COST * 5);
14205   format %{ "fsubs   $dst, $src1, $src2" %}
14206 
14207   ins_encode %{
14208     __ fsubs(as_FloatRegister($dst$$reg),
14209              as_FloatRegister($src1$$reg),
14210              as_FloatRegister($src2$$reg));
14211   %}
14212 
14213   ins_pipe(fp_dop_reg_reg_s);
14214 %}
14215 
14216 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14217   match(Set dst (SubD src1 src2));
14218 
14219   ins_cost(INSN_COST * 5);
14220   format %{ "fsubd   $dst, $src1, $src2" %}
14221 
14222   ins_encode %{
14223     __ fsubd(as_FloatRegister($dst$$reg),
14224              as_FloatRegister($src1$$reg),
14225              as_FloatRegister($src2$$reg));
14226   %}
14227 
14228   ins_pipe(fp_dop_reg_reg_d);
14229 %}
14230 
14231 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14232   match(Set dst (MulF src1 src2));
14233 
14234   ins_cost(INSN_COST * 6);
14235   format %{ "fmuls   $dst, $src1, $src2" %}
14236 
14237   ins_encode %{
14238     __ fmuls(as_FloatRegister($dst$$reg),
14239              as_FloatRegister($src1$$reg),
14240              as_FloatRegister($src2$$reg));
14241   %}
14242 
14243   ins_pipe(fp_dop_reg_reg_s);
14244 %}
14245 
14246 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14247   match(Set dst (MulD src1 src2));
14248 
14249   ins_cost(INSN_COST * 6);
14250   format %{ "fmuld   $dst, $src1, $src2" %}
14251 
14252   ins_encode %{
14253     __ fmuld(as_FloatRegister($dst$$reg),
14254              as_FloatRegister($src1$$reg),
14255              as_FloatRegister($src2$$reg));
14256   %}
14257 
14258   ins_pipe(fp_dop_reg_reg_d);
14259 %}
14260 
14261 // src1 * src2 + src3
14262 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14263   predicate(UseFMA);
14264   match(Set dst (FmaF src3 (Binary src1 src2)));
14265 
14266   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
14267 
14268   ins_encode %{
14269     __ fmadds(as_FloatRegister($dst$$reg),
14270              as_FloatRegister($src1$$reg),
14271              as_FloatRegister($src2$$reg),
14272              as_FloatRegister($src3$$reg));
14273   %}
14274 
14275   ins_pipe(pipe_class_default);
14276 %}
14277 
14278 // src1 * src2 + src3
14279 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14280   predicate(UseFMA);
14281   match(Set dst (FmaD src3 (Binary src1 src2)));
14282 
14283   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
14284 
14285   ins_encode %{
14286     __ fmaddd(as_FloatRegister($dst$$reg),
14287              as_FloatRegister($src1$$reg),
14288              as_FloatRegister($src2$$reg),
14289              as_FloatRegister($src3$$reg));
14290   %}
14291 
14292   ins_pipe(pipe_class_default);
14293 %}
14294 
14295 // -src1 * src2 + src3
14296 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14297   predicate(UseFMA);
14298   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14299   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14300 
14301   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
14302 
14303   ins_encode %{
14304     __ fmsubs(as_FloatRegister($dst$$reg),
14305               as_FloatRegister($src1$$reg),
14306               as_FloatRegister($src2$$reg),
14307               as_FloatRegister($src3$$reg));
14308   %}
14309 
14310   ins_pipe(pipe_class_default);
14311 %}
14312 
14313 // -src1 * src2 + src3
14314 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14315   predicate(UseFMA);
14316   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14317   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14318 
14319   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
14320 
14321   ins_encode %{
14322     __ fmsubd(as_FloatRegister($dst$$reg),
14323               as_FloatRegister($src1$$reg),
14324               as_FloatRegister($src2$$reg),
14325               as_FloatRegister($src3$$reg));
14326   %}
14327 
14328   ins_pipe(pipe_class_default);
14329 %}
14330 
14331 // -src1 * src2 - src3
14332 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14333   predicate(UseFMA);
14334   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14335   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14336 
14337   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14338 
14339   ins_encode %{
14340     __ fnmadds(as_FloatRegister($dst$$reg),
14341                as_FloatRegister($src1$$reg),
14342                as_FloatRegister($src2$$reg),
14343                as_FloatRegister($src3$$reg));
14344   %}
14345 
14346   ins_pipe(pipe_class_default);
14347 %}
14348 
14349 // -src1 * src2 - src3
14350 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14351   predicate(UseFMA);
14352   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14353   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14354 
14355   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14356 
14357   ins_encode %{
14358     __ fnmaddd(as_FloatRegister($dst$$reg),
14359                as_FloatRegister($src1$$reg),
14360                as_FloatRegister($src2$$reg),
14361                as_FloatRegister($src3$$reg));
14362   %}
14363 
14364   ins_pipe(pipe_class_default);
14365 %}
14366 
14367 // src1 * src2 - src3
14368 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14369   predicate(UseFMA);
14370   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14371 
14372   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14373 
14374   ins_encode %{
14375     __ fnmsubs(as_FloatRegister($dst$$reg),
14376                as_FloatRegister($src1$$reg),
14377                as_FloatRegister($src2$$reg),
14378                as_FloatRegister($src3$$reg));
14379   %}
14380 
14381   ins_pipe(pipe_class_default);
14382 %}
14383 
14384 // src1 * src2 - src3
14385 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14386   predicate(UseFMA);
14387   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14388 
14389   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14390 
14391   ins_encode %{
14392   // n.b. insn name should be fnmsubd
14393     __ fnmsub(as_FloatRegister($dst$$reg),
14394               as_FloatRegister($src1$$reg),
14395               as_FloatRegister($src2$$reg),
14396               as_FloatRegister($src3$$reg));
14397   %}
14398 
14399   ins_pipe(pipe_class_default);
14400 %}
14401 
14402 
14403 // Math.max(FF)F
14404 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14405   match(Set dst (MaxF src1 src2));
14406 
14407   format %{ "fmaxs   $dst, $src1, $src2" %}
14408   ins_encode %{
14409     __ fmaxs(as_FloatRegister($dst$$reg),
14410              as_FloatRegister($src1$$reg),
14411              as_FloatRegister($src2$$reg));
14412   %}
14413 
14414   ins_pipe(fp_dop_reg_reg_s);
14415 %}
14416 
14417 // Math.min(FF)F
14418 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14419   match(Set dst (MinF src1 src2));
14420 
14421   format %{ "fmins   $dst, $src1, $src2" %}
14422   ins_encode %{
14423     __ fmins(as_FloatRegister($dst$$reg),
14424              as_FloatRegister($src1$$reg),
14425              as_FloatRegister($src2$$reg));
14426   %}
14427 
14428   ins_pipe(fp_dop_reg_reg_s);
14429 %}
14430 
14431 // Math.max(DD)D
14432 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14433   match(Set dst (MaxD src1 src2));
14434 
14435   format %{ "fmaxd   $dst, $src1, $src2" %}
14436   ins_encode %{
14437     __ fmaxd(as_FloatRegister($dst$$reg),
14438              as_FloatRegister($src1$$reg),
14439              as_FloatRegister($src2$$reg));
14440   %}
14441 
14442   ins_pipe(fp_dop_reg_reg_d);
14443 %}
14444 
14445 // Math.min(DD)D
14446 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14447   match(Set dst (MinD src1 src2));
14448 
14449   format %{ "fmind   $dst, $src1, $src2" %}
14450   ins_encode %{
14451     __ fmind(as_FloatRegister($dst$$reg),
14452              as_FloatRegister($src1$$reg),
14453              as_FloatRegister($src2$$reg));
14454   %}
14455 
14456   ins_pipe(fp_dop_reg_reg_d);
14457 %}
14458 
14459 
14460 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14461   match(Set dst (DivF src1  src2));
14462 
14463   ins_cost(INSN_COST * 18);
14464   format %{ "fdivs   $dst, $src1, $src2" %}
14465 
14466   ins_encode %{
14467     __ fdivs(as_FloatRegister($dst$$reg),
14468              as_FloatRegister($src1$$reg),
14469              as_FloatRegister($src2$$reg));
14470   %}
14471 
14472   ins_pipe(fp_div_s);
14473 %}
14474 
14475 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14476   match(Set dst (DivD src1  src2));
14477 
14478   ins_cost(INSN_COST * 32);
14479   format %{ "fdivd   $dst, $src1, $src2" %}
14480 
14481   ins_encode %{
14482     __ fdivd(as_FloatRegister($dst$$reg),
14483              as_FloatRegister($src1$$reg),
14484              as_FloatRegister($src2$$reg));
14485   %}
14486 
14487   ins_pipe(fp_div_d);
14488 %}
14489 
14490 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14491   match(Set dst (NegF src));
14492 
14493   ins_cost(INSN_COST * 3);
14494   format %{ "fneg   $dst, $src" %}
14495 
14496   ins_encode %{
14497     __ fnegs(as_FloatRegister($dst$$reg),
14498              as_FloatRegister($src$$reg));
14499   %}
14500 
14501   ins_pipe(fp_uop_s);
14502 %}
14503 
14504 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14505   match(Set dst (NegD src));
14506 
14507   ins_cost(INSN_COST * 3);
14508   format %{ "fnegd   $dst, $src" %}
14509 
14510   ins_encode %{
14511     __ fnegd(as_FloatRegister($dst$$reg),
14512              as_FloatRegister($src$$reg));
14513   %}
14514 
14515   ins_pipe(fp_uop_d);
14516 %}
14517 
14518 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14519 %{
14520   match(Set dst (AbsI src));
14521 
14522   effect(KILL cr);
14523   ins_cost(INSN_COST * 2);
14524   format %{ "cmpw  $src, zr\n\t"
14525             "cnegw $dst, $src, Assembler::LT\t# int abs"
14526   %}
14527 
14528   ins_encode %{
14529     __ cmpw(as_Register($src$$reg), zr);
14530     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14531   %}
14532   ins_pipe(pipe_class_default);
14533 %}
14534 
14535 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14536 %{
14537   match(Set dst (AbsL src));
14538 
14539   effect(KILL cr);
14540   ins_cost(INSN_COST * 2);
14541   format %{ "cmp  $src, zr\n\t"
14542             "cneg $dst, $src, Assembler::LT\t# long abs"
14543   %}
14544 
14545   ins_encode %{
14546     __ cmp(as_Register($src$$reg), zr);
14547     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14548   %}
14549   ins_pipe(pipe_class_default);
14550 %}
14551 
14552 instruct absF_reg(vRegF dst, vRegF src) %{
14553   match(Set dst (AbsF src));
14554 
14555   ins_cost(INSN_COST * 3);
14556   format %{ "fabss   $dst, $src" %}
14557   ins_encode %{
14558     __ fabss(as_FloatRegister($dst$$reg),
14559              as_FloatRegister($src$$reg));
14560   %}
14561 
14562   ins_pipe(fp_uop_s);
14563 %}
14564 
14565 instruct absD_reg(vRegD dst, vRegD src) %{
14566   match(Set dst (AbsD src));
14567 
14568   ins_cost(INSN_COST * 3);
14569   format %{ "fabsd   $dst, $src" %}
14570   ins_encode %{
14571     __ fabsd(as_FloatRegister($dst$$reg),
14572              as_FloatRegister($src$$reg));
14573   %}
14574 
14575   ins_pipe(fp_uop_d);
14576 %}
14577 
14578 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14579   match(Set dst (AbsF (SubF src1 src2)));
14580 
14581   ins_cost(INSN_COST * 3);
14582   format %{ "fabds   $dst, $src1, $src2" %}
14583   ins_encode %{
14584     __ fabds(as_FloatRegister($dst$$reg),
14585              as_FloatRegister($src1$$reg),
14586              as_FloatRegister($src2$$reg));
14587   %}
14588 
14589   ins_pipe(fp_uop_s);
14590 %}
14591 
14592 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14593   match(Set dst (AbsD (SubD src1 src2)));
14594 
14595   ins_cost(INSN_COST * 3);
14596   format %{ "fabdd   $dst, $src1, $src2" %}
14597   ins_encode %{
14598     __ fabdd(as_FloatRegister($dst$$reg),
14599              as_FloatRegister($src1$$reg),
14600              as_FloatRegister($src2$$reg));
14601   %}
14602 
14603   ins_pipe(fp_uop_d);
14604 %}
14605 
14606 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14607   match(Set dst (SqrtD src));
14608 
14609   ins_cost(INSN_COST * 50);
14610   format %{ "fsqrtd  $dst, $src" %}
14611   ins_encode %{
14612     __ fsqrtd(as_FloatRegister($dst$$reg),
14613              as_FloatRegister($src$$reg));
14614   %}
14615 
14616   ins_pipe(fp_div_s);
14617 %}
14618 
14619 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14620   match(Set dst (SqrtF src));
14621 
14622   ins_cost(INSN_COST * 50);
14623   format %{ "fsqrts  $dst, $src" %}
14624   ins_encode %{
14625     __ fsqrts(as_FloatRegister($dst$$reg),
14626              as_FloatRegister($src$$reg));
14627   %}
14628 
14629   ins_pipe(fp_div_d);
14630 %}
14631 
14632 // Math.rint, floor, ceil
14633 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14634   match(Set dst (RoundDoubleMode src rmode));
14635   format %{ "frint  $dst, $src, $rmode" %}
14636   ins_encode %{
14637     switch ($rmode$$constant) {
14638       case RoundDoubleModeNode::rmode_rint:
14639         __ frintnd(as_FloatRegister($dst$$reg),
14640                    as_FloatRegister($src$$reg));
14641         break;
14642       case RoundDoubleModeNode::rmode_floor:
14643         __ frintmd(as_FloatRegister($dst$$reg),
14644                    as_FloatRegister($src$$reg));
14645         break;
14646       case RoundDoubleModeNode::rmode_ceil:
14647         __ frintpd(as_FloatRegister($dst$$reg),
14648                    as_FloatRegister($src$$reg));
14649         break;
14650     }
14651   %}
14652   ins_pipe(fp_uop_d);
14653 %}
14654 
14655 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14656   match(Set dst (CopySignD src1 (Binary src2 zero)));
14657   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14658   format %{ "CopySignD  $dst $src1 $src2" %}
14659   ins_encode %{
14660     FloatRegister dst = as_FloatRegister($dst$$reg),
14661                   src1 = as_FloatRegister($src1$$reg),
14662                   src2 = as_FloatRegister($src2$$reg),
14663                   zero = as_FloatRegister($zero$$reg);
14664     __ fnegd(dst, zero);
14665     __ bsl(dst, __ T8B, src2, src1);
14666   %}
14667   ins_pipe(fp_uop_d);
14668 %}
14669 
14670 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14671   match(Set dst (CopySignF src1 src2));
14672   effect(TEMP_DEF dst, USE src1, USE src2);
14673   format %{ "CopySignF  $dst $src1 $src2" %}
14674   ins_encode %{
14675     FloatRegister dst = as_FloatRegister($dst$$reg),
14676                   src1 = as_FloatRegister($src1$$reg),
14677                   src2 = as_FloatRegister($src2$$reg);
14678     __ movi(dst, __ T2S, 0x80, 24);
14679     __ bsl(dst, __ T8B, src2, src1);
14680   %}
14681   ins_pipe(fp_uop_d);
14682 %}
14683 
14684 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14685   match(Set dst (SignumD src (Binary zero one)));
14686   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14687   format %{ "signumD  $dst, $src" %}
14688   ins_encode %{
14689     FloatRegister src = as_FloatRegister($src$$reg),
14690                   dst = as_FloatRegister($dst$$reg),
14691                   zero = as_FloatRegister($zero$$reg),
14692                   one = as_FloatRegister($one$$reg);
14693     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14694     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14695     // Bit selection instruction gets bit from "one" for each enabled bit in
14696     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14697     // NaN the whole "src" will be copied because "dst" is zero. For all other
14698     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14699     // from "src", and all other bits are copied from 1.0.
14700     __ bsl(dst, __ T8B, one, src);
14701   %}
14702   ins_pipe(fp_uop_d);
14703 %}
14704 
14705 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14706   match(Set dst (SignumF src (Binary zero one)));
14707   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14708   format %{ "signumF  $dst, $src" %}
14709   ins_encode %{
14710     FloatRegister src = as_FloatRegister($src$$reg),
14711                   dst = as_FloatRegister($dst$$reg),
14712                   zero = as_FloatRegister($zero$$reg),
14713                   one = as_FloatRegister($one$$reg);
14714     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14715     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14716     // Bit selection instruction gets bit from "one" for each enabled bit in
14717     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14718     // NaN the whole "src" will be copied because "dst" is zero. For all other
14719     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14720     // from "src", and all other bits are copied from 1.0.
14721     __ bsl(dst, __ T8B, one, src);
14722   %}
14723   ins_pipe(fp_uop_d);
14724 %}
14725 
14726 instruct onspinwait() %{
14727   match(OnSpinWait);
14728   ins_cost(INSN_COST);
14729 
14730   format %{ "onspinwait" %}
14731 
14732   ins_encode %{
14733     __ spin_wait();
14734   %}
14735   ins_pipe(pipe_class_empty);
14736 %}
14737 
14738 // ============================================================================
14739 // Logical Instructions
14740 
14741 // Integer Logical Instructions
14742 
14743 // And Instructions
14744 
14745 
14746 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14747   match(Set dst (AndI src1 src2));
14748 
14749   format %{ "andw  $dst, $src1, $src2\t# int" %}
14750 
14751   ins_cost(INSN_COST);
14752   ins_encode %{
14753     __ andw(as_Register($dst$$reg),
14754             as_Register($src1$$reg),
14755             as_Register($src2$$reg));
14756   %}
14757 
14758   ins_pipe(ialu_reg_reg);
14759 %}
14760 
14761 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14762   match(Set dst (AndI src1 src2));
14763 
14764   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14765 
14766   ins_cost(INSN_COST);
14767   ins_encode %{
14768     __ andw(as_Register($dst$$reg),
14769             as_Register($src1$$reg),
14770             (uint64_t)($src2$$constant));
14771   %}
14772 
14773   ins_pipe(ialu_reg_imm);
14774 %}
14775 
14776 // Or Instructions
14777 
14778 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14779   match(Set dst (OrI src1 src2));
14780 
14781   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14782 
14783   ins_cost(INSN_COST);
14784   ins_encode %{
14785     __ orrw(as_Register($dst$$reg),
14786             as_Register($src1$$reg),
14787             as_Register($src2$$reg));
14788   %}
14789 
14790   ins_pipe(ialu_reg_reg);
14791 %}
14792 
14793 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14794   match(Set dst (OrI src1 src2));
14795 
14796   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14797 
14798   ins_cost(INSN_COST);
14799   ins_encode %{
14800     __ orrw(as_Register($dst$$reg),
14801             as_Register($src1$$reg),
14802             (uint64_t)($src2$$constant));
14803   %}
14804 
14805   ins_pipe(ialu_reg_imm);
14806 %}
14807 
14808 // Xor Instructions
14809 
14810 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14811   match(Set dst (XorI src1 src2));
14812 
14813   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14814 
14815   ins_cost(INSN_COST);
14816   ins_encode %{
14817     __ eorw(as_Register($dst$$reg),
14818             as_Register($src1$$reg),
14819             as_Register($src2$$reg));
14820   %}
14821 
14822   ins_pipe(ialu_reg_reg);
14823 %}
14824 
14825 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14826   match(Set dst (XorI src1 src2));
14827 
14828   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14829 
14830   ins_cost(INSN_COST);
14831   ins_encode %{
14832     __ eorw(as_Register($dst$$reg),
14833             as_Register($src1$$reg),
14834             (uint64_t)($src2$$constant));
14835   %}
14836 
14837   ins_pipe(ialu_reg_imm);
14838 %}
14839 
14840 // Long Logical Instructions
14841 // TODO
14842 
14843 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14844   match(Set dst (AndL src1 src2));
14845 
14846   format %{ "and  $dst, $src1, $src2\t# int" %}
14847 
14848   ins_cost(INSN_COST);
14849   ins_encode %{
14850     __ andr(as_Register($dst$$reg),
14851             as_Register($src1$$reg),
14852             as_Register($src2$$reg));
14853   %}
14854 
14855   ins_pipe(ialu_reg_reg);
14856 %}
14857 
14858 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14859   match(Set dst (AndL src1 src2));
14860 
14861   format %{ "and  $dst, $src1, $src2\t# int" %}
14862 
14863   ins_cost(INSN_COST);
14864   ins_encode %{
14865     __ andr(as_Register($dst$$reg),
14866             as_Register($src1$$reg),
14867             (uint64_t)($src2$$constant));
14868   %}
14869 
14870   ins_pipe(ialu_reg_imm);
14871 %}
14872 
14873 // Or Instructions
14874 
14875 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14876   match(Set dst (OrL src1 src2));
14877 
14878   format %{ "orr  $dst, $src1, $src2\t# int" %}
14879 
14880   ins_cost(INSN_COST);
14881   ins_encode %{
14882     __ orr(as_Register($dst$$reg),
14883            as_Register($src1$$reg),
14884            as_Register($src2$$reg));
14885   %}
14886 
14887   ins_pipe(ialu_reg_reg);
14888 %}
14889 
14890 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14891   match(Set dst (OrL src1 src2));
14892 
14893   format %{ "orr  $dst, $src1, $src2\t# int" %}
14894 
14895   ins_cost(INSN_COST);
14896   ins_encode %{
14897     __ orr(as_Register($dst$$reg),
14898            as_Register($src1$$reg),
14899            (uint64_t)($src2$$constant));
14900   %}
14901 
14902   ins_pipe(ialu_reg_imm);
14903 %}
14904 
14905 // Xor Instructions
14906 
14907 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14908   match(Set dst (XorL src1 src2));
14909 
14910   format %{ "eor  $dst, $src1, $src2\t# int" %}
14911 
14912   ins_cost(INSN_COST);
14913   ins_encode %{
14914     __ eor(as_Register($dst$$reg),
14915            as_Register($src1$$reg),
14916            as_Register($src2$$reg));
14917   %}
14918 
14919   ins_pipe(ialu_reg_reg);
14920 %}
14921 
14922 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14923   match(Set dst (XorL src1 src2));
14924 
14925   ins_cost(INSN_COST);
14926   format %{ "eor  $dst, $src1, $src2\t# int" %}
14927 
14928   ins_encode %{
14929     __ eor(as_Register($dst$$reg),
14930            as_Register($src1$$reg),
14931            (uint64_t)($src2$$constant));
14932   %}
14933 
14934   ins_pipe(ialu_reg_imm);
14935 %}
14936 
14937 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14938 %{
14939   match(Set dst (ConvI2L src));
14940 
14941   ins_cost(INSN_COST);
14942   format %{ "sxtw  $dst, $src\t# i2l" %}
14943   ins_encode %{
14944     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14945   %}
14946   ins_pipe(ialu_reg_shift);
14947 %}
14948 
14949 // this pattern occurs in bigmath arithmetic
14950 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14951 %{
14952   match(Set dst (AndL (ConvI2L src) mask));
14953 
14954   ins_cost(INSN_COST);
14955   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14956   ins_encode %{
14957     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14958   %}
14959 
14960   ins_pipe(ialu_reg_shift);
14961 %}
14962 
14963 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14964   match(Set dst (ConvL2I src));
14965 
14966   ins_cost(INSN_COST);
14967   format %{ "movw  $dst, $src \t// l2i" %}
14968 
14969   ins_encode %{
14970     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14971   %}
14972 
14973   ins_pipe(ialu_reg);
14974 %}
14975 
14976 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14977 %{
14978   match(Set dst (Conv2B src));
14979   effect(KILL cr);
14980 
14981   format %{
14982     "cmpw $src, zr\n\t"
14983     "cset $dst, ne"
14984   %}
14985 
14986   ins_encode %{
14987     __ cmpw(as_Register($src$$reg), zr);
14988     __ cset(as_Register($dst$$reg), Assembler::NE);
14989   %}
14990 
14991   ins_pipe(ialu_reg);
14992 %}
14993 
14994 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14995 %{
14996   match(Set dst (Conv2B src));
14997   effect(KILL cr);
14998 
14999   format %{
15000     "cmp  $src, zr\n\t"
15001     "cset $dst, ne"
15002   %}
15003 
15004   ins_encode %{
15005     __ cmp(as_Register($src$$reg), zr);
15006     __ cset(as_Register($dst$$reg), Assembler::NE);
15007   %}
15008 
15009   ins_pipe(ialu_reg);
15010 %}
15011 
15012 instruct convD2F_reg(vRegF dst, vRegD src) %{
15013   match(Set dst (ConvD2F src));
15014 
15015   ins_cost(INSN_COST * 5);
15016   format %{ "fcvtd  $dst, $src \t// d2f" %}
15017 
15018   ins_encode %{
15019     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15020   %}
15021 
15022   ins_pipe(fp_d2f);
15023 %}
15024 
15025 instruct convF2D_reg(vRegD dst, vRegF src) %{
15026   match(Set dst (ConvF2D src));
15027 
15028   ins_cost(INSN_COST * 5);
15029   format %{ "fcvts  $dst, $src \t// f2d" %}
15030 
15031   ins_encode %{
15032     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15033   %}
15034 
15035   ins_pipe(fp_f2d);
15036 %}
15037 
15038 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15039   match(Set dst (ConvF2I src));
15040 
15041   ins_cost(INSN_COST * 5);
15042   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
15043 
15044   ins_encode %{
15045     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15046   %}
15047 
15048   ins_pipe(fp_f2i);
15049 %}
15050 
15051 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
15052   match(Set dst (ConvF2L src));
15053 
15054   ins_cost(INSN_COST * 5);
15055   format %{ "fcvtzs  $dst, $src \t// f2l" %}
15056 
15057   ins_encode %{
15058     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15059   %}
15060 
15061   ins_pipe(fp_f2l);
15062 %}
15063 
15064 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
15065   match(Set dst (ConvF2HF src));
15066   format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
15067             "smov $dst, $tmp\t# move result from $tmp to $dst"
15068   %}
15069   effect(TEMP tmp);
15070   ins_encode %{
15071       __ fcvtsh($tmp$$FloatRegister, $src$$FloatRegister);
15072       __ smov($dst$$Register, $tmp$$FloatRegister, __ H, 0);
15073   %}
15074   ins_pipe(pipe_slow);
15075 %}
15076 
15077 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
15078   match(Set dst (ConvHF2F src));
15079   format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
15080             "fcvt $dst, $tmp\t# convert half to single precision"
15081   %}
15082   effect(TEMP tmp);
15083   ins_encode %{
15084       __ mov($tmp$$FloatRegister, __ H, 0, $src$$Register);
15085       __ fcvths($dst$$FloatRegister, $tmp$$FloatRegister);
15086   %}
15087   ins_pipe(pipe_slow);
15088 %}
15089 
15090 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
15091   match(Set dst (ConvI2F src));
15092 
15093   ins_cost(INSN_COST * 5);
15094   format %{ "scvtfws  $dst, $src \t// i2f" %}
15095 
15096   ins_encode %{
15097     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15098   %}
15099 
15100   ins_pipe(fp_i2f);
15101 %}
15102 
15103 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
15104   match(Set dst (ConvL2F src));
15105 
15106   ins_cost(INSN_COST * 5);
15107   format %{ "scvtfs  $dst, $src \t// l2f" %}
15108 
15109   ins_encode %{
15110     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15111   %}
15112 
15113   ins_pipe(fp_l2f);
15114 %}
15115 
15116 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
15117   match(Set dst (ConvD2I src));
15118 
15119   ins_cost(INSN_COST * 5);
15120   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
15121 
15122   ins_encode %{
15123     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15124   %}
15125 
15126   ins_pipe(fp_d2i);
15127 %}
15128 
15129 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15130   match(Set dst (ConvD2L src));
15131 
15132   ins_cost(INSN_COST * 5);
15133   format %{ "fcvtzd  $dst, $src \t// d2l" %}
15134 
15135   ins_encode %{
15136     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15137   %}
15138 
15139   ins_pipe(fp_d2l);
15140 %}
15141 
15142 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
15143   match(Set dst (ConvI2D src));
15144 
15145   ins_cost(INSN_COST * 5);
15146   format %{ "scvtfwd  $dst, $src \t// i2d" %}
15147 
15148   ins_encode %{
15149     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15150   %}
15151 
15152   ins_pipe(fp_i2d);
15153 %}
15154 
15155 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
15156   match(Set dst (ConvL2D src));
15157 
15158   ins_cost(INSN_COST * 5);
15159   format %{ "scvtfd  $dst, $src \t// l2d" %}
15160 
15161   ins_encode %{
15162     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15163   %}
15164 
15165   ins_pipe(fp_l2d);
15166 %}
15167 
15168 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
15169 %{
15170   match(Set dst (RoundD src));
15171   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15172   format %{ "java_round_double $dst,$src"%}
15173   ins_encode %{
15174     __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
15175                          as_FloatRegister($ftmp$$reg));
15176   %}
15177   ins_pipe(pipe_slow);
15178 %}
15179 
15180 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
15181 %{
15182   match(Set dst (RoundF src));
15183   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15184   format %{ "java_round_float $dst,$src"%}
15185   ins_encode %{
15186     __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
15187                         as_FloatRegister($ftmp$$reg));
15188   %}
15189   ins_pipe(pipe_slow);
15190 %}
15191 
15192 // stack <-> reg and reg <-> reg shuffles with no conversion
15193 
15194 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15195 
15196   match(Set dst (MoveF2I src));
15197 
15198   effect(DEF dst, USE src);
15199 
15200   ins_cost(4 * INSN_COST);
15201 
15202   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15203 
15204   ins_encode %{
15205     __ ldrw($dst$$Register, Address(sp, $src$$disp));
15206   %}
15207 
15208   ins_pipe(iload_reg_reg);
15209 
15210 %}
15211 
15212 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15213 
15214   match(Set dst (MoveI2F src));
15215 
15216   effect(DEF dst, USE src);
15217 
15218   ins_cost(4 * INSN_COST);
15219 
15220   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15221 
15222   ins_encode %{
15223     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15224   %}
15225 
15226   ins_pipe(pipe_class_memory);
15227 
15228 %}
15229 
15230 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15231 
15232   match(Set dst (MoveD2L src));
15233 
15234   effect(DEF dst, USE src);
15235 
15236   ins_cost(4 * INSN_COST);
15237 
15238   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15239 
15240   ins_encode %{
15241     __ ldr($dst$$Register, Address(sp, $src$$disp));
15242   %}
15243 
15244   ins_pipe(iload_reg_reg);
15245 
15246 %}
15247 
15248 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15249 
15250   match(Set dst (MoveL2D src));
15251 
15252   effect(DEF dst, USE src);
15253 
15254   ins_cost(4 * INSN_COST);
15255 
15256   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15257 
15258   ins_encode %{
15259     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15260   %}
15261 
15262   ins_pipe(pipe_class_memory);
15263 
15264 %}
15265 
15266 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15267 
15268   match(Set dst (MoveF2I src));
15269 
15270   effect(DEF dst, USE src);
15271 
15272   ins_cost(INSN_COST);
15273 
15274   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15275 
15276   ins_encode %{
15277     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15278   %}
15279 
15280   ins_pipe(pipe_class_memory);
15281 
15282 %}
15283 
15284 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15285 
15286   match(Set dst (MoveI2F src));
15287 
15288   effect(DEF dst, USE src);
15289 
15290   ins_cost(INSN_COST);
15291 
15292   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15293 
15294   ins_encode %{
15295     __ strw($src$$Register, Address(sp, $dst$$disp));
15296   %}
15297 
15298   ins_pipe(istore_reg_reg);
15299 
15300 %}
15301 
15302 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15303 
15304   match(Set dst (MoveD2L src));
15305 
15306   effect(DEF dst, USE src);
15307 
15308   ins_cost(INSN_COST);
15309 
15310   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15311 
15312   ins_encode %{
15313     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15314   %}
15315 
15316   ins_pipe(pipe_class_memory);
15317 
15318 %}
15319 
15320 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15321 
15322   match(Set dst (MoveL2D src));
15323 
15324   effect(DEF dst, USE src);
15325 
15326   ins_cost(INSN_COST);
15327 
15328   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15329 
15330   ins_encode %{
15331     __ str($src$$Register, Address(sp, $dst$$disp));
15332   %}
15333 
15334   ins_pipe(istore_reg_reg);
15335 
15336 %}
15337 
15338 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15339 
15340   match(Set dst (MoveF2I src));
15341 
15342   effect(DEF dst, USE src);
15343 
15344   ins_cost(INSN_COST);
15345 
15346   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15347 
15348   ins_encode %{
15349     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15350   %}
15351 
15352   ins_pipe(fp_f2i);
15353 
15354 %}
15355 
15356 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15357 
15358   match(Set dst (MoveI2F src));
15359 
15360   effect(DEF dst, USE src);
15361 
15362   ins_cost(INSN_COST);
15363 
15364   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15365 
15366   ins_encode %{
15367     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15368   %}
15369 
15370   ins_pipe(fp_i2f);
15371 
15372 %}
15373 
15374 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15375 
15376   match(Set dst (MoveD2L src));
15377 
15378   effect(DEF dst, USE src);
15379 
15380   ins_cost(INSN_COST);
15381 
15382   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15383 
15384   ins_encode %{
15385     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15386   %}
15387 
15388   ins_pipe(fp_d2l);
15389 
15390 %}
15391 
15392 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15393 
15394   match(Set dst (MoveL2D src));
15395 
15396   effect(DEF dst, USE src);
15397 
15398   ins_cost(INSN_COST);
15399 
15400   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15401 
15402   ins_encode %{
15403     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15404   %}
15405 
15406   ins_pipe(fp_l2d);
15407 
15408 %}
15409 
15410 // ============================================================================
15411 // clearing of an array
15412 
15413 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15414 %{
15415   match(Set dummy (ClearArray cnt base));
15416   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15417 
15418   ins_cost(4 * INSN_COST);
15419   format %{ "ClearArray $cnt, $base" %}
15420 
15421   ins_encode %{
15422     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15423     if (tpc == NULL) {
15424       ciEnv::current()->record_failure("CodeCache is full");
15425       return;
15426     }
15427   %}
15428 
15429   ins_pipe(pipe_class_memory);
15430 %}
15431 
15432 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15433 %{
15434   predicate((uint64_t)n->in(2)->get_long()
15435             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15436   match(Set dummy (ClearArray cnt base));
15437   effect(TEMP temp, USE_KILL base, KILL cr);
15438 
15439   ins_cost(4 * INSN_COST);
15440   format %{ "ClearArray $cnt, $base" %}
15441 
15442   ins_encode %{
15443     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15444     if (tpc == NULL) {
15445       ciEnv::current()->record_failure("CodeCache is full");
15446       return;
15447     }
15448   %}
15449 
15450   ins_pipe(pipe_class_memory);
15451 %}
15452 
15453 // ============================================================================
15454 // Overflow Math Instructions
15455 
15456 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15457 %{
15458   match(Set cr (OverflowAddI op1 op2));
15459 
15460   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15461   ins_cost(INSN_COST);
15462   ins_encode %{
15463     __ cmnw($op1$$Register, $op2$$Register);
15464   %}
15465 
15466   ins_pipe(icmp_reg_reg);
15467 %}
15468 
15469 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15470 %{
15471   match(Set cr (OverflowAddI op1 op2));
15472 
15473   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15474   ins_cost(INSN_COST);
15475   ins_encode %{
15476     __ cmnw($op1$$Register, $op2$$constant);
15477   %}
15478 
15479   ins_pipe(icmp_reg_imm);
15480 %}
15481 
15482 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15483 %{
15484   match(Set cr (OverflowAddL op1 op2));
15485 
15486   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15487   ins_cost(INSN_COST);
15488   ins_encode %{
15489     __ cmn($op1$$Register, $op2$$Register);
15490   %}
15491 
15492   ins_pipe(icmp_reg_reg);
15493 %}
15494 
15495 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15496 %{
15497   match(Set cr (OverflowAddL op1 op2));
15498 
15499   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
15500   ins_cost(INSN_COST);
15501   ins_encode %{
15502     __ adds(zr, $op1$$Register, $op2$$constant);
15503   %}
15504 
15505   ins_pipe(icmp_reg_imm);
15506 %}
15507 
15508 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15509 %{
15510   match(Set cr (OverflowSubI op1 op2));
15511 
15512   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15513   ins_cost(INSN_COST);
15514   ins_encode %{
15515     __ cmpw($op1$$Register, $op2$$Register);
15516   %}
15517 
15518   ins_pipe(icmp_reg_reg);
15519 %}
15520 
15521 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15522 %{
15523   match(Set cr (OverflowSubI op1 op2));
15524 
15525   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15526   ins_cost(INSN_COST);
15527   ins_encode %{
15528     __ cmpw($op1$$Register, $op2$$constant);
15529   %}
15530 
15531   ins_pipe(icmp_reg_imm);
15532 %}
15533 
15534 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15535 %{
15536   match(Set cr (OverflowSubL op1 op2));
15537 
15538   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15539   ins_cost(INSN_COST);
15540   ins_encode %{
15541     __ cmp($op1$$Register, $op2$$Register);
15542   %}
15543 
15544   ins_pipe(icmp_reg_reg);
15545 %}
15546 
15547 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15548 %{
15549   match(Set cr (OverflowSubL op1 op2));
15550 
15551   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15552   ins_cost(INSN_COST);
15553   ins_encode %{
15554     __ subs(zr, $op1$$Register, $op2$$constant);
15555   %}
15556 
15557   ins_pipe(icmp_reg_imm);
15558 %}
15559 
15560 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15561 %{
15562   match(Set cr (OverflowSubI zero op1));
15563 
15564   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15565   ins_cost(INSN_COST);
15566   ins_encode %{
15567     __ cmpw(zr, $op1$$Register);
15568   %}
15569 
15570   ins_pipe(icmp_reg_imm);
15571 %}
15572 
15573 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15574 %{
15575   match(Set cr (OverflowSubL zero op1));
15576 
15577   format %{ "cmp   zr, $op1\t# overflow check long" %}
15578   ins_cost(INSN_COST);
15579   ins_encode %{
15580     __ cmp(zr, $op1$$Register);
15581   %}
15582 
15583   ins_pipe(icmp_reg_imm);
15584 %}
15585 
15586 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15587 %{
15588   match(Set cr (OverflowMulI op1 op2));
15589 
15590   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15591             "cmp   rscratch1, rscratch1, sxtw\n\t"
15592             "movw  rscratch1, #0x80000000\n\t"
15593             "cselw rscratch1, rscratch1, zr, NE\n\t"
15594             "cmpw  rscratch1, #1" %}
15595   ins_cost(5 * INSN_COST);
15596   ins_encode %{
15597     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15598     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15599     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15600     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15601     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15602   %}
15603 
15604   ins_pipe(pipe_slow);
15605 %}
15606 
15607 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15608 %{
15609   match(If cmp (OverflowMulI op1 op2));
15610   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15611             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15612   effect(USE labl, KILL cr);
15613 
15614   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15615             "cmp   rscratch1, rscratch1, sxtw\n\t"
15616             "b$cmp   $labl" %}
15617   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15618   ins_encode %{
15619     Label* L = $labl$$label;
15620     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15621     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15622     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15623     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15624   %}
15625 
15626   ins_pipe(pipe_serial);
15627 %}
15628 
15629 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15630 %{
15631   match(Set cr (OverflowMulL op1 op2));
15632 
15633   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15634             "smulh rscratch2, $op1, $op2\n\t"
15635             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15636             "movw  rscratch1, #0x80000000\n\t"
15637             "cselw rscratch1, rscratch1, zr, NE\n\t"
15638             "cmpw  rscratch1, #1" %}
15639   ins_cost(6 * INSN_COST);
15640   ins_encode %{
15641     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15642     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15643     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15644     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15645     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15646     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15647   %}
15648 
15649   ins_pipe(pipe_slow);
15650 %}
15651 
15652 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15653 %{
15654   match(If cmp (OverflowMulL op1 op2));
15655   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15656             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15657   effect(USE labl, KILL cr);
15658 
15659   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15660             "smulh rscratch2, $op1, $op2\n\t"
15661             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15662             "b$cmp $labl" %}
15663   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15664   ins_encode %{
15665     Label* L = $labl$$label;
15666     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15667     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15668     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15669     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15670     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15671   %}
15672 
15673   ins_pipe(pipe_serial);
15674 %}
15675 
15676 // ============================================================================
15677 // Compare Instructions
15678 
15679 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15680 %{
15681   match(Set cr (CmpI op1 op2));
15682 
15683   effect(DEF cr, USE op1, USE op2);
15684 
15685   ins_cost(INSN_COST);
15686   format %{ "cmpw  $op1, $op2" %}
15687 
15688   ins_encode(aarch64_enc_cmpw(op1, op2));
15689 
15690   ins_pipe(icmp_reg_reg);
15691 %}
15692 
15693 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15694 %{
15695   match(Set cr (CmpI op1 zero));
15696 
15697   effect(DEF cr, USE op1);
15698 
15699   ins_cost(INSN_COST);
15700   format %{ "cmpw $op1, 0" %}
15701 
15702   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15703 
15704   ins_pipe(icmp_reg_imm);
15705 %}
15706 
15707 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15708 %{
15709   match(Set cr (CmpI op1 op2));
15710 
15711   effect(DEF cr, USE op1);
15712 
15713   ins_cost(INSN_COST);
15714   format %{ "cmpw  $op1, $op2" %}
15715 
15716   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15717 
15718   ins_pipe(icmp_reg_imm);
15719 %}
15720 
15721 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15722 %{
15723   match(Set cr (CmpI op1 op2));
15724 
15725   effect(DEF cr, USE op1);
15726 
15727   ins_cost(INSN_COST * 2);
15728   format %{ "cmpw  $op1, $op2" %}
15729 
15730   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15731 
15732   ins_pipe(icmp_reg_imm);
15733 %}
15734 
15735 // Unsigned compare Instructions; really, same as signed compare
15736 // except it should only be used to feed an If or a CMovI which takes a
15737 // cmpOpU.
15738 
15739 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15740 %{
15741   match(Set cr (CmpU op1 op2));
15742 
15743   effect(DEF cr, USE op1, USE op2);
15744 
15745   ins_cost(INSN_COST);
15746   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15747 
15748   ins_encode(aarch64_enc_cmpw(op1, op2));
15749 
15750   ins_pipe(icmp_reg_reg);
15751 %}
15752 
15753 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15754 %{
15755   match(Set cr (CmpU op1 zero));
15756 
15757   effect(DEF cr, USE op1);
15758 
15759   ins_cost(INSN_COST);
15760   format %{ "cmpw $op1, #0\t# unsigned" %}
15761 
15762   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15763 
15764   ins_pipe(icmp_reg_imm);
15765 %}
15766 
15767 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15768 %{
15769   match(Set cr (CmpU op1 op2));
15770 
15771   effect(DEF cr, USE op1);
15772 
15773   ins_cost(INSN_COST);
15774   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15775 
15776   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15777 
15778   ins_pipe(icmp_reg_imm);
15779 %}
15780 
15781 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15782 %{
15783   match(Set cr (CmpU op1 op2));
15784 
15785   effect(DEF cr, USE op1);
15786 
15787   ins_cost(INSN_COST * 2);
15788   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15789 
15790   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15791 
15792   ins_pipe(icmp_reg_imm);
15793 %}
15794 
15795 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15796 %{
15797   match(Set cr (CmpL op1 op2));
15798 
15799   effect(DEF cr, USE op1, USE op2);
15800 
15801   ins_cost(INSN_COST);
15802   format %{ "cmp  $op1, $op2" %}
15803 
15804   ins_encode(aarch64_enc_cmp(op1, op2));
15805 
15806   ins_pipe(icmp_reg_reg);
15807 %}
15808 
15809 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15810 %{
15811   match(Set cr (CmpL op1 zero));
15812 
15813   effect(DEF cr, USE op1);
15814 
15815   ins_cost(INSN_COST);
15816   format %{ "tst  $op1" %}
15817 
15818   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15819 
15820   ins_pipe(icmp_reg_imm);
15821 %}
15822 
15823 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15824 %{
15825   match(Set cr (CmpL op1 op2));
15826 
15827   effect(DEF cr, USE op1);
15828 
15829   ins_cost(INSN_COST);
15830   format %{ "cmp  $op1, $op2" %}
15831 
15832   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15833 
15834   ins_pipe(icmp_reg_imm);
15835 %}
15836 
15837 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15838 %{
15839   match(Set cr (CmpL op1 op2));
15840 
15841   effect(DEF cr, USE op1);
15842 
15843   ins_cost(INSN_COST * 2);
15844   format %{ "cmp  $op1, $op2" %}
15845 
15846   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15847 
15848   ins_pipe(icmp_reg_imm);
15849 %}
15850 
15851 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15852 %{
15853   match(Set cr (CmpUL op1 op2));
15854 
15855   effect(DEF cr, USE op1, USE op2);
15856 
15857   ins_cost(INSN_COST);
15858   format %{ "cmp  $op1, $op2" %}
15859 
15860   ins_encode(aarch64_enc_cmp(op1, op2));
15861 
15862   ins_pipe(icmp_reg_reg);
15863 %}
15864 
15865 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15866 %{
15867   match(Set cr (CmpUL op1 zero));
15868 
15869   effect(DEF cr, USE op1);
15870 
15871   ins_cost(INSN_COST);
15872   format %{ "tst  $op1" %}
15873 
15874   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15875 
15876   ins_pipe(icmp_reg_imm);
15877 %}
15878 
15879 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15880 %{
15881   match(Set cr (CmpUL op1 op2));
15882 
15883   effect(DEF cr, USE op1);
15884 
15885   ins_cost(INSN_COST);
15886   format %{ "cmp  $op1, $op2" %}
15887 
15888   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15889 
15890   ins_pipe(icmp_reg_imm);
15891 %}
15892 
15893 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15894 %{
15895   match(Set cr (CmpUL op1 op2));
15896 
15897   effect(DEF cr, USE op1);
15898 
15899   ins_cost(INSN_COST * 2);
15900   format %{ "cmp  $op1, $op2" %}
15901 
15902   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15903 
15904   ins_pipe(icmp_reg_imm);
15905 %}
15906 
15907 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15908 %{
15909   match(Set cr (CmpP op1 op2));
15910 
15911   effect(DEF cr, USE op1, USE op2);
15912 
15913   ins_cost(INSN_COST);
15914   format %{ "cmp  $op1, $op2\t // ptr" %}
15915 
15916   ins_encode(aarch64_enc_cmpp(op1, op2));
15917 
15918   ins_pipe(icmp_reg_reg);
15919 %}
15920 
15921 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15922 %{
15923   match(Set cr (CmpN op1 op2));
15924 
15925   effect(DEF cr, USE op1, USE op2);
15926 
15927   ins_cost(INSN_COST);
15928   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15929 
15930   ins_encode(aarch64_enc_cmpn(op1, op2));
15931 
15932   ins_pipe(icmp_reg_reg);
15933 %}
15934 
15935 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15936 %{
15937   match(Set cr (CmpP op1 zero));
15938 
15939   effect(DEF cr, USE op1, USE zero);
15940 
15941   ins_cost(INSN_COST);
15942   format %{ "cmp  $op1, 0\t // ptr" %}
15943 
15944   ins_encode(aarch64_enc_testp(op1));
15945 
15946   ins_pipe(icmp_reg_imm);
15947 %}
15948 
15949 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15950 %{
15951   match(Set cr (CmpN op1 zero));
15952 
15953   effect(DEF cr, USE op1, USE zero);
15954 
15955   ins_cost(INSN_COST);
15956   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15957 
15958   ins_encode(aarch64_enc_testn(op1));
15959 
15960   ins_pipe(icmp_reg_imm);
15961 %}
15962 
15963 // FP comparisons
15964 //
15965 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15966 // using normal cmpOp. See declaration of rFlagsReg for details.
15967 
15968 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15969 %{
15970   match(Set cr (CmpF src1 src2));
15971 
15972   ins_cost(3 * INSN_COST);
15973   format %{ "fcmps $src1, $src2" %}
15974 
15975   ins_encode %{
15976     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15977   %}
15978 
15979   ins_pipe(pipe_class_compare);
15980 %}
15981 
15982 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15983 %{
15984   match(Set cr (CmpF src1 src2));
15985 
15986   ins_cost(3 * INSN_COST);
15987   format %{ "fcmps $src1, 0.0" %}
15988 
15989   ins_encode %{
15990     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15991   %}
15992 
15993   ins_pipe(pipe_class_compare);
15994 %}
15995 // FROM HERE
15996 
15997 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15998 %{
15999   match(Set cr (CmpD src1 src2));
16000 
16001   ins_cost(3 * INSN_COST);
16002   format %{ "fcmpd $src1, $src2" %}
16003 
16004   ins_encode %{
16005     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16006   %}
16007 
16008   ins_pipe(pipe_class_compare);
16009 %}
16010 
16011 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
16012 %{
16013   match(Set cr (CmpD src1 src2));
16014 
16015   ins_cost(3 * INSN_COST);
16016   format %{ "fcmpd $src1, 0.0" %}
16017 
16018   ins_encode %{
16019     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
16020   %}
16021 
16022   ins_pipe(pipe_class_compare);
16023 %}
16024 
16025 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
16026 %{
16027   match(Set dst (CmpF3 src1 src2));
16028   effect(KILL cr);
16029 
16030   ins_cost(5 * INSN_COST);
16031   format %{ "fcmps $src1, $src2\n\t"
16032             "csinvw($dst, zr, zr, eq\n\t"
16033             "csnegw($dst, $dst, $dst, lt)"
16034   %}
16035 
16036   ins_encode %{
16037     Label done;
16038     FloatRegister s1 = as_FloatRegister($src1$$reg);
16039     FloatRegister s2 = as_FloatRegister($src2$$reg);
16040     Register d = as_Register($dst$$reg);
16041     __ fcmps(s1, s2);
16042     // installs 0 if EQ else -1
16043     __ csinvw(d, zr, zr, Assembler::EQ);
16044     // keeps -1 if less or unordered else installs 1
16045     __ csnegw(d, d, d, Assembler::LT);
16046     __ bind(done);
16047   %}
16048 
16049   ins_pipe(pipe_class_default);
16050 
16051 %}
16052 
16053 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
16054 %{
16055   match(Set dst (CmpD3 src1 src2));
16056   effect(KILL cr);
16057 
16058   ins_cost(5 * INSN_COST);
16059   format %{ "fcmpd $src1, $src2\n\t"
16060             "csinvw($dst, zr, zr, eq\n\t"
16061             "csnegw($dst, $dst, $dst, lt)"
16062   %}
16063 
16064   ins_encode %{
16065     Label done;
16066     FloatRegister s1 = as_FloatRegister($src1$$reg);
16067     FloatRegister s2 = as_FloatRegister($src2$$reg);
16068     Register d = as_Register($dst$$reg);
16069     __ fcmpd(s1, s2);
16070     // installs 0 if EQ else -1
16071     __ csinvw(d, zr, zr, Assembler::EQ);
16072     // keeps -1 if less or unordered else installs 1
16073     __ csnegw(d, d, d, Assembler::LT);
16074     __ bind(done);
16075   %}
16076   ins_pipe(pipe_class_default);
16077 
16078 %}
16079 
16080 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
16081 %{
16082   match(Set dst (CmpF3 src1 zero));
16083   effect(KILL cr);
16084 
16085   ins_cost(5 * INSN_COST);
16086   format %{ "fcmps $src1, 0.0\n\t"
16087             "csinvw($dst, zr, zr, eq\n\t"
16088             "csnegw($dst, $dst, $dst, lt)"
16089   %}
16090 
16091   ins_encode %{
16092     Label done;
16093     FloatRegister s1 = as_FloatRegister($src1$$reg);
16094     Register d = as_Register($dst$$reg);
16095     __ fcmps(s1, 0.0);
16096     // installs 0 if EQ else -1
16097     __ csinvw(d, zr, zr, Assembler::EQ);
16098     // keeps -1 if less or unordered else installs 1
16099     __ csnegw(d, d, d, Assembler::LT);
16100     __ bind(done);
16101   %}
16102 
16103   ins_pipe(pipe_class_default);
16104 
16105 %}
16106 
16107 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
16108 %{
16109   match(Set dst (CmpD3 src1 zero));
16110   effect(KILL cr);
16111 
16112   ins_cost(5 * INSN_COST);
16113   format %{ "fcmpd $src1, 0.0\n\t"
16114             "csinvw($dst, zr, zr, eq\n\t"
16115             "csnegw($dst, $dst, $dst, lt)"
16116   %}
16117 
16118   ins_encode %{
16119     Label done;
16120     FloatRegister s1 = as_FloatRegister($src1$$reg);
16121     Register d = as_Register($dst$$reg);
16122     __ fcmpd(s1, 0.0);
16123     // installs 0 if EQ else -1
16124     __ csinvw(d, zr, zr, Assembler::EQ);
16125     // keeps -1 if less or unordered else installs 1
16126     __ csnegw(d, d, d, Assembler::LT);
16127     __ bind(done);
16128   %}
16129   ins_pipe(pipe_class_default);
16130 
16131 %}
16132 
16133 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
16134 %{
16135   match(Set dst (CmpLTMask p q));
16136   effect(KILL cr);
16137 
16138   ins_cost(3 * INSN_COST);
16139 
16140   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
16141             "csetw $dst, lt\n\t"
16142             "subw $dst, zr, $dst"
16143   %}
16144 
16145   ins_encode %{
16146     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
16147     __ csetw(as_Register($dst$$reg), Assembler::LT);
16148     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
16149   %}
16150 
16151   ins_pipe(ialu_reg_reg);
16152 %}
16153 
16154 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
16155 %{
16156   match(Set dst (CmpLTMask src zero));
16157   effect(KILL cr);
16158 
16159   ins_cost(INSN_COST);
16160 
16161   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
16162 
16163   ins_encode %{
16164     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
16165   %}
16166 
16167   ins_pipe(ialu_reg_shift);
16168 %}
16169 
16170 // ============================================================================
16171 // Max and Min
16172 
16173 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
16174 
16175 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
16176 %{
16177   effect(DEF cr, USE src);
16178   ins_cost(INSN_COST);
16179   format %{ "cmpw $src, 0" %}
16180 
16181   ins_encode %{
16182     __ cmpw($src$$Register, 0);
16183   %}
16184   ins_pipe(icmp_reg_imm);
16185 %}
16186 
16187 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16188 %{
16189   match(Set dst (MinI src1 src2));
16190   ins_cost(INSN_COST * 3);
16191 
16192   expand %{
16193     rFlagsReg cr;
16194     compI_reg_reg(cr, src1, src2);
16195     cmovI_reg_reg_lt(dst, src1, src2, cr);
16196   %}
16197 %}
16198 
16199 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16200 %{
16201   match(Set dst (MaxI src1 src2));
16202   ins_cost(INSN_COST * 3);
16203 
16204   expand %{
16205     rFlagsReg cr;
16206     compI_reg_reg(cr, src1, src2);
16207     cmovI_reg_reg_gt(dst, src1, src2, cr);
16208   %}
16209 %}
16210 
16211 
16212 // ============================================================================
16213 // Branch Instructions
16214 
16215 // Direct Branch.
16216 instruct branch(label lbl)
16217 %{
16218   match(Goto);
16219 
16220   effect(USE lbl);
16221 
16222   ins_cost(BRANCH_COST);
16223   format %{ "b  $lbl" %}
16224 
16225   ins_encode(aarch64_enc_b(lbl));
16226 
16227   ins_pipe(pipe_branch);
16228 %}
16229 
16230 // Conditional Near Branch
16231 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16232 %{
16233   // Same match rule as `branchConFar'.
16234   match(If cmp cr);
16235 
16236   effect(USE lbl);
16237 
16238   ins_cost(BRANCH_COST);
16239   // If set to 1 this indicates that the current instruction is a
16240   // short variant of a long branch. This avoids using this
16241   // instruction in first-pass matching. It will then only be used in
16242   // the `Shorten_branches' pass.
16243   // ins_short_branch(1);
16244   format %{ "b$cmp  $lbl" %}
16245 
16246   ins_encode(aarch64_enc_br_con(cmp, lbl));
16247 
16248   ins_pipe(pipe_branch_cond);
16249 %}
16250 
16251 // Conditional Near Branch Unsigned
16252 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16253 %{
16254   // Same match rule as `branchConFar'.
16255   match(If cmp cr);
16256 
16257   effect(USE lbl);
16258 
16259   ins_cost(BRANCH_COST);
16260   // If set to 1 this indicates that the current instruction is a
16261   // short variant of a long branch. This avoids using this
16262   // instruction in first-pass matching. It will then only be used in
16263   // the `Shorten_branches' pass.
16264   // ins_short_branch(1);
16265   format %{ "b$cmp  $lbl\t# unsigned" %}
16266 
16267   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16268 
16269   ins_pipe(pipe_branch_cond);
16270 %}
16271 
16272 // Make use of CBZ and CBNZ.  These instructions, as well as being
16273 // shorter than (cmp; branch), have the additional benefit of not
16274 // killing the flags.
16275 
16276 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16277   match(If cmp (CmpI op1 op2));
16278   effect(USE labl);
16279 
16280   ins_cost(BRANCH_COST);
16281   format %{ "cbw$cmp   $op1, $labl" %}
16282   ins_encode %{
16283     Label* L = $labl$$label;
16284     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16285     if (cond == Assembler::EQ)
16286       __ cbzw($op1$$Register, *L);
16287     else
16288       __ cbnzw($op1$$Register, *L);
16289   %}
16290   ins_pipe(pipe_cmp_branch);
16291 %}
16292 
16293 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16294   match(If cmp (CmpL op1 op2));
16295   effect(USE labl);
16296 
16297   ins_cost(BRANCH_COST);
16298   format %{ "cb$cmp   $op1, $labl" %}
16299   ins_encode %{
16300     Label* L = $labl$$label;
16301     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16302     if (cond == Assembler::EQ)
16303       __ cbz($op1$$Register, *L);
16304     else
16305       __ cbnz($op1$$Register, *L);
16306   %}
16307   ins_pipe(pipe_cmp_branch);
16308 %}
16309 
16310 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16311   match(If cmp (CmpP op1 op2));
16312   effect(USE labl);
16313 
16314   ins_cost(BRANCH_COST);
16315   format %{ "cb$cmp   $op1, $labl" %}
16316   ins_encode %{
16317     Label* L = $labl$$label;
16318     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16319     if (cond == Assembler::EQ)
16320       __ cbz($op1$$Register, *L);
16321     else
16322       __ cbnz($op1$$Register, *L);
16323   %}
16324   ins_pipe(pipe_cmp_branch);
16325 %}
16326 
16327 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16328   match(If cmp (CmpN op1 op2));
16329   effect(USE labl);
16330 
16331   ins_cost(BRANCH_COST);
16332   format %{ "cbw$cmp   $op1, $labl" %}
16333   ins_encode %{
16334     Label* L = $labl$$label;
16335     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16336     if (cond == Assembler::EQ)
16337       __ cbzw($op1$$Register, *L);
16338     else
16339       __ cbnzw($op1$$Register, *L);
16340   %}
16341   ins_pipe(pipe_cmp_branch);
16342 %}
16343 
16344 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16345   match(If cmp (CmpP (DecodeN oop) zero));
16346   effect(USE labl);
16347 
16348   ins_cost(BRANCH_COST);
16349   format %{ "cb$cmp   $oop, $labl" %}
16350   ins_encode %{
16351     Label* L = $labl$$label;
16352     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16353     if (cond == Assembler::EQ)
16354       __ cbzw($oop$$Register, *L);
16355     else
16356       __ cbnzw($oop$$Register, *L);
16357   %}
16358   ins_pipe(pipe_cmp_branch);
16359 %}
16360 
16361 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16362   match(If cmp (CmpU op1 op2));
16363   effect(USE labl);
16364 
16365   ins_cost(BRANCH_COST);
16366   format %{ "cbw$cmp   $op1, $labl" %}
16367   ins_encode %{
16368     Label* L = $labl$$label;
16369     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16370     if (cond == Assembler::EQ || cond == Assembler::LS)
16371       __ cbzw($op1$$Register, *L);
16372     else
16373       __ cbnzw($op1$$Register, *L);
16374   %}
16375   ins_pipe(pipe_cmp_branch);
16376 %}
16377 
16378 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16379   match(If cmp (CmpUL op1 op2));
16380   effect(USE labl);
16381 
16382   ins_cost(BRANCH_COST);
16383   format %{ "cb$cmp   $op1, $labl" %}
16384   ins_encode %{
16385     Label* L = $labl$$label;
16386     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16387     if (cond == Assembler::EQ || cond == Assembler::LS)
16388       __ cbz($op1$$Register, *L);
16389     else
16390       __ cbnz($op1$$Register, *L);
16391   %}
16392   ins_pipe(pipe_cmp_branch);
16393 %}
16394 
16395 // Test bit and Branch
16396 
16397 // Patterns for short (< 32KiB) variants
16398 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16399   match(If cmp (CmpL op1 op2));
16400   effect(USE labl);
16401 
16402   ins_cost(BRANCH_COST);
16403   format %{ "cb$cmp   $op1, $labl # long" %}
16404   ins_encode %{
16405     Label* L = $labl$$label;
16406     Assembler::Condition cond =
16407       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16408     __ tbr(cond, $op1$$Register, 63, *L);
16409   %}
16410   ins_pipe(pipe_cmp_branch);
16411   ins_short_branch(1);
16412 %}
16413 
16414 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16415   match(If cmp (CmpI op1 op2));
16416   effect(USE labl);
16417 
16418   ins_cost(BRANCH_COST);
16419   format %{ "cb$cmp   $op1, $labl # int" %}
16420   ins_encode %{
16421     Label* L = $labl$$label;
16422     Assembler::Condition cond =
16423       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16424     __ tbr(cond, $op1$$Register, 31, *L);
16425   %}
16426   ins_pipe(pipe_cmp_branch);
16427   ins_short_branch(1);
16428 %}
16429 
16430 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16431   match(If cmp (CmpL (AndL op1 op2) op3));
16432   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16433   effect(USE labl);
16434 
16435   ins_cost(BRANCH_COST);
16436   format %{ "tb$cmp   $op1, $op2, $labl" %}
16437   ins_encode %{
16438     Label* L = $labl$$label;
16439     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16440     int bit = exact_log2_long($op2$$constant);
16441     __ tbr(cond, $op1$$Register, bit, *L);
16442   %}
16443   ins_pipe(pipe_cmp_branch);
16444   ins_short_branch(1);
16445 %}
16446 
16447 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16448   match(If cmp (CmpI (AndI op1 op2) op3));
16449   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16450   effect(USE labl);
16451 
16452   ins_cost(BRANCH_COST);
16453   format %{ "tb$cmp   $op1, $op2, $labl" %}
16454   ins_encode %{
16455     Label* L = $labl$$label;
16456     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16457     int bit = exact_log2((juint)$op2$$constant);
16458     __ tbr(cond, $op1$$Register, bit, *L);
16459   %}
16460   ins_pipe(pipe_cmp_branch);
16461   ins_short_branch(1);
16462 %}
16463 
16464 // And far variants
16465 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16466   match(If cmp (CmpL op1 op2));
16467   effect(USE labl);
16468 
16469   ins_cost(BRANCH_COST);
16470   format %{ "cb$cmp   $op1, $labl # long" %}
16471   ins_encode %{
16472     Label* L = $labl$$label;
16473     Assembler::Condition cond =
16474       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16475     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16476   %}
16477   ins_pipe(pipe_cmp_branch);
16478 %}
16479 
16480 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16481   match(If cmp (CmpI op1 op2));
16482   effect(USE labl);
16483 
16484   ins_cost(BRANCH_COST);
16485   format %{ "cb$cmp   $op1, $labl # int" %}
16486   ins_encode %{
16487     Label* L = $labl$$label;
16488     Assembler::Condition cond =
16489       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16490     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16491   %}
16492   ins_pipe(pipe_cmp_branch);
16493 %}
16494 
16495 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16496   match(If cmp (CmpL (AndL op1 op2) op3));
16497   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16498   effect(USE labl);
16499 
16500   ins_cost(BRANCH_COST);
16501   format %{ "tb$cmp   $op1, $op2, $labl" %}
16502   ins_encode %{
16503     Label* L = $labl$$label;
16504     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16505     int bit = exact_log2_long($op2$$constant);
16506     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16507   %}
16508   ins_pipe(pipe_cmp_branch);
16509 %}
16510 
16511 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16512   match(If cmp (CmpI (AndI op1 op2) op3));
16513   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16514   effect(USE labl);
16515 
16516   ins_cost(BRANCH_COST);
16517   format %{ "tb$cmp   $op1, $op2, $labl" %}
16518   ins_encode %{
16519     Label* L = $labl$$label;
16520     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16521     int bit = exact_log2((juint)$op2$$constant);
16522     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16523   %}
16524   ins_pipe(pipe_cmp_branch);
16525 %}
16526 
16527 // Test bits
16528 
16529 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16530   match(Set cr (CmpL (AndL op1 op2) op3));
16531   predicate(Assembler::operand_valid_for_logical_immediate
16532             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16533 
16534   ins_cost(INSN_COST);
16535   format %{ "tst $op1, $op2 # long" %}
16536   ins_encode %{
16537     __ tst($op1$$Register, $op2$$constant);
16538   %}
16539   ins_pipe(ialu_reg_reg);
16540 %}
16541 
16542 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16543   match(Set cr (CmpI (AndI op1 op2) op3));
16544   predicate(Assembler::operand_valid_for_logical_immediate
16545             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16546 
16547   ins_cost(INSN_COST);
16548   format %{ "tst $op1, $op2 # int" %}
16549   ins_encode %{
16550     __ tstw($op1$$Register, $op2$$constant);
16551   %}
16552   ins_pipe(ialu_reg_reg);
16553 %}
16554 
16555 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16556   match(Set cr (CmpL (AndL op1 op2) op3));
16557 
16558   ins_cost(INSN_COST);
16559   format %{ "tst $op1, $op2 # long" %}
16560   ins_encode %{
16561     __ tst($op1$$Register, $op2$$Register);
16562   %}
16563   ins_pipe(ialu_reg_reg);
16564 %}
16565 
16566 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16567   match(Set cr (CmpI (AndI op1 op2) op3));
16568 
16569   ins_cost(INSN_COST);
16570   format %{ "tstw $op1, $op2 # int" %}
16571   ins_encode %{
16572     __ tstw($op1$$Register, $op2$$Register);
16573   %}
16574   ins_pipe(ialu_reg_reg);
16575 %}
16576 
16577 
16578 // Conditional Far Branch
16579 // Conditional Far Branch Unsigned
16580 // TODO: fixme
16581 
16582 // counted loop end branch near
16583 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16584 %{
16585   match(CountedLoopEnd cmp cr);
16586 
16587   effect(USE lbl);
16588 
16589   ins_cost(BRANCH_COST);
16590   // short variant.
16591   // ins_short_branch(1);
16592   format %{ "b$cmp $lbl \t// counted loop end" %}
16593 
16594   ins_encode(aarch64_enc_br_con(cmp, lbl));
16595 
16596   ins_pipe(pipe_branch);
16597 %}
16598 
16599 // counted loop end branch near Unsigned
16600 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16601 %{
16602   match(CountedLoopEnd cmp cr);
16603 
16604   effect(USE lbl);
16605 
16606   ins_cost(BRANCH_COST);
16607   // short variant.
16608   // ins_short_branch(1);
16609   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16610 
16611   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16612 
16613   ins_pipe(pipe_branch);
16614 %}
16615 
16616 // counted loop end branch far
16617 // counted loop end branch far unsigned
16618 // TODO: fixme
16619 
16620 // ============================================================================
16621 // inlined locking and unlocking
16622 
16623 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16624 %{
16625   match(Set cr (FastLock object box));
16626   effect(TEMP tmp, TEMP tmp2);
16627 
16628   // TODO
16629   // identify correct cost
16630   ins_cost(5 * INSN_COST);
16631   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16632 
16633   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16634 
16635   ins_pipe(pipe_serial);
16636 %}
16637 
16638 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16639 %{
16640   match(Set cr (FastUnlock object box));
16641   effect(TEMP tmp, TEMP tmp2);
16642 
16643   ins_cost(5 * INSN_COST);
16644   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16645 
16646   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16647 
16648   ins_pipe(pipe_serial);
16649 %}
16650 
16651 
16652 // ============================================================================
16653 // Safepoint Instructions
16654 
16655 // TODO
16656 // provide a near and far version of this code
16657 
16658 instruct safePoint(rFlagsReg cr, iRegP poll)
16659 %{
16660   match(SafePoint poll);
16661   effect(KILL cr);
16662 
16663   format %{
16664     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16665   %}
16666   ins_encode %{
16667     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16668   %}
16669   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16670 %}
16671 
16672 
16673 // ============================================================================
16674 // Procedure Call/Return Instructions
16675 
16676 // Call Java Static Instruction
16677 
16678 instruct CallStaticJavaDirect(method meth)
16679 %{
16680   match(CallStaticJava);
16681 
16682   effect(USE meth);
16683 
16684   ins_cost(CALL_COST);
16685 
16686   format %{ "call,static $meth \t// ==> " %}
16687 
16688   ins_encode(aarch64_enc_java_static_call(meth),
16689              aarch64_enc_call_epilog);
16690 
16691   ins_pipe(pipe_class_call);
16692 %}
16693 
16694 // TO HERE
16695 
16696 // Call Java Dynamic Instruction
16697 instruct CallDynamicJavaDirect(method meth)
16698 %{
16699   match(CallDynamicJava);
16700 
16701   effect(USE meth);
16702 
16703   ins_cost(CALL_COST);
16704 
16705   format %{ "CALL,dynamic $meth \t// ==> " %}
16706 
16707   ins_encode(aarch64_enc_java_dynamic_call(meth),
16708              aarch64_enc_call_epilog);
16709 
16710   ins_pipe(pipe_class_call);
16711 %}
16712 
16713 // Call Runtime Instruction
16714 
16715 instruct CallRuntimeDirect(method meth)
16716 %{
16717   match(CallRuntime);
16718 
16719   effect(USE meth);
16720 
16721   ins_cost(CALL_COST);
16722 
16723   format %{ "CALL, runtime $meth" %}
16724 
16725   ins_encode( aarch64_enc_java_to_runtime(meth) );
16726 
16727   ins_pipe(pipe_class_call);
16728 %}
16729 
16730 // Call Runtime Instruction
16731 
16732 instruct CallLeafDirect(method meth)
16733 %{
16734   match(CallLeaf);
16735 
16736   effect(USE meth);
16737 
16738   ins_cost(CALL_COST);
16739 
16740   format %{ "CALL, runtime leaf $meth" %}
16741 
16742   ins_encode( aarch64_enc_java_to_runtime(meth) );
16743 
16744   ins_pipe(pipe_class_call);
16745 %}
16746 
16747 // Call Runtime Instruction
16748 
16749 instruct CallLeafNoFPDirect(method meth)
16750 %{
16751   match(CallLeafNoFP);
16752 
16753   effect(USE meth);
16754 
16755   ins_cost(CALL_COST);
16756 
16757   format %{ "CALL, runtime leaf nofp $meth" %}
16758 
16759   ins_encode( aarch64_enc_java_to_runtime(meth) );
16760 
16761   ins_pipe(pipe_class_call);
16762 %}
16763 
16764 // Tail Call; Jump from runtime stub to Java code.
16765 // Also known as an 'interprocedural jump'.
16766 // Target of jump will eventually return to caller.
16767 // TailJump below removes the return address.
16768 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16769 %{
16770   match(TailCall jump_target method_ptr);
16771 
16772   ins_cost(CALL_COST);
16773 
16774   format %{ "br $jump_target\t# $method_ptr holds method" %}
16775 
16776   ins_encode(aarch64_enc_tail_call(jump_target));
16777 
16778   ins_pipe(pipe_class_call);
16779 %}
16780 
16781 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16782 %{
16783   match(TailJump jump_target ex_oop);
16784 
16785   ins_cost(CALL_COST);
16786 
16787   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16788 
16789   ins_encode(aarch64_enc_tail_jmp(jump_target));
16790 
16791   ins_pipe(pipe_class_call);
16792 %}
16793 
16794 // Create exception oop: created by stack-crawling runtime code.
16795 // Created exception is now available to this handler, and is setup
16796 // just prior to jumping to this handler. No code emitted.
16797 // TODO check
16798 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16799 instruct CreateException(iRegP_R0 ex_oop)
16800 %{
16801   match(Set ex_oop (CreateEx));
16802 
16803   format %{ " -- \t// exception oop; no code emitted" %}
16804 
16805   size(0);
16806 
16807   ins_encode( /*empty*/ );
16808 
16809   ins_pipe(pipe_class_empty);
16810 %}
16811 
16812 // Rethrow exception: The exception oop will come in the first
16813 // argument position. Then JUMP (not call) to the rethrow stub code.
16814 instruct RethrowException() %{
16815   match(Rethrow);
16816   ins_cost(CALL_COST);
16817 
16818   format %{ "b rethrow_stub" %}
16819 
16820   ins_encode( aarch64_enc_rethrow() );
16821 
16822   ins_pipe(pipe_class_call);
16823 %}
16824 
16825 
16826 // Return Instruction
16827 // epilog node loads ret address into lr as part of frame pop
16828 instruct Ret()
16829 %{
16830   match(Return);
16831 
16832   format %{ "ret\t// return register" %}
16833 
16834   ins_encode( aarch64_enc_ret() );
16835 
16836   ins_pipe(pipe_branch);
16837 %}
16838 
16839 // Die now.
16840 instruct ShouldNotReachHere() %{
16841   match(Halt);
16842 
16843   ins_cost(CALL_COST);
16844   format %{ "ShouldNotReachHere" %}
16845 
16846   ins_encode %{
16847     if (is_reachable()) {
16848       __ stop(_halt_reason);
16849     }
16850   %}
16851 
16852   ins_pipe(pipe_class_default);
16853 %}
16854 
16855 // ============================================================================
16856 // Partial Subtype Check
16857 //
16858 // superklass array for an instance of the superklass.  Set a hidden
16859 // internal cache on a hit (cache is checked with exposed code in
16860 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16861 // encoding ALSO sets flags.
16862 
16863 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16864 %{
16865   match(Set result (PartialSubtypeCheck sub super));
16866   effect(KILL cr, KILL temp);
16867 
16868   ins_cost(1100);  // slightly larger than the next version
16869   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16870 
16871   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16872 
16873   opcode(0x1); // Force zero of result reg on hit
16874 
16875   ins_pipe(pipe_class_memory);
16876 %}
16877 
16878 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16879 %{
16880   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16881   effect(KILL temp, KILL result);
16882 
16883   ins_cost(1100);  // slightly larger than the next version
16884   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16885 
16886   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16887 
16888   opcode(0x0); // Don't zero result reg on hit
16889 
16890   ins_pipe(pipe_class_memory);
16891 %}
16892 
16893 // Intrisics for String.compareTo()
16894 
16895 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16896                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16897 %{
16898   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16899   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16900   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16901 
16902   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16903   ins_encode %{
16904     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16905     __ string_compare($str1$$Register, $str2$$Register,
16906                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16907                       $tmp1$$Register, $tmp2$$Register,
16908                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16909   %}
16910   ins_pipe(pipe_class_memory);
16911 %}
16912 
16913 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16914                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16915 %{
16916   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16917   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16918   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16919 
16920   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16921   ins_encode %{
16922     __ string_compare($str1$$Register, $str2$$Register,
16923                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16924                       $tmp1$$Register, $tmp2$$Register,
16925                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16926   %}
16927   ins_pipe(pipe_class_memory);
16928 %}
16929 
16930 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16931                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16932                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16933 %{
16934   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16935   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16936   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16937          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16938 
16939   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16940   ins_encode %{
16941     __ string_compare($str1$$Register, $str2$$Register,
16942                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16943                       $tmp1$$Register, $tmp2$$Register,
16944                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16945                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16946   %}
16947   ins_pipe(pipe_class_memory);
16948 %}
16949 
16950 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16951                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16952                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16953 %{
16954   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16955   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16956   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16957          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16958 
16959   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16960   ins_encode %{
16961     __ string_compare($str1$$Register, $str2$$Register,
16962                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16963                       $tmp1$$Register, $tmp2$$Register,
16964                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16965                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16966   %}
16967   ins_pipe(pipe_class_memory);
16968 %}
16969 
16970 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16971 // these string_compare variants as NEON register type for convenience so that the prototype of
16972 // string_compare can be shared with all variants.
16973 
16974 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16975                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16976                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16977                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16978 %{
16979   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16980   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16981   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16982          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16983 
16984   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16985   ins_encode %{
16986     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16987     __ string_compare($str1$$Register, $str2$$Register,
16988                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16989                       $tmp1$$Register, $tmp2$$Register,
16990                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16991                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16992                       StrIntrinsicNode::LL);
16993   %}
16994   ins_pipe(pipe_class_memory);
16995 %}
16996 
16997 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16998                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16999                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
17000                               pRegGov_P1 pgtmp2, rFlagsReg cr)
17001 %{
17002   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
17003   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17004   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17005          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17006 
17007   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
17008   ins_encode %{
17009     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17010     __ string_compare($str1$$Register, $str2$$Register,
17011                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
17012                       $tmp1$$Register, $tmp2$$Register,
17013                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17014                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17015                       StrIntrinsicNode::LU);
17016   %}
17017   ins_pipe(pipe_class_memory);
17018 %}
17019 
17020 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17021                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17022                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
17023                               pRegGov_P1 pgtmp2, rFlagsReg cr)
17024 %{
17025   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
17026   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17027   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17028          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17029 
17030   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
17031   ins_encode %{
17032     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17033     __ string_compare($str1$$Register, $str2$$Register,
17034                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
17035                       $tmp1$$Register, $tmp2$$Register,
17036                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17037                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17038                       StrIntrinsicNode::UL);
17039   %}
17040   ins_pipe(pipe_class_memory);
17041 %}
17042 
17043 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17044                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17045                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
17046                               pRegGov_P1 pgtmp2, rFlagsReg cr)
17047 %{
17048   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
17049   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17050   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17051          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17052 
17053   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
17054   ins_encode %{
17055     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17056     __ string_compare($str1$$Register, $str2$$Register,
17057                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
17058                       $tmp1$$Register, $tmp2$$Register,
17059                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17060                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17061                       StrIntrinsicNode::UU);
17062   %}
17063   ins_pipe(pipe_class_memory);
17064 %}
17065 
17066 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17067        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17068        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
17069 %{
17070   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17071   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17072   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17073          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
17074   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
17075 
17076   ins_encode %{
17077     __ string_indexof($str1$$Register, $str2$$Register,
17078                       $cnt1$$Register, $cnt2$$Register,
17079                       $tmp1$$Register, $tmp2$$Register,
17080                       $tmp3$$Register, $tmp4$$Register,
17081                       $tmp5$$Register, $tmp6$$Register,
17082                       -1, $result$$Register, StrIntrinsicNode::UU);
17083   %}
17084   ins_pipe(pipe_class_memory);
17085 %}
17086 
17087 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17088        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17089        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
17090 %{
17091   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17092   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17093   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17094          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
17095   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
17096 
17097   ins_encode %{
17098     __ string_indexof($str1$$Register, $str2$$Register,
17099                       $cnt1$$Register, $cnt2$$Register,
17100                       $tmp1$$Register, $tmp2$$Register,
17101                       $tmp3$$Register, $tmp4$$Register,
17102                       $tmp5$$Register, $tmp6$$Register,
17103                       -1, $result$$Register, StrIntrinsicNode::LL);
17104   %}
17105   ins_pipe(pipe_class_memory);
17106 %}
17107 
17108 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17109        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17110        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
17111 %{
17112   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17113   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17114   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17115          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
17116   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
17117 
17118   ins_encode %{
17119     __ string_indexof($str1$$Register, $str2$$Register,
17120                       $cnt1$$Register, $cnt2$$Register,
17121                       $tmp1$$Register, $tmp2$$Register,
17122                       $tmp3$$Register, $tmp4$$Register,
17123                       $tmp5$$Register, $tmp6$$Register,
17124                       -1, $result$$Register, StrIntrinsicNode::UL);
17125   %}
17126   ins_pipe(pipe_class_memory);
17127 %}
17128 
17129 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17130                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17131                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17132 %{
17133   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17134   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17135   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17136          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17137   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
17138 
17139   ins_encode %{
17140     int icnt2 = (int)$int_cnt2$$constant;
17141     __ string_indexof($str1$$Register, $str2$$Register,
17142                       $cnt1$$Register, zr,
17143                       $tmp1$$Register, $tmp2$$Register,
17144                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17145                       icnt2, $result$$Register, StrIntrinsicNode::UU);
17146   %}
17147   ins_pipe(pipe_class_memory);
17148 %}
17149 
17150 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17151                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17152                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17153 %{
17154   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17155   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17156   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17157          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17158   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
17159 
17160   ins_encode %{
17161     int icnt2 = (int)$int_cnt2$$constant;
17162     __ string_indexof($str1$$Register, $str2$$Register,
17163                       $cnt1$$Register, zr,
17164                       $tmp1$$Register, $tmp2$$Register,
17165                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17166                       icnt2, $result$$Register, StrIntrinsicNode::LL);
17167   %}
17168   ins_pipe(pipe_class_memory);
17169 %}
17170 
17171 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17172                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17173                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17174 %{
17175   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17176   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17177   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17178          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17179   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
17180 
17181   ins_encode %{
17182     int icnt2 = (int)$int_cnt2$$constant;
17183     __ string_indexof($str1$$Register, $str2$$Register,
17184                       $cnt1$$Register, zr,
17185                       $tmp1$$Register, $tmp2$$Register,
17186                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17187                       icnt2, $result$$Register, StrIntrinsicNode::UL);
17188   %}
17189   ins_pipe(pipe_class_memory);
17190 %}
17191 
17192 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17193                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17194                              iRegINoSp tmp3, rFlagsReg cr)
17195 %{
17196   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17197   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17198   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17199          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17200 
17201   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17202 
17203   ins_encode %{
17204     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17205                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
17206                            $tmp3$$Register);
17207   %}
17208   ins_pipe(pipe_class_memory);
17209 %}
17210 
17211 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17212                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17213                               iRegINoSp tmp3, rFlagsReg cr)
17214 %{
17215   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17216   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17217   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17218          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17219 
17220   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17221 
17222   ins_encode %{
17223     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17224                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
17225                             $tmp3$$Register);
17226   %}
17227   ins_pipe(pipe_class_memory);
17228 %}
17229 
17230 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17231                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17232                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17233   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17234   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17235   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17236   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17237   ins_encode %{
17238     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17239                                $result$$Register, $ztmp1$$FloatRegister,
17240                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17241                                $ptmp$$PRegister, true /* isL */);
17242   %}
17243   ins_pipe(pipe_class_memory);
17244 %}
17245 
17246 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17247                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17248                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17249   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17250   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17251   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17252   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17253   ins_encode %{
17254     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17255                                $result$$Register, $ztmp1$$FloatRegister,
17256                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17257                                $ptmp$$PRegister, false /* isL */);
17258   %}
17259   ins_pipe(pipe_class_memory);
17260 %}
17261 
17262 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17263                         iRegI_R0 result, rFlagsReg cr)
17264 %{
17265   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17266   match(Set result (StrEquals (Binary str1 str2) cnt));
17267   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17268 
17269   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17270   ins_encode %{
17271     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17272     __ string_equals($str1$$Register, $str2$$Register,
17273                      $result$$Register, $cnt$$Register, 1);
17274   %}
17275   ins_pipe(pipe_class_memory);
17276 %}
17277 
17278 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17279                         iRegI_R0 result, rFlagsReg cr)
17280 %{
17281   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17282   match(Set result (StrEquals (Binary str1 str2) cnt));
17283   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17284 
17285   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17286   ins_encode %{
17287     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17288     __ string_equals($str1$$Register, $str2$$Register,
17289                      $result$$Register, $cnt$$Register, 2);
17290   %}
17291   ins_pipe(pipe_class_memory);
17292 %}
17293 
17294 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17295                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17296                        iRegP_R10 tmp, rFlagsReg cr)
17297 %{
17298   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17299   match(Set result (AryEq ary1 ary2));
17300   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17301 
17302   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
17303   ins_encode %{
17304     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17305                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17306                                    $result$$Register, $tmp$$Register, 1);
17307     if (tpc == NULL) {
17308       ciEnv::current()->record_failure("CodeCache is full");
17309       return;
17310     }
17311   %}
17312   ins_pipe(pipe_class_memory);
17313 %}
17314 
17315 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17316                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17317                        iRegP_R10 tmp, rFlagsReg cr)
17318 %{
17319   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17320   match(Set result (AryEq ary1 ary2));
17321   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17322 
17323   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
17324   ins_encode %{
17325     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17326                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17327                                    $result$$Register, $tmp$$Register, 2);
17328     if (tpc == NULL) {
17329       ciEnv::current()->record_failure("CodeCache is full");
17330       return;
17331     }
17332   %}
17333   ins_pipe(pipe_class_memory);
17334 %}
17335 
17336 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17337 %{
17338   match(Set result (CountPositives ary1 len));
17339   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17340   format %{ "count positives byte[] $ary1,$len -> $result" %}
17341   ins_encode %{
17342     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17343     if (tpc == NULL) {
17344       ciEnv::current()->record_failure("CodeCache is full");
17345       return;
17346     }
17347   %}
17348   ins_pipe( pipe_slow );
17349 %}
17350 
17351 // fast char[] to byte[] compression
17352 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17353                          vRegD_V0 tmp1, vRegD_V1 tmp2,
17354                          vRegD_V2 tmp3, vRegD_V3 tmp4,
17355                          iRegI_R0 result, rFlagsReg cr)
17356 %{
17357   match(Set result (StrCompressedCopy src (Binary dst len)));
17358   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4,
17359          USE_KILL src, USE_KILL dst, USE len, KILL cr);
17360 
17361   format %{ "String Compress $src,$dst,$len -> $result  // KILL $src,$dst" %}
17362   ins_encode %{
17363     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17364                            $result$$Register,
17365                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17366                            $tmp3$$FloatRegister, $tmp4$$FloatRegister);
17367   %}
17368   ins_pipe(pipe_slow);
17369 %}
17370 
17371 // fast byte[] to char[] inflation
17372 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
17373                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
17374 %{
17375   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17376   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17377 
17378   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
17379   ins_encode %{
17380     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17381                                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17382                                         $tmp3$$FloatRegister, $tmp4$$Register);
17383     if (tpc == NULL) {
17384       ciEnv::current()->record_failure("CodeCache is full");
17385       return;
17386     }
17387   %}
17388   ins_pipe(pipe_class_memory);
17389 %}
17390 
17391 // encode char[] to byte[] in ISO_8859_1
17392 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17393                           vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17394                           vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17395                           iRegI_R0 result, rFlagsReg cr)
17396 %{
17397   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17398   match(Set result (EncodeISOArray src (Binary dst len)));
17399   effect(USE_KILL src, USE_KILL dst, USE len,
17400          KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17401 
17402   format %{ "Encode ISO array $src,$dst,$len -> $result" %}
17403   ins_encode %{
17404     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17405                         $result$$Register, false,
17406                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17407                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17408   %}
17409   ins_pipe(pipe_class_memory);
17410 %}
17411 
17412 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17413                             vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17414                             vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17415                             iRegI_R0 result, rFlagsReg cr)
17416 %{
17417   predicate(((EncodeISOArrayNode*)n)->is_ascii());
17418   match(Set result (EncodeISOArray src (Binary dst len)));
17419   effect(USE_KILL src, USE_KILL dst, USE len,
17420          KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17421 
17422   format %{ "Encode ASCII array $src,$dst,$len -> $result" %}
17423   ins_encode %{
17424     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17425                         $result$$Register, true,
17426                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17427                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17428   %}
17429   ins_pipe(pipe_class_memory);
17430 %}
17431 
17432 //----------------------------- CompressBits/ExpandBits ------------------------
17433 
17434 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17435                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17436   match(Set dst (CompressBits src mask));
17437   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17438   format %{ "mov    $tsrc, $src\n\t"
17439             "mov    $tmask, $mask\n\t"
17440             "bext   $tdst, $tsrc, $tmask\n\t"
17441             "mov    $dst, $tdst"
17442           %}
17443   ins_encode %{
17444     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17445     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17446     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17447     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17448   %}
17449   ins_pipe(pipe_slow);
17450 %}
17451 
17452 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17453                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17454   match(Set dst (CompressBits (LoadI mem) mask));
17455   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17456   format %{ "ldrs   $tsrc, $mem\n\t"
17457             "ldrs   $tmask, $mask\n\t"
17458             "bext   $tdst, $tsrc, $tmask\n\t"
17459             "mov    $dst, $tdst"
17460           %}
17461   ins_encode %{
17462     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17463               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17464     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17465     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17466     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17467   %}
17468   ins_pipe(pipe_slow);
17469 %}
17470 
17471 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17472                            vRegD tdst, vRegD tsrc, vRegD tmask) %{
17473   match(Set dst (CompressBits src mask));
17474   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17475   format %{ "mov    $tsrc, $src\n\t"
17476             "mov    $tmask, $mask\n\t"
17477             "bext   $tdst, $tsrc, $tmask\n\t"
17478             "mov    $dst, $tdst"
17479           %}
17480   ins_encode %{
17481     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17482     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17483     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17484     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17485   %}
17486   ins_pipe(pipe_slow);
17487 %}
17488 
17489 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17490                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17491   match(Set dst (CompressBits (LoadL mem) mask));
17492   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17493   format %{ "ldrd   $tsrc, $mem\n\t"
17494             "ldrd   $tmask, $mask\n\t"
17495             "bext   $tdst, $tsrc, $tmask\n\t"
17496             "mov    $dst, $tdst"
17497           %}
17498   ins_encode %{
17499     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17500               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17501     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17502     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17503     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17504   %}
17505   ins_pipe(pipe_slow);
17506 %}
17507 
17508 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17509                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17510   match(Set dst (ExpandBits src mask));
17511   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17512   format %{ "mov    $tsrc, $src\n\t"
17513             "mov    $tmask, $mask\n\t"
17514             "bdep   $tdst, $tsrc, $tmask\n\t"
17515             "mov    $dst, $tdst"
17516           %}
17517   ins_encode %{
17518     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17519     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17520     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17521     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17522   %}
17523   ins_pipe(pipe_slow);
17524 %}
17525 
17526 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17527                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17528   match(Set dst (ExpandBits (LoadI mem) mask));
17529   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17530   format %{ "ldrs   $tsrc, $mem\n\t"
17531             "ldrs   $tmask, $mask\n\t"
17532             "bdep   $tdst, $tsrc, $tmask\n\t"
17533             "mov    $dst, $tdst"
17534           %}
17535   ins_encode %{
17536     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17537               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17538     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17539     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17540     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17541   %}
17542   ins_pipe(pipe_slow);
17543 %}
17544 
17545 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17546                          vRegD tdst, vRegD tsrc, vRegD tmask) %{
17547   match(Set dst (ExpandBits src mask));
17548   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17549   format %{ "mov    $tsrc, $src\n\t"
17550             "mov    $tmask, $mask\n\t"
17551             "bdep   $tdst, $tsrc, $tmask\n\t"
17552             "mov    $dst, $tdst"
17553           %}
17554   ins_encode %{
17555     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17556     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17557     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17558     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17559   %}
17560   ins_pipe(pipe_slow);
17561 %}
17562 
17563 
17564 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17565                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17566   match(Set dst (ExpandBits (LoadL mem) mask));
17567   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17568   format %{ "ldrd   $tsrc, $mem\n\t"
17569             "ldrd   $tmask, $mask\n\t"
17570             "bdep   $tdst, $tsrc, $tmask\n\t"
17571             "mov    $dst, $tdst"
17572           %}
17573   ins_encode %{
17574     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17575               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17576     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17577     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17578     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17579   %}
17580   ins_pipe(pipe_slow);
17581 %}
17582 
17583 // ============================================================================
17584 // This name is KNOWN by the ADLC and cannot be changed.
17585 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17586 // for this guy.
17587 instruct tlsLoadP(thread_RegP dst)
17588 %{
17589   match(Set dst (ThreadLocal));
17590 
17591   ins_cost(0);
17592 
17593   format %{ " -- \t// $dst=Thread::current(), empty" %}
17594 
17595   size(0);
17596 
17597   ins_encode( /*empty*/ );
17598 
17599   ins_pipe(pipe_class_empty);
17600 %}
17601 
17602 //----------PEEPHOLE RULES-----------------------------------------------------
17603 // These must follow all instruction definitions as they use the names
17604 // defined in the instructions definitions.
17605 //
17606 // peepmatch ( root_instr_name [preceding_instruction]* );
17607 //
17608 // peepconstraint %{
17609 // (instruction_number.operand_name relational_op instruction_number.operand_name
17610 //  [, ...] );
17611 // // instruction numbers are zero-based using left to right order in peepmatch
17612 //
17613 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17614 // // provide an instruction_number.operand_name for each operand that appears
17615 // // in the replacement instruction's match rule
17616 //
17617 // ---------VM FLAGS---------------------------------------------------------
17618 //
17619 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17620 //
17621 // Each peephole rule is given an identifying number starting with zero and
17622 // increasing by one in the order seen by the parser.  An individual peephole
17623 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17624 // on the command-line.
17625 //
17626 // ---------CURRENT LIMITATIONS----------------------------------------------
17627 //
17628 // Only match adjacent instructions in same basic block
17629 // Only equality constraints
17630 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17631 // Only one replacement instruction
17632 //
17633 // ---------EXAMPLE----------------------------------------------------------
17634 //
17635 // // pertinent parts of existing instructions in architecture description
17636 // instruct movI(iRegINoSp dst, iRegI src)
17637 // %{
17638 //   match(Set dst (CopyI src));
17639 // %}
17640 //
17641 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17642 // %{
17643 //   match(Set dst (AddI dst src));
17644 //   effect(KILL cr);
17645 // %}
17646 //
17647 // // Change (inc mov) to lea
17648 // peephole %{
17649 //   // increment preceded by register-register move
17650 //   peepmatch ( incI_iReg movI );
17651 //   // require that the destination register of the increment
17652 //   // match the destination register of the move
17653 //   peepconstraint ( 0.dst == 1.dst );
17654 //   // construct a replacement instruction that sets
17655 //   // the destination to ( move's source register + one )
17656 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17657 // %}
17658 //
17659 
17660 // Implementation no longer uses movX instructions since
17661 // machine-independent system no longer uses CopyX nodes.
17662 //
17663 // peephole
17664 // %{
17665 //   peepmatch (incI_iReg movI);
17666 //   peepconstraint (0.dst == 1.dst);
17667 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17668 // %}
17669 
17670 // peephole
17671 // %{
17672 //   peepmatch (decI_iReg movI);
17673 //   peepconstraint (0.dst == 1.dst);
17674 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17675 // %}
17676 
17677 // peephole
17678 // %{
17679 //   peepmatch (addI_iReg_imm movI);
17680 //   peepconstraint (0.dst == 1.dst);
17681 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17682 // %}
17683 
17684 // peephole
17685 // %{
17686 //   peepmatch (incL_iReg movL);
17687 //   peepconstraint (0.dst == 1.dst);
17688 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17689 // %}
17690 
17691 // peephole
17692 // %{
17693 //   peepmatch (decL_iReg movL);
17694 //   peepconstraint (0.dst == 1.dst);
17695 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17696 // %}
17697 
17698 // peephole
17699 // %{
17700 //   peepmatch (addL_iReg_imm movL);
17701 //   peepconstraint (0.dst == 1.dst);
17702 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17703 // %}
17704 
17705 // peephole
17706 // %{
17707 //   peepmatch (addP_iReg_imm movP);
17708 //   peepconstraint (0.dst == 1.dst);
17709 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17710 // %}
17711 
17712 // // Change load of spilled value to only a spill
17713 // instruct storeI(memory mem, iRegI src)
17714 // %{
17715 //   match(Set mem (StoreI mem src));
17716 // %}
17717 //
17718 // instruct loadI(iRegINoSp dst, memory mem)
17719 // %{
17720 //   match(Set dst (LoadI mem));
17721 // %}
17722 //
17723 
17724 //----------SMARTSPILL RULES---------------------------------------------------
17725 // These must follow all instruction definitions as they use the names
17726 // defined in the instructions definitions.
17727 
17728 // Local Variables:
17729 // mode: c++
17730 // End: