1 //
    2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // architecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182 
  183   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  184   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  185   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  186   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  187 
  188   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  189   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  190   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  191   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  192 
  193   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  194   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  195   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  196   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  197 
  198   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  199   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  200   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  201   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  202 
  203   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  204   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  205   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  206   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  207 
  208   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  209   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  210   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  211   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  212 
  213   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  214   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  215   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  216   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  217 
  218   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  219   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  220   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  221   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  222 
  223   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  224   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  225   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  226   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  227 
  228   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  229   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  230   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  231   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  232 
  233   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  234   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  235   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  236   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  237 
  238   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  239   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  240   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  241   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  242 
  243   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  244   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  245   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  246   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  247 
  248   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  249   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  250   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  251   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  252 
  253   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  254   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  255   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  256   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  257 
  258   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  259   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  260   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  261   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  262 
  263   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  264   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  265   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  266   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  267 
  268   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  269   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  270   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  271   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  272 
  273   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  274   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  275   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  276   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  277 
  278   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  279   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  280   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  281   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  282 
  283   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  284   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  285   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  286   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  287 
  288   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  289   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  290   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  291   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  292 
  293   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  294   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  295   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  296   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  297 
  298   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  299   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  300   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  301   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  302 
  303   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  304   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  305   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  306   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  307 
  308   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  309   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  310   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  311   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  312 
  313   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  314   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  315   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  316   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  317 
  318   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  319   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  320   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  321   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  322 
  323   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  324   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  325   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  326   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  327 
  328   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  329   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  330   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  331   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  332 
  333   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  334   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  335   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  336   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  337 
  338 // ----------------------------
  339 // SVE Predicate Registers
  340 // ----------------------------
  341   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  342   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  343   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  344   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  345   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  346   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  347   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  348   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  349   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  350   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  351   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  352   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  353   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  354   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  355   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  356   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  357 
  358 // ----------------------------
  359 // Special Registers
  360 // ----------------------------
  361 
  362 // the AArch64 CSPR status flag register is not directly accessible as
  363 // instruction operand. the FPSR status flag register is a system
  364 // register which can be written/read using MSR/MRS but again does not
  365 // appear as an operand (a code identifying the FSPR occurs as an
  366 // immediate value in the instruction).
  367 
  368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  369 
  370 // Specify priority of register selection within phases of register
  371 // allocation.  Highest priority is first.  A useful heuristic is to
  372 // give registers a low priority when they are required by machine
  373 // instructions, like EAX and EDX on I486, and choose no-save registers
  374 // before save-on-call, & save-on-call before save-on-entry.  Registers
  375 // which participate in fixed calling sequences should come last.
  376 // Registers which are used as pairs must fall on an even boundary.
  377 
  378 alloc_class chunk0(
  379     // volatiles
  380     R10, R10_H,
  381     R11, R11_H,
  382     R12, R12_H,
  383     R13, R13_H,
  384     R14, R14_H,
  385     R15, R15_H,
  386     R16, R16_H,
  387     R17, R17_H,
  388     R18, R18_H,
  389 
  390     // arg registers
  391     R0, R0_H,
  392     R1, R1_H,
  393     R2, R2_H,
  394     R3, R3_H,
  395     R4, R4_H,
  396     R5, R5_H,
  397     R6, R6_H,
  398     R7, R7_H,
  399 
  400     // non-volatiles
  401     R19, R19_H,
  402     R20, R20_H,
  403     R21, R21_H,
  404     R22, R22_H,
  405     R23, R23_H,
  406     R24, R24_H,
  407     R25, R25_H,
  408     R26, R26_H,
  409 
  410     // non-allocatable registers
  411 
  412     R27, R27_H, // heapbase
  413     R28, R28_H, // thread
  414     R29, R29_H, // fp
  415     R30, R30_H, // lr
  416     R31, R31_H, // sp
  417     R8, R8_H,   // rscratch1
  418     R9, R9_H,   // rscratch2
  419 );
  420 
  421 alloc_class chunk1(
  422 
  423     // no save
  424     V16, V16_H, V16_J, V16_K,
  425     V17, V17_H, V17_J, V17_K,
  426     V18, V18_H, V18_J, V18_K,
  427     V19, V19_H, V19_J, V19_K,
  428     V20, V20_H, V20_J, V20_K,
  429     V21, V21_H, V21_J, V21_K,
  430     V22, V22_H, V22_J, V22_K,
  431     V23, V23_H, V23_J, V23_K,
  432     V24, V24_H, V24_J, V24_K,
  433     V25, V25_H, V25_J, V25_K,
  434     V26, V26_H, V26_J, V26_K,
  435     V27, V27_H, V27_J, V27_K,
  436     V28, V28_H, V28_J, V28_K,
  437     V29, V29_H, V29_J, V29_K,
  438     V30, V30_H, V30_J, V30_K,
  439     V31, V31_H, V31_J, V31_K,
  440 
  441     // arg registers
  442     V0, V0_H, V0_J, V0_K,
  443     V1, V1_H, V1_J, V1_K,
  444     V2, V2_H, V2_J, V2_K,
  445     V3, V3_H, V3_J, V3_K,
  446     V4, V4_H, V4_J, V4_K,
  447     V5, V5_H, V5_J, V5_K,
  448     V6, V6_H, V6_J, V6_K,
  449     V7, V7_H, V7_J, V7_K,
  450 
  451     // non-volatiles
  452     V8, V8_H, V8_J, V8_K,
  453     V9, V9_H, V9_J, V9_K,
  454     V10, V10_H, V10_J, V10_K,
  455     V11, V11_H, V11_J, V11_K,
  456     V12, V12_H, V12_J, V12_K,
  457     V13, V13_H, V13_J, V13_K,
  458     V14, V14_H, V14_J, V14_K,
  459     V15, V15_H, V15_J, V15_K,
  460 );
  461 
  462 alloc_class chunk2 (
  463     // Governing predicates for load/store and arithmetic
  464     P0,
  465     P1,
  466     P2,
  467     P3,
  468     P4,
  469     P5,
  470     P6,
  471 
  472     // Extra predicates
  473     P8,
  474     P9,
  475     P10,
  476     P11,
  477     P12,
  478     P13,
  479     P14,
  480     P15,
  481 
  482     // Preserved for all-true predicate
  483     P7,
  484 );
  485 
  486 alloc_class chunk3(RFLAGS);
  487 
  488 //----------Architecture Description Register Classes--------------------------
  489 // Several register classes are automatically defined based upon information in
  490 // this architecture description.
  491 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  493 //
  494 
  495 // Class for all 32 bit general purpose registers
  496 reg_class all_reg32(
  497     R0,
  498     R1,
  499     R2,
  500     R3,
  501     R4,
  502     R5,
  503     R6,
  504     R7,
  505     R10,
  506     R11,
  507     R12,
  508     R13,
  509     R14,
  510     R15,
  511     R16,
  512     R17,
  513     R18,
  514     R19,
  515     R20,
  516     R21,
  517     R22,
  518     R23,
  519     R24,
  520     R25,
  521     R26,
  522     R27,
  523     R28,
  524     R29,
  525     R30,
  526     R31
  527 );
  528 
  529 
  530 // Class for all 32 bit integer registers (excluding SP which
  531 // will never be used as an integer register)
  532 reg_class any_reg32 %{
  533   return _ANY_REG32_mask;
  534 %}
  535 
  536 // Singleton class for R0 int register
  537 reg_class int_r0_reg(R0);
  538 
  539 // Singleton class for R2 int register
  540 reg_class int_r2_reg(R2);
  541 
  542 // Singleton class for R3 int register
  543 reg_class int_r3_reg(R3);
  544 
  545 // Singleton class for R4 int register
  546 reg_class int_r4_reg(R4);
  547 
  548 // Singleton class for R31 int register
  549 reg_class int_r31_reg(R31);
  550 
  551 // Class for all 64 bit general purpose registers
  552 reg_class all_reg(
  553     R0, R0_H,
  554     R1, R1_H,
  555     R2, R2_H,
  556     R3, R3_H,
  557     R4, R4_H,
  558     R5, R5_H,
  559     R6, R6_H,
  560     R7, R7_H,
  561     R10, R10_H,
  562     R11, R11_H,
  563     R12, R12_H,
  564     R13, R13_H,
  565     R14, R14_H,
  566     R15, R15_H,
  567     R16, R16_H,
  568     R17, R17_H,
  569     R18, R18_H,
  570     R19, R19_H,
  571     R20, R20_H,
  572     R21, R21_H,
  573     R22, R22_H,
  574     R23, R23_H,
  575     R24, R24_H,
  576     R25, R25_H,
  577     R26, R26_H,
  578     R27, R27_H,
  579     R28, R28_H,
  580     R29, R29_H,
  581     R30, R30_H,
  582     R31, R31_H
  583 );
  584 
  585 // Class for all long integer registers (including SP)
  586 reg_class any_reg %{
  587   return _ANY_REG_mask;
  588 %}
  589 
  590 // Class for non-allocatable 32 bit registers
  591 reg_class non_allocatable_reg32(
  592 #ifdef R18_RESERVED
  593     // See comment in register_aarch64.hpp
  594     R18,                        // tls on Windows
  595 #endif
  596     R28,                        // thread
  597     R30,                        // lr
  598     R31                         // sp
  599 );
  600 
  601 // Class for non-allocatable 64 bit registers
  602 reg_class non_allocatable_reg(
  603 #ifdef R18_RESERVED
  604     // See comment in register_aarch64.hpp
  605     R18, R18_H,                 // tls on Windows, platform register on macOS
  606 #endif
  607     R28, R28_H,                 // thread
  608     R30, R30_H,                 // lr
  609     R31, R31_H                  // sp
  610 );
  611 
  612 // Class for all non-special integer registers
  613 reg_class no_special_reg32 %{
  614   return _NO_SPECIAL_REG32_mask;
  615 %}
  616 
  617 // Class for all non-special long integer registers
  618 reg_class no_special_reg %{
  619   return _NO_SPECIAL_REG_mask;
  620 %}
  621 
  622 // Class for 64 bit register r0
  623 reg_class r0_reg(
  624     R0, R0_H
  625 );
  626 
  627 // Class for 64 bit register r1
  628 reg_class r1_reg(
  629     R1, R1_H
  630 );
  631 
  632 // Class for 64 bit register r2
  633 reg_class r2_reg(
  634     R2, R2_H
  635 );
  636 
  637 // Class for 64 bit register r3
  638 reg_class r3_reg(
  639     R3, R3_H
  640 );
  641 
  642 // Class for 64 bit register r4
  643 reg_class r4_reg(
  644     R4, R4_H
  645 );
  646 
  647 // Class for 64 bit register r5
  648 reg_class r5_reg(
  649     R5, R5_H
  650 );
  651 
  652 // Class for 64 bit register r10
  653 reg_class r10_reg(
  654     R10, R10_H
  655 );
  656 
  657 // Class for 64 bit register r11
  658 reg_class r11_reg(
  659     R11, R11_H
  660 );
  661 
  662 // Class for method register
  663 reg_class method_reg(
  664     R12, R12_H
  665 );
  666 
  667 // Class for thread register
  668 reg_class thread_reg(
  669     R28, R28_H
  670 );
  671 
  672 // Class for frame pointer register
  673 reg_class fp_reg(
  674     R29, R29_H
  675 );
  676 
  677 // Class for link register
  678 reg_class lr_reg(
  679     R30, R30_H
  680 );
  681 
  682 // Class for long sp register
  683 reg_class sp_reg(
  684   R31, R31_H
  685 );
  686 
  687 // Class for all pointer registers
  688 reg_class ptr_reg %{
  689   return _PTR_REG_mask;
  690 %}
  691 
  692 // Class for all non_special pointer registers
  693 reg_class no_special_ptr_reg %{
  694   return _NO_SPECIAL_PTR_REG_mask;
  695 %}
  696 
  697 // Class for all non_special pointer registers (excluding rfp)
  698 reg_class no_special_no_rfp_ptr_reg %{
  699   return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
  700 %}
  701 
  702 // Class for all float registers
  703 reg_class float_reg(
  704     V0,
  705     V1,
  706     V2,
  707     V3,
  708     V4,
  709     V5,
  710     V6,
  711     V7,
  712     V8,
  713     V9,
  714     V10,
  715     V11,
  716     V12,
  717     V13,
  718     V14,
  719     V15,
  720     V16,
  721     V17,
  722     V18,
  723     V19,
  724     V20,
  725     V21,
  726     V22,
  727     V23,
  728     V24,
  729     V25,
  730     V26,
  731     V27,
  732     V28,
  733     V29,
  734     V30,
  735     V31
  736 );
  737 
  738 // Double precision float registers have virtual `high halves' that
  739 // are needed by the allocator.
  740 // Class for all double registers
  741 reg_class double_reg(
  742     V0, V0_H,
  743     V1, V1_H,
  744     V2, V2_H,
  745     V3, V3_H,
  746     V4, V4_H,
  747     V5, V5_H,
  748     V6, V6_H,
  749     V7, V7_H,
  750     V8, V8_H,
  751     V9, V9_H,
  752     V10, V10_H,
  753     V11, V11_H,
  754     V12, V12_H,
  755     V13, V13_H,
  756     V14, V14_H,
  757     V15, V15_H,
  758     V16, V16_H,
  759     V17, V17_H,
  760     V18, V18_H,
  761     V19, V19_H,
  762     V20, V20_H,
  763     V21, V21_H,
  764     V22, V22_H,
  765     V23, V23_H,
  766     V24, V24_H,
  767     V25, V25_H,
  768     V26, V26_H,
  769     V27, V27_H,
  770     V28, V28_H,
  771     V29, V29_H,
  772     V30, V30_H,
  773     V31, V31_H
  774 );
  775 
  776 // Class for all SVE vector registers.
  777 reg_class vectora_reg (
  778     V0, V0_H, V0_J, V0_K,
  779     V1, V1_H, V1_J, V1_K,
  780     V2, V2_H, V2_J, V2_K,
  781     V3, V3_H, V3_J, V3_K,
  782     V4, V4_H, V4_J, V4_K,
  783     V5, V5_H, V5_J, V5_K,
  784     V6, V6_H, V6_J, V6_K,
  785     V7, V7_H, V7_J, V7_K,
  786     V8, V8_H, V8_J, V8_K,
  787     V9, V9_H, V9_J, V9_K,
  788     V10, V10_H, V10_J, V10_K,
  789     V11, V11_H, V11_J, V11_K,
  790     V12, V12_H, V12_J, V12_K,
  791     V13, V13_H, V13_J, V13_K,
  792     V14, V14_H, V14_J, V14_K,
  793     V15, V15_H, V15_J, V15_K,
  794     V16, V16_H, V16_J, V16_K,
  795     V17, V17_H, V17_J, V17_K,
  796     V18, V18_H, V18_J, V18_K,
  797     V19, V19_H, V19_J, V19_K,
  798     V20, V20_H, V20_J, V20_K,
  799     V21, V21_H, V21_J, V21_K,
  800     V22, V22_H, V22_J, V22_K,
  801     V23, V23_H, V23_J, V23_K,
  802     V24, V24_H, V24_J, V24_K,
  803     V25, V25_H, V25_J, V25_K,
  804     V26, V26_H, V26_J, V26_K,
  805     V27, V27_H, V27_J, V27_K,
  806     V28, V28_H, V28_J, V28_K,
  807     V29, V29_H, V29_J, V29_K,
  808     V30, V30_H, V30_J, V30_K,
  809     V31, V31_H, V31_J, V31_K,
  810 );
  811 
  812 // Class for all 64bit vector registers
  813 reg_class vectord_reg(
  814     V0, V0_H,
  815     V1, V1_H,
  816     V2, V2_H,
  817     V3, V3_H,
  818     V4, V4_H,
  819     V5, V5_H,
  820     V6, V6_H,
  821     V7, V7_H,
  822     V8, V8_H,
  823     V9, V9_H,
  824     V10, V10_H,
  825     V11, V11_H,
  826     V12, V12_H,
  827     V13, V13_H,
  828     V14, V14_H,
  829     V15, V15_H,
  830     V16, V16_H,
  831     V17, V17_H,
  832     V18, V18_H,
  833     V19, V19_H,
  834     V20, V20_H,
  835     V21, V21_H,
  836     V22, V22_H,
  837     V23, V23_H,
  838     V24, V24_H,
  839     V25, V25_H,
  840     V26, V26_H,
  841     V27, V27_H,
  842     V28, V28_H,
  843     V29, V29_H,
  844     V30, V30_H,
  845     V31, V31_H
  846 );
  847 
  848 // Class for all 128bit vector registers
  849 reg_class vectorx_reg(
  850     V0, V0_H, V0_J, V0_K,
  851     V1, V1_H, V1_J, V1_K,
  852     V2, V2_H, V2_J, V2_K,
  853     V3, V3_H, V3_J, V3_K,
  854     V4, V4_H, V4_J, V4_K,
  855     V5, V5_H, V5_J, V5_K,
  856     V6, V6_H, V6_J, V6_K,
  857     V7, V7_H, V7_J, V7_K,
  858     V8, V8_H, V8_J, V8_K,
  859     V9, V9_H, V9_J, V9_K,
  860     V10, V10_H, V10_J, V10_K,
  861     V11, V11_H, V11_J, V11_K,
  862     V12, V12_H, V12_J, V12_K,
  863     V13, V13_H, V13_J, V13_K,
  864     V14, V14_H, V14_J, V14_K,
  865     V15, V15_H, V15_J, V15_K,
  866     V16, V16_H, V16_J, V16_K,
  867     V17, V17_H, V17_J, V17_K,
  868     V18, V18_H, V18_J, V18_K,
  869     V19, V19_H, V19_J, V19_K,
  870     V20, V20_H, V20_J, V20_K,
  871     V21, V21_H, V21_J, V21_K,
  872     V22, V22_H, V22_J, V22_K,
  873     V23, V23_H, V23_J, V23_K,
  874     V24, V24_H, V24_J, V24_K,
  875     V25, V25_H, V25_J, V25_K,
  876     V26, V26_H, V26_J, V26_K,
  877     V27, V27_H, V27_J, V27_K,
  878     V28, V28_H, V28_J, V28_K,
  879     V29, V29_H, V29_J, V29_K,
  880     V30, V30_H, V30_J, V30_K,
  881     V31, V31_H, V31_J, V31_K
  882 );
  883 
  884 // Class for vector register V10
  885 reg_class v10_veca_reg(
  886     V10, V10_H, V10_J, V10_K
  887 );
  888 
  889 // Class for vector register V11
  890 reg_class v11_veca_reg(
  891     V11, V11_H, V11_J, V11_K
  892 );
  893 
  894 // Class for vector register V12
  895 reg_class v12_veca_reg(
  896     V12, V12_H, V12_J, V12_K
  897 );
  898 
  899 // Class for vector register V13
  900 reg_class v13_veca_reg(
  901     V13, V13_H, V13_J, V13_K
  902 );
  903 
  904 // Class for vector register V17
  905 reg_class v17_veca_reg(
  906     V17, V17_H, V17_J, V17_K
  907 );
  908 
  909 // Class for vector register V18
  910 reg_class v18_veca_reg(
  911     V18, V18_H, V18_J, V18_K
  912 );
  913 
  914 // Class for vector register V23
  915 reg_class v23_veca_reg(
  916     V23, V23_H, V23_J, V23_K
  917 );
  918 
  919 // Class for vector register V24
  920 reg_class v24_veca_reg(
  921     V24, V24_H, V24_J, V24_K
  922 );
  923 
  924 // Class for 128 bit register v0
  925 reg_class v0_reg(
  926     V0, V0_H
  927 );
  928 
  929 // Class for 128 bit register v1
  930 reg_class v1_reg(
  931     V1, V1_H
  932 );
  933 
  934 // Class for 128 bit register v2
  935 reg_class v2_reg(
  936     V2, V2_H
  937 );
  938 
  939 // Class for 128 bit register v3
  940 reg_class v3_reg(
  941     V3, V3_H
  942 );
  943 
  944 // Class for 128 bit register v4
  945 reg_class v4_reg(
  946     V4, V4_H
  947 );
  948 
  949 // Class for 128 bit register v5
  950 reg_class v5_reg(
  951     V5, V5_H
  952 );
  953 
  954 // Class for 128 bit register v6
  955 reg_class v6_reg(
  956     V6, V6_H
  957 );
  958 
  959 // Class for 128 bit register v7
  960 reg_class v7_reg(
  961     V7, V7_H
  962 );
  963 
  964 // Class for 128 bit register v8
  965 reg_class v8_reg(
  966     V8, V8_H
  967 );
  968 
  969 // Class for 128 bit register v9
  970 reg_class v9_reg(
  971     V9, V9_H
  972 );
  973 
  974 // Class for 128 bit register v10
  975 reg_class v10_reg(
  976     V10, V10_H
  977 );
  978 
  979 // Class for 128 bit register v11
  980 reg_class v11_reg(
  981     V11, V11_H
  982 );
  983 
  984 // Class for 128 bit register v12
  985 reg_class v12_reg(
  986     V12, V12_H
  987 );
  988 
  989 // Class for 128 bit register v13
  990 reg_class v13_reg(
  991     V13, V13_H
  992 );
  993 
  994 // Class for 128 bit register v14
  995 reg_class v14_reg(
  996     V14, V14_H
  997 );
  998 
  999 // Class for 128 bit register v15
 1000 reg_class v15_reg(
 1001     V15, V15_H
 1002 );
 1003 
 1004 // Class for 128 bit register v16
 1005 reg_class v16_reg(
 1006     V16, V16_H
 1007 );
 1008 
 1009 // Class for 128 bit register v17
 1010 reg_class v17_reg(
 1011     V17, V17_H
 1012 );
 1013 
 1014 // Class for 128 bit register v18
 1015 reg_class v18_reg(
 1016     V18, V18_H
 1017 );
 1018 
 1019 // Class for 128 bit register v19
 1020 reg_class v19_reg(
 1021     V19, V19_H
 1022 );
 1023 
 1024 // Class for 128 bit register v20
 1025 reg_class v20_reg(
 1026     V20, V20_H
 1027 );
 1028 
 1029 // Class for 128 bit register v21
 1030 reg_class v21_reg(
 1031     V21, V21_H
 1032 );
 1033 
 1034 // Class for 128 bit register v22
 1035 reg_class v22_reg(
 1036     V22, V22_H
 1037 );
 1038 
 1039 // Class for 128 bit register v23
 1040 reg_class v23_reg(
 1041     V23, V23_H
 1042 );
 1043 
 1044 // Class for 128 bit register v24
 1045 reg_class v24_reg(
 1046     V24, V24_H
 1047 );
 1048 
 1049 // Class for 128 bit register v25
 1050 reg_class v25_reg(
 1051     V25, V25_H
 1052 );
 1053 
 1054 // Class for 128 bit register v26
 1055 reg_class v26_reg(
 1056     V26, V26_H
 1057 );
 1058 
 1059 // Class for 128 bit register v27
 1060 reg_class v27_reg(
 1061     V27, V27_H
 1062 );
 1063 
 1064 // Class for 128 bit register v28
 1065 reg_class v28_reg(
 1066     V28, V28_H
 1067 );
 1068 
 1069 // Class for 128 bit register v29
 1070 reg_class v29_reg(
 1071     V29, V29_H
 1072 );
 1073 
 1074 // Class for 128 bit register v30
 1075 reg_class v30_reg(
 1076     V30, V30_H
 1077 );
 1078 
 1079 // Class for 128 bit register v31
 1080 reg_class v31_reg(
 1081     V31, V31_H
 1082 );
 1083 
 1084 // Class for all SVE predicate registers.
 1085 reg_class pr_reg (
 1086     P0,
 1087     P1,
 1088     P2,
 1089     P3,
 1090     P4,
 1091     P5,
 1092     P6,
 1093     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1094     P8,
 1095     P9,
 1096     P10,
 1097     P11,
 1098     P12,
 1099     P13,
 1100     P14,
 1101     P15
 1102 );
 1103 
 1104 // Class for SVE governing predicate registers, which are used
 1105 // to determine the active elements of a predicated instruction.
 1106 reg_class gov_pr (
 1107     P0,
 1108     P1,
 1109     P2,
 1110     P3,
 1111     P4,
 1112     P5,
 1113     P6,
 1114     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1115 );
 1116 
 1117 reg_class p0_reg(P0);
 1118 reg_class p1_reg(P1);
 1119 
 1120 // Singleton class for condition codes
 1121 reg_class int_flags(RFLAGS);
 1122 
 1123 %}
 1124 
 1125 //----------DEFINITION BLOCK---------------------------------------------------
 1126 // Define name --> value mappings to inform the ADLC of an integer valued name
 1127 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1128 // Format:
 1129 //        int_def  <name>         ( <int_value>, <expression>);
 1130 // Generated Code in ad_<arch>.hpp
 1131 //        #define  <name>   (<expression>)
 1132 //        // value == <int_value>
 1133 // Generated code in ad_<arch>.cpp adlc_verification()
 1134 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1135 //
 1136 
 1137 // we follow the ppc-aix port in using a simple cost model which ranks
 1138 // register operations as cheap, memory ops as more expensive and
 1139 // branches as most expensive. the first two have a low as well as a
 1140 // normal cost. huge cost appears to be a way of saying don't do
 1141 // something
 1142 
 1143 definitions %{
 1144   // The default cost (of a register move instruction).
 1145   int_def INSN_COST            (    100,     100);
 1146   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1147   int_def CALL_COST            (    200,     2 * INSN_COST);
 1148   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1149 %}
 1150 
 1151 
 1152 //----------SOURCE BLOCK-------------------------------------------------------
 1153 // This is a block of C++ code which provides values, functions, and
 1154 // definitions necessary in the rest of the architecture description
 1155 
 1156 source_hpp %{
 1157 
 1158 #include "asm/macroAssembler.hpp"
 1159 #include "gc/shared/barrierSetAssembler.hpp"
 1160 #include "gc/shared/cardTable.hpp"
 1161 #include "gc/shared/cardTableBarrierSet.hpp"
 1162 #include "gc/shared/collectedHeap.hpp"
 1163 #include "opto/addnode.hpp"
 1164 #include "opto/convertnode.hpp"
 1165 #include "runtime/objectMonitor.hpp"
 1166 
 1167 extern RegMask _ANY_REG32_mask;
 1168 extern RegMask _ANY_REG_mask;
 1169 extern RegMask _PTR_REG_mask;
 1170 extern RegMask _NO_SPECIAL_REG32_mask;
 1171 extern RegMask _NO_SPECIAL_REG_mask;
 1172 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1173 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
 1174 
 1175 class CallStubImpl {
 1176 
 1177   //--------------------------------------------------------------
 1178   //---<  Used for optimization in Compile::shorten_branches  >---
 1179   //--------------------------------------------------------------
 1180 
 1181  public:
 1182   // Size of call trampoline stub.
 1183   static uint size_call_trampoline() {
 1184     return 0; // no call trampolines on this platform
 1185   }
 1186 
 1187   // number of relocations needed by a call trampoline stub
 1188   static uint reloc_call_trampoline() {
 1189     return 0; // no call trampolines on this platform
 1190   }
 1191 };
 1192 
 1193 class HandlerImpl {
 1194 
 1195  public:
 1196 
 1197   static int emit_exception_handler(C2_MacroAssembler *masm);
 1198   static int emit_deopt_handler(C2_MacroAssembler* masm);
 1199 
 1200   static uint size_exception_handler() {
 1201     return MacroAssembler::far_codestub_branch_size();
 1202   }
 1203 
 1204   static uint size_deopt_handler() {
 1205     // count one adr and one far branch instruction
 1206     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1207   }
 1208 };
 1209 
 1210 class Node::PD {
 1211 public:
 1212   enum NodeFlags {
 1213     _last_flag = Node::_last_flag
 1214   };
 1215 };
 1216 
 1217   bool is_CAS(int opcode, bool maybe_volatile);
 1218 
 1219   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1220 
 1221   bool unnecessary_acquire(const Node *barrier);
 1222   bool needs_acquiring_load(const Node *load);
 1223 
 1224   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1225 
 1226   bool unnecessary_release(const Node *barrier);
 1227   bool unnecessary_volatile(const Node *barrier);
 1228   bool needs_releasing_store(const Node *store);
 1229 
 1230   // predicate controlling translation of CompareAndSwapX
 1231   bool needs_acquiring_load_exclusive(const Node *load);
 1232 
 1233   // predicate controlling addressing modes
 1234   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1235 
 1236   // Convert BootTest condition to Assembler condition.
 1237   // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 1238   Assembler::Condition to_assembler_cond(BoolTest::mask cond);
 1239 %}
 1240 
 1241 source %{
 1242 
 1243   // Derived RegMask with conditionally allocatable registers
 1244 
 1245   void PhaseOutput::pd_perform_mach_node_analysis() {
 1246   }
 1247 
 1248   int MachNode::pd_alignment_required() const {
 1249     return 1;
 1250   }
 1251 
 1252   int MachNode::compute_padding(int current_offset) const {
 1253     return 0;
 1254   }
 1255 
 1256   RegMask _ANY_REG32_mask;
 1257   RegMask _ANY_REG_mask;
 1258   RegMask _PTR_REG_mask;
 1259   RegMask _NO_SPECIAL_REG32_mask;
 1260   RegMask _NO_SPECIAL_REG_mask;
 1261   RegMask _NO_SPECIAL_PTR_REG_mask;
 1262   RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
 1263 
 1264   void reg_mask_init() {
 1265     // We derive below RegMask(s) from the ones which are auto-generated from
 1266     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1267     // registers conditionally reserved.
 1268 
 1269     _ANY_REG32_mask = _ALL_REG32_mask;
 1270     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1271 
 1272     _ANY_REG_mask = _ALL_REG_mask;
 1273 
 1274     _PTR_REG_mask = _ALL_REG_mask;
 1275 
 1276     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1277     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1278 
 1279     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1280     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1281 
 1282     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1283     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1284 
 1285     // r27 is not allocatable when compressed oops is on and heapbase is not
 1286     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1287     if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
 1288       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1289       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1290       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1291     }
 1292 
 1293     // r29 is not allocatable when PreserveFramePointer is on
 1294     if (PreserveFramePointer) {
 1295       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1296       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1297       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1298     }
 1299 
 1300     _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
 1301     _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1302   }
 1303 
 1304   // Optimizaton of volatile gets and puts
 1305   // -------------------------------------
 1306   //
 1307   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1308   // use to implement volatile reads and writes. For a volatile read
 1309   // we simply need
 1310   //
 1311   //   ldar<x>
 1312   //
 1313   // and for a volatile write we need
 1314   //
 1315   //   stlr<x>
 1316   //
 1317   // Alternatively, we can implement them by pairing a normal
 1318   // load/store with a memory barrier. For a volatile read we need
 1319   //
 1320   //   ldr<x>
 1321   //   dmb ishld
 1322   //
 1323   // for a volatile write
 1324   //
 1325   //   dmb ish
 1326   //   str<x>
 1327   //   dmb ish
 1328   //
 1329   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1330   // sequences. These are normally translated to an instruction
 1331   // sequence like the following
 1332   //
 1333   //   dmb      ish
 1334   // retry:
 1335   //   ldxr<x>   rval raddr
 1336   //   cmp       rval rold
 1337   //   b.ne done
 1338   //   stlxr<x>  rval, rnew, rold
 1339   //   cbnz      rval retry
 1340   // done:
 1341   //   cset      r0, eq
 1342   //   dmb ishld
 1343   //
 1344   // Note that the exclusive store is already using an stlxr
 1345   // instruction. That is required to ensure visibility to other
 1346   // threads of the exclusive write (assuming it succeeds) before that
 1347   // of any subsequent writes.
 1348   //
 1349   // The following instruction sequence is an improvement on the above
 1350   //
 1351   // retry:
 1352   //   ldaxr<x>  rval raddr
 1353   //   cmp       rval rold
 1354   //   b.ne done
 1355   //   stlxr<x>  rval, rnew, rold
 1356   //   cbnz      rval retry
 1357   // done:
 1358   //   cset      r0, eq
 1359   //
 1360   // We don't need the leading dmb ish since the stlxr guarantees
 1361   // visibility of prior writes in the case that the swap is
 1362   // successful. Crucially we don't have to worry about the case where
 1363   // the swap is not successful since no valid program should be
 1364   // relying on visibility of prior changes by the attempting thread
 1365   // in the case where the CAS fails.
 1366   //
 1367   // Similarly, we don't need the trailing dmb ishld if we substitute
 1368   // an ldaxr instruction since that will provide all the guarantees we
 1369   // require regarding observation of changes made by other threads
 1370   // before any change to the CAS address observed by the load.
 1371   //
 1372   // In order to generate the desired instruction sequence we need to
 1373   // be able to identify specific 'signature' ideal graph node
 1374   // sequences which i) occur as a translation of a volatile reads or
 1375   // writes or CAS operations and ii) do not occur through any other
 1376   // translation or graph transformation. We can then provide
 1377   // alternative aldc matching rules which translate these node
 1378   // sequences to the desired machine code sequences. Selection of the
 1379   // alternative rules can be implemented by predicates which identify
 1380   // the relevant node sequences.
 1381   //
 1382   // The ideal graph generator translates a volatile read to the node
 1383   // sequence
 1384   //
 1385   //   LoadX[mo_acquire]
 1386   //   MemBarAcquire
 1387   //
 1388   // As a special case when using the compressed oops optimization we
 1389   // may also see this variant
 1390   //
 1391   //   LoadN[mo_acquire]
 1392   //   DecodeN
 1393   //   MemBarAcquire
 1394   //
 1395   // A volatile write is translated to the node sequence
 1396   //
 1397   //   MemBarRelease
 1398   //   StoreX[mo_release] {CardMark}-optional
 1399   //   MemBarVolatile
 1400   //
 1401   // n.b. the above node patterns are generated with a strict
 1402   // 'signature' configuration of input and output dependencies (see
 1403   // the predicates below for exact details). The card mark may be as
 1404   // simple as a few extra nodes or, in a few GC configurations, may
 1405   // include more complex control flow between the leading and
 1406   // trailing memory barriers. However, whatever the card mark
 1407   // configuration these signatures are unique to translated volatile
 1408   // reads/stores -- they will not appear as a result of any other
 1409   // bytecode translation or inlining nor as a consequence of
 1410   // optimizing transforms.
 1411   //
 1412   // We also want to catch inlined unsafe volatile gets and puts and
 1413   // be able to implement them using either ldar<x>/stlr<x> or some
 1414   // combination of ldr<x>/stlr<x> and dmb instructions.
 1415   //
 1416   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1417   // normal volatile put node sequence containing an extra cpuorder
 1418   // membar
 1419   //
 1420   //   MemBarRelease
 1421   //   MemBarCPUOrder
 1422   //   StoreX[mo_release] {CardMark}-optional
 1423   //   MemBarCPUOrder
 1424   //   MemBarVolatile
 1425   //
 1426   // n.b. as an aside, a cpuorder membar is not itself subject to
 1427   // matching and translation by adlc rules.  However, the rule
 1428   // predicates need to detect its presence in order to correctly
 1429   // select the desired adlc rules.
 1430   //
 1431   // Inlined unsafe volatile gets manifest as a slightly different
 1432   // node sequence to a normal volatile get because of the
 1433   // introduction of some CPUOrder memory barriers to bracket the
 1434   // Load. However, but the same basic skeleton of a LoadX feeding a
 1435   // MemBarAcquire, possibly through an optional DecodeN, is still
 1436   // present
 1437   //
 1438   //   MemBarCPUOrder
 1439   //        ||       \\
 1440   //   MemBarCPUOrder LoadX[mo_acquire]
 1441   //        ||            |
 1442   //        ||       {DecodeN} optional
 1443   //        ||       /
 1444   //     MemBarAcquire
 1445   //
 1446   // In this case the acquire membar does not directly depend on the
 1447   // load. However, we can be sure that the load is generated from an
 1448   // inlined unsafe volatile get if we see it dependent on this unique
 1449   // sequence of membar nodes. Similarly, given an acquire membar we
 1450   // can know that it was added because of an inlined unsafe volatile
 1451   // get if it is fed and feeds a cpuorder membar and if its feed
 1452   // membar also feeds an acquiring load.
 1453   //
 1454   // Finally an inlined (Unsafe) CAS operation is translated to the
 1455   // following ideal graph
 1456   //
 1457   //   MemBarRelease
 1458   //   MemBarCPUOrder
 1459   //   CompareAndSwapX {CardMark}-optional
 1460   //   MemBarCPUOrder
 1461   //   MemBarAcquire
 1462   //
 1463   // So, where we can identify these volatile read and write
 1464   // signatures we can choose to plant either of the above two code
 1465   // sequences. For a volatile read we can simply plant a normal
 1466   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1467   // also choose to inhibit translation of the MemBarAcquire and
 1468   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1469   //
 1470   // When we recognise a volatile store signature we can choose to
 1471   // plant at a dmb ish as a translation for the MemBarRelease, a
 1472   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1473   // Alternatively, we can inhibit translation of the MemBarRelease
 1474   // and MemBarVolatile and instead plant a simple stlr<x>
 1475   // instruction.
 1476   //
 1477   // when we recognise a CAS signature we can choose to plant a dmb
 1478   // ish as a translation for the MemBarRelease, the conventional
 1479   // macro-instruction sequence for the CompareAndSwap node (which
 1480   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1481   // Alternatively, we can elide generation of the dmb instructions
 1482   // and plant the alternative CompareAndSwap macro-instruction
 1483   // sequence (which uses ldaxr<x>).
 1484   //
 1485   // Of course, the above only applies when we see these signature
 1486   // configurations. We still want to plant dmb instructions in any
 1487   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1488   // MemBarVolatile. For example, at the end of a constructor which
 1489   // writes final/volatile fields we will see a MemBarRelease
 1490   // instruction and this needs a 'dmb ish' lest we risk the
 1491   // constructed object being visible without making the
 1492   // final/volatile field writes visible.
 1493   //
 1494   // n.b. the translation rules below which rely on detection of the
 1495   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1496   // If we see anything other than the signature configurations we
 1497   // always just translate the loads and stores to ldr<x> and str<x>
 1498   // and translate acquire, release and volatile membars to the
 1499   // relevant dmb instructions.
 1500   //
 1501 
 1502   // is_CAS(int opcode, bool maybe_volatile)
 1503   //
 1504   // return true if opcode is one of the possible CompareAndSwapX
 1505   // values otherwise false.
 1506 
 1507   bool is_CAS(int opcode, bool maybe_volatile)
 1508   {
 1509     switch(opcode) {
 1510       // We handle these
 1511     case Op_CompareAndSwapI:
 1512     case Op_CompareAndSwapL:
 1513     case Op_CompareAndSwapP:
 1514     case Op_CompareAndSwapN:
 1515     case Op_ShenandoahCompareAndSwapP:
 1516     case Op_ShenandoahCompareAndSwapN:
 1517     case Op_CompareAndSwapB:
 1518     case Op_CompareAndSwapS:
 1519     case Op_GetAndSetI:
 1520     case Op_GetAndSetL:
 1521     case Op_GetAndSetP:
 1522     case Op_GetAndSetN:
 1523     case Op_GetAndAddI:
 1524     case Op_GetAndAddL:
 1525       return true;
 1526     case Op_CompareAndExchangeI:
 1527     case Op_CompareAndExchangeN:
 1528     case Op_CompareAndExchangeB:
 1529     case Op_CompareAndExchangeS:
 1530     case Op_CompareAndExchangeL:
 1531     case Op_CompareAndExchangeP:
 1532     case Op_WeakCompareAndSwapB:
 1533     case Op_WeakCompareAndSwapS:
 1534     case Op_WeakCompareAndSwapI:
 1535     case Op_WeakCompareAndSwapL:
 1536     case Op_WeakCompareAndSwapP:
 1537     case Op_WeakCompareAndSwapN:
 1538     case Op_ShenandoahWeakCompareAndSwapP:
 1539     case Op_ShenandoahWeakCompareAndSwapN:
 1540     case Op_ShenandoahCompareAndExchangeP:
 1541     case Op_ShenandoahCompareAndExchangeN:
 1542       return maybe_volatile;
 1543     default:
 1544       return false;
 1545     }
 1546   }
 1547 
 1548   // helper to determine the maximum number of Phi nodes we may need to
 1549   // traverse when searching from a card mark membar for the merge mem
 1550   // feeding a trailing membar or vice versa
 1551 
 1552 // predicates controlling emit of ldr<x>/ldar<x>
 1553 
 1554 bool unnecessary_acquire(const Node *barrier)
 1555 {
 1556   assert(barrier->is_MemBar(), "expecting a membar");
 1557 
 1558   MemBarNode* mb = barrier->as_MemBar();
 1559 
 1560   if (mb->trailing_load()) {
 1561     return true;
 1562   }
 1563 
 1564   if (mb->trailing_load_store()) {
 1565     Node* load_store = mb->in(MemBarNode::Precedent);
 1566     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1567     return is_CAS(load_store->Opcode(), true);
 1568   }
 1569 
 1570   return false;
 1571 }
 1572 
 1573 bool needs_acquiring_load(const Node *n)
 1574 {
 1575   assert(n->is_Load(), "expecting a load");
 1576   LoadNode *ld = n->as_Load();
 1577   return ld->is_acquire();
 1578 }
 1579 
 1580 bool unnecessary_release(const Node *n)
 1581 {
 1582   assert((n->is_MemBar() &&
 1583           n->Opcode() == Op_MemBarRelease),
 1584          "expecting a release membar");
 1585 
 1586   MemBarNode *barrier = n->as_MemBar();
 1587   if (!barrier->leading()) {
 1588     return false;
 1589   } else {
 1590     Node* trailing = barrier->trailing_membar();
 1591     MemBarNode* trailing_mb = trailing->as_MemBar();
 1592     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1593     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1594 
 1595     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1596     if (mem->is_Store()) {
 1597       assert(mem->as_Store()->is_release(), "");
 1598       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1599       return true;
 1600     } else {
 1601       assert(mem->is_LoadStore(), "");
 1602       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1603       return is_CAS(mem->Opcode(), true);
 1604     }
 1605   }
 1606   return false;
 1607 }
 1608 
 1609 bool unnecessary_volatile(const Node *n)
 1610 {
 1611   // assert n->is_MemBar();
 1612   MemBarNode *mbvol = n->as_MemBar();
 1613 
 1614   bool release = mbvol->trailing_store();
 1615   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1616 #ifdef ASSERT
 1617   if (release) {
 1618     Node* leading = mbvol->leading_membar();
 1619     assert(leading->Opcode() == Op_MemBarRelease, "");
 1620     assert(leading->as_MemBar()->leading_store(), "");
 1621     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1622   }
 1623 #endif
 1624 
 1625   return release;
 1626 }
 1627 
 1628 // predicates controlling emit of str<x>/stlr<x>
 1629 
 1630 bool needs_releasing_store(const Node *n)
 1631 {
 1632   // assert n->is_Store();
 1633   StoreNode *st = n->as_Store();
 1634   return st->trailing_membar() != nullptr;
 1635 }
 1636 
 1637 // predicate controlling translation of CAS
 1638 //
 1639 // returns true if CAS needs to use an acquiring load otherwise false
 1640 
 1641 bool needs_acquiring_load_exclusive(const Node *n)
 1642 {
 1643   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1644   LoadStoreNode* ldst = n->as_LoadStore();
 1645   if (is_CAS(n->Opcode(), false)) {
 1646     assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
 1647   } else {
 1648     return ldst->trailing_membar() != nullptr;
 1649   }
 1650 
 1651   // so we can just return true here
 1652   return true;
 1653 }
 1654 
 1655 #define __ masm->
 1656 
 1657 // advance declarations for helper functions to convert register
 1658 // indices to register objects
 1659 
 1660 // the ad file has to provide implementations of certain methods
 1661 // expected by the generic code
 1662 //
 1663 // REQUIRED FUNCTIONALITY
 1664 
 1665 //=============================================================================
 1666 
 1667 // !!!!! Special hack to get all types of calls to specify the byte offset
 1668 //       from the start of the call to the point where the return address
 1669 //       will point.
 1670 
 1671 int MachCallStaticJavaNode::ret_addr_offset()
 1672 {
 1673   // call should be a simple bl
 1674   int off = 4;
 1675   return off;
 1676 }
 1677 
 1678 int MachCallDynamicJavaNode::ret_addr_offset()
 1679 {
 1680   return 16; // movz, movk, movk, bl
 1681 }
 1682 
 1683 int MachCallRuntimeNode::ret_addr_offset() {
 1684   // for generated stubs the call will be
 1685   //   bl(addr)
 1686   // or with far branches
 1687   //   bl(trampoline_stub)
 1688   // for real runtime callouts it will be six instructions
 1689   // see aarch64_enc_java_to_runtime
 1690   //   adr(rscratch2, retaddr)
 1691   //   str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
 1692   //   lea(rscratch1, RuntimeAddress(addr)
 1693   //   blr(rscratch1)
 1694   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1695   if (cb) {
 1696     return 1 * NativeInstruction::instruction_size;
 1697   } else if (_entry_point == nullptr) {
 1698     // See CallLeafNoFPIndirect
 1699     return 1 * NativeInstruction::instruction_size;
 1700   } else {
 1701     return 6 * NativeInstruction::instruction_size;
 1702   }
 1703 }
 1704 
 1705 //=============================================================================
 1706 
 1707 #ifndef PRODUCT
 1708 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1709   st->print("BREAKPOINT");
 1710 }
 1711 #endif
 1712 
 1713 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1714   __ brk(0);
 1715 }
 1716 
 1717 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1718   return MachNode::size(ra_);
 1719 }
 1720 
 1721 //=============================================================================
 1722 
 1723 #ifndef PRODUCT
 1724   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1725     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1726   }
 1727 #endif
 1728 
 1729   void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
 1730     for (int i = 0; i < _count; i++) {
 1731       __ nop();
 1732     }
 1733   }
 1734 
 1735   uint MachNopNode::size(PhaseRegAlloc*) const {
 1736     return _count * NativeInstruction::instruction_size;
 1737   }
 1738 
 1739 //=============================================================================
 1740 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1741 
 1742 int ConstantTable::calculate_table_base_offset() const {
 1743   return 0;  // absolute addressing, no offset
 1744 }
 1745 
 1746 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1747 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1748   ShouldNotReachHere();
 1749 }
 1750 
 1751 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
 1752   // Empty encoding
 1753 }
 1754 
 1755 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1756   return 0;
 1757 }
 1758 
 1759 #ifndef PRODUCT
 1760 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1761   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1762 }
 1763 #endif
 1764 
 1765 #ifndef PRODUCT
 1766 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1767   Compile* C = ra_->C;
 1768 
 1769   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1770 
 1771   if (C->output()->need_stack_bang(framesize))
 1772     st->print("# stack bang size=%d\n\t", framesize);
 1773 
 1774   if (VM_Version::use_rop_protection()) {
 1775     st->print("ldr  zr, [lr]\n\t");
 1776     st->print("paciaz\n\t");
 1777   }
 1778   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1779     st->print("sub  sp, sp, #%d\n\t", framesize);
 1780     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1781     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1782   } else {
 1783     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1784     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1785     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1786     st->print("sub  sp, sp, rscratch1");
 1787   }
 1788   if (C->stub_function() == nullptr) {
 1789     st->print("\n\t");
 1790     st->print("ldr  rscratch1, [guard]\n\t");
 1791     st->print("dmb ishld\n\t");
 1792     st->print("ldr  rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
 1793     st->print("cmp  rscratch1, rscratch2\n\t");
 1794     st->print("b.eq skip");
 1795     st->print("\n\t");
 1796     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1797     st->print("b skip\n\t");
 1798     st->print("guard: int\n\t");
 1799     st->print("\n\t");
 1800     st->print("skip:\n\t");
 1801   }
 1802 }
 1803 #endif
 1804 
 1805 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1806   Compile* C = ra_->C;
 1807 
 1808 
 1809   __ verified_entry(C, 0);
 1810 
 1811   if (C->stub_function() == nullptr) {
 1812     __ entry_barrier();
 1813   }
 1814 
 1815   if (!Compile::current()->output()->in_scratch_emit_size()) {
 1816     __ bind(*_verified_entry);
 1817   }
 1818 
 1819   if (VerifyStackAtCalls) {
 1820     Unimplemented();
 1821   }
 1822 
 1823   C->output()->set_frame_complete(__ offset());
 1824 
 1825   if (C->has_mach_constant_base_node()) {
 1826     // NOTE: We set the table base offset here because users might be
 1827     // emitted before MachConstantBaseNode.
 1828     ConstantTable& constant_table = C->output()->constant_table();
 1829     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1830   }
 1831 }
 1832 
 1833 int MachPrologNode::reloc() const
 1834 {
 1835   return 0;
 1836 }
 1837 
 1838 //=============================================================================
 1839 
 1840 #ifndef PRODUCT
 1841 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1842   Compile* C = ra_->C;
 1843   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1844 
 1845   st->print("# pop frame %d\n\t",framesize);
 1846 
 1847   if (framesize == 0) {
 1848     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1849   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1850     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1851     st->print("add  sp, sp, #%d\n\t", framesize);
 1852   } else {
 1853     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1854     st->print("add  sp, sp, rscratch1\n\t");
 1855     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1856   }
 1857   if (VM_Version::use_rop_protection()) {
 1858     st->print("autiaz\n\t");
 1859     st->print("ldr  zr, [lr]\n\t");
 1860   }
 1861 
 1862   if (do_polling() && C->is_method_compilation()) {
 1863     st->print("# test polling word\n\t");
 1864     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1865     st->print("cmp  sp, rscratch1\n\t");
 1866     st->print("bhi #slow_path");
 1867   }
 1868 }
 1869 #endif
 1870 
 1871 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1872   Compile* C = ra_->C;
 1873   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1874 
 1875   __ remove_frame(framesize, C->needs_stack_repair());
 1876 
 1877   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1878     __ reserved_stack_check();
 1879   }
 1880 
 1881   if (do_polling() && C->is_method_compilation()) {
 1882     Label dummy_label;
 1883     Label* code_stub = &dummy_label;
 1884     if (!C->output()->in_scratch_emit_size()) {
 1885       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
 1886       C->output()->add_stub(stub);
 1887       code_stub = &stub->entry();
 1888     }
 1889     __ relocate(relocInfo::poll_return_type);
 1890     __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
 1891   }
 1892 }
 1893 
 1894 int MachEpilogNode::reloc() const {
 1895   // Return number of relocatable values contained in this instruction.
 1896   return 1; // 1 for polling page.
 1897 }
 1898 
 1899 const Pipeline * MachEpilogNode::pipeline() const {
 1900   return MachNode::pipeline_class();
 1901 }
 1902 
 1903 //=============================================================================
 1904 
 1905 static enum RC rc_class(OptoReg::Name reg) {
 1906 
 1907   if (reg == OptoReg::Bad) {
 1908     return rc_bad;
 1909   }
 1910 
 1911   // we have 32 int registers * 2 halves
 1912   int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
 1913 
 1914   if (reg < slots_of_int_registers) {
 1915     return rc_int;
 1916   }
 1917 
 1918   // we have 32 float register * 8 halves
 1919   int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
 1920   if (reg < slots_of_int_registers + slots_of_float_registers) {
 1921     return rc_float;
 1922   }
 1923 
 1924   int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
 1925   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 1926     return rc_predicate;
 1927   }
 1928 
 1929   // Between predicate regs & stack is the flags.
 1930   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 1931 
 1932   return rc_stack;
 1933 }
 1934 
 1935 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 1936   Compile* C = ra_->C;
 1937 
 1938   // Get registers to move.
 1939   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 1940   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 1941   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 1942   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 1943 
 1944   enum RC src_hi_rc = rc_class(src_hi);
 1945   enum RC src_lo_rc = rc_class(src_lo);
 1946   enum RC dst_hi_rc = rc_class(dst_hi);
 1947   enum RC dst_lo_rc = rc_class(dst_lo);
 1948 
 1949   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 1950 
 1951   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 1952     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 1953            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 1954            "expected aligned-adjacent pairs");
 1955   }
 1956 
 1957   if (src_lo == dst_lo && src_hi == dst_hi) {
 1958     return 0;            // Self copy, no move.
 1959   }
 1960 
 1961   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 1962               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 1963   int src_offset = ra_->reg2offset(src_lo);
 1964   int dst_offset = ra_->reg2offset(dst_lo);
 1965 
 1966   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 1967     uint ireg = ideal_reg();
 1968     if (ireg == Op_VecA && masm) {
 1969       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 1970       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1971         // stack->stack
 1972         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 1973                                                 sve_vector_reg_size_in_bytes);
 1974       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 1975         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 1976                             sve_vector_reg_size_in_bytes);
 1977       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 1978         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 1979                               sve_vector_reg_size_in_bytes);
 1980       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 1981         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1982                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 1983                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 1984       } else {
 1985         ShouldNotReachHere();
 1986       }
 1987     } else if (masm) {
 1988       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 1989       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 1990       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1991         // stack->stack
 1992         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 1993         if (ireg == Op_VecD) {
 1994           __ unspill(rscratch1, true, src_offset);
 1995           __ spill(rscratch1, true, dst_offset);
 1996         } else {
 1997           __ spill_copy128(src_offset, dst_offset);
 1998         }
 1999       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2000         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2001                ireg == Op_VecD ? __ T8B : __ T16B,
 2002                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2003       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2004         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2005                  ireg == Op_VecD ? __ D : __ Q,
 2006                  ra_->reg2offset(dst_lo));
 2007       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2008         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2009                    ireg == Op_VecD ? __ D : __ Q,
 2010                    ra_->reg2offset(src_lo));
 2011       } else {
 2012         ShouldNotReachHere();
 2013       }
 2014     }
 2015   } else if (masm) {
 2016     switch (src_lo_rc) {
 2017     case rc_int:
 2018       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2019         if (is64) {
 2020             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2021                    as_Register(Matcher::_regEncode[src_lo]));
 2022         } else {
 2023             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2024                     as_Register(Matcher::_regEncode[src_lo]));
 2025         }
 2026       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2027         if (is64) {
 2028             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2029                      as_Register(Matcher::_regEncode[src_lo]));
 2030         } else {
 2031             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2032                      as_Register(Matcher::_regEncode[src_lo]));
 2033         }
 2034       } else {                    // gpr --> stack spill
 2035         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2036         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2037       }
 2038       break;
 2039     case rc_float:
 2040       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2041         if (is64) {
 2042             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2043                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2044         } else {
 2045             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2046                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2047         }
 2048       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2049         if (is64) {
 2050             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2051                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2052         } else {
 2053             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2054                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2055         }
 2056       } else {                    // fpr --> stack spill
 2057         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2058         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2059                  is64 ? __ D : __ S, dst_offset);
 2060       }
 2061       break;
 2062     case rc_stack:
 2063       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2064         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2065       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2066         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2067                    is64 ? __ D : __ S, src_offset);
 2068       } else if (dst_lo_rc == rc_predicate) {
 2069         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2070                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2071       } else {                    // stack --> stack copy
 2072         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2073         if (ideal_reg() == Op_RegVectMask) {
 2074           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2075                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2076         } else {
 2077           __ unspill(rscratch1, is64, src_offset);
 2078           __ spill(rscratch1, is64, dst_offset);
 2079         }
 2080       }
 2081       break;
 2082     case rc_predicate:
 2083       if (dst_lo_rc == rc_predicate) {
 2084         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2085       } else if (dst_lo_rc == rc_stack) {
 2086         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2087                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2088       } else {
 2089         assert(false, "bad src and dst rc_class combination.");
 2090         ShouldNotReachHere();
 2091       }
 2092       break;
 2093     default:
 2094       assert(false, "bad rc_class for spill");
 2095       ShouldNotReachHere();
 2096     }
 2097   }
 2098 
 2099   if (st) {
 2100     st->print("spill ");
 2101     if (src_lo_rc == rc_stack) {
 2102       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2103     } else {
 2104       st->print("%s -> ", Matcher::regName[src_lo]);
 2105     }
 2106     if (dst_lo_rc == rc_stack) {
 2107       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2108     } else {
 2109       st->print("%s", Matcher::regName[dst_lo]);
 2110     }
 2111     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2112       int vsize = 0;
 2113       switch (ideal_reg()) {
 2114       case Op_VecD:
 2115         vsize = 64;
 2116         break;
 2117       case Op_VecX:
 2118         vsize = 128;
 2119         break;
 2120       case Op_VecA:
 2121         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2122         break;
 2123       default:
 2124         assert(false, "bad register type for spill");
 2125         ShouldNotReachHere();
 2126       }
 2127       st->print("\t# vector spill size = %d", vsize);
 2128     } else if (ideal_reg() == Op_RegVectMask) {
 2129       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2130       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2131       st->print("\t# predicate spill size = %d", vsize);
 2132     } else {
 2133       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2134     }
 2135   }
 2136 
 2137   return 0;
 2138 
 2139 }
 2140 
 2141 #ifndef PRODUCT
 2142 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2143   if (!ra_)
 2144     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2145   else
 2146     implementation(nullptr, ra_, false, st);
 2147 }
 2148 #endif
 2149 
 2150 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 2151   implementation(masm, ra_, false, nullptr);
 2152 }
 2153 
 2154 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2155   return MachNode::size(ra_);
 2156 }
 2157 
 2158 //=============================================================================
 2159 
 2160 #ifndef PRODUCT
 2161 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2162   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2163   int reg = ra_->get_reg_first(this);
 2164   st->print("add %s, rsp, #%d]\t# box lock",
 2165             Matcher::regName[reg], offset);
 2166 }
 2167 #endif
 2168 
 2169 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 2170   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2171   int reg    = ra_->get_encode(this);
 2172 
 2173   // This add will handle any 24-bit signed offset. 24 bits allows an
 2174   // 8 megabyte stack frame.
 2175   __ add(as_Register(reg), sp, offset);
 2176 }
 2177 
 2178 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2179   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2180   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2181 
 2182   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2183     return NativeInstruction::instruction_size;
 2184   } else {
 2185     return 2 * NativeInstruction::instruction_size;
 2186   }
 2187 }
 2188 
 2189 ///=============================================================================
 2190 #ifndef PRODUCT
 2191 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2192 {
 2193   st->print_cr("# MachVEPNode");
 2194   if (!_verified) {
 2195     st->print_cr("\t load_class");
 2196   } else {
 2197     st->print_cr("\t unpack_inline_arg");
 2198   }
 2199 }
 2200 #endif
 2201 
 2202 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
 2203 {
 2204   if (!_verified) {
 2205     __ ic_check(1);
 2206   } else {
 2207     // TODO 8284443 Avoid creation of temporary frame
 2208     if (ra_->C->stub_function() == nullptr) {
 2209       __ verified_entry(ra_->C, 0);
 2210       __ entry_barrier();
 2211       int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
 2212       __ remove_frame(framesize, false);
 2213     }
 2214     // Unpack inline type args passed as oop and then jump to
 2215     // the verified entry point (skipping the unverified entry).
 2216     int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
 2217     // Emit code for verified entry and save increment for stack repair on return
 2218     __ verified_entry(ra_->C, sp_inc);
 2219     if (Compile::current()->output()->in_scratch_emit_size()) {
 2220       Label dummy_verified_entry;
 2221       __ b(dummy_verified_entry);
 2222     } else {
 2223       __ b(*_verified_entry);
 2224     }
 2225   }
 2226 }
 2227 
 2228 //=============================================================================
 2229 #ifndef PRODUCT
 2230 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2231 {
 2232   st->print_cr("# MachUEPNode");
 2233   if (UseCompressedClassPointers) {
 2234     st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2235     st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
 2236     st->print_cr("\tcmpw rscratch1, r10");
 2237   } else {
 2238     st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2239     st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
 2240     st->print_cr("\tcmp rscratch1, r10");
 2241   }
 2242   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2243 }
 2244 #endif
 2245 
 2246 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 2247 {
 2248   __ ic_check(InteriorEntryAlignment);
 2249 }
 2250 
 2251 // REQUIRED EMIT CODE
 2252 
 2253 //=============================================================================
 2254 
 2255 // Emit exception handler code.
 2256 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
 2257 {
 2258   // mov rscratch1 #exception_blob_entry_point
 2259   // br rscratch1
 2260   // Note that the code buffer's insts_mark is always relative to insts.
 2261   // That's why we must use the macroassembler to generate a handler.
 2262   address base = __ start_a_stub(size_exception_handler());
 2263   if (base == nullptr) {
 2264     ciEnv::current()->record_failure("CodeCache is full");
 2265     return 0;  // CodeBuffer::expand failed
 2266   }
 2267   int offset = __ offset();
 2268   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2269   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2270   __ end_a_stub();
 2271   return offset;
 2272 }
 2273 
 2274 // Emit deopt handler code.
 2275 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
 2276 {
 2277   // Note that the code buffer's insts_mark is always relative to insts.
 2278   // That's why we must use the macroassembler to generate a handler.
 2279   address base = __ start_a_stub(size_deopt_handler());
 2280   if (base == nullptr) {
 2281     ciEnv::current()->record_failure("CodeCache is full");
 2282     return 0;  // CodeBuffer::expand failed
 2283   }
 2284   int offset = __ offset();
 2285 
 2286   __ adr(lr, __ pc());
 2287   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2288 
 2289   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2290   __ end_a_stub();
 2291   return offset;
 2292 }
 2293 
 2294 // REQUIRED MATCHER CODE
 2295 
 2296 //=============================================================================
 2297 
 2298 bool Matcher::match_rule_supported(int opcode) {
 2299   if (!has_match_rule(opcode))
 2300     return false;
 2301 
 2302   switch (opcode) {
 2303     case Op_OnSpinWait:
 2304       return VM_Version::supports_on_spin_wait();
 2305     case Op_CacheWB:
 2306     case Op_CacheWBPreSync:
 2307     case Op_CacheWBPostSync:
 2308       if (!VM_Version::supports_data_cache_line_flush()) {
 2309         return false;
 2310       }
 2311       break;
 2312     case Op_ExpandBits:
 2313     case Op_CompressBits:
 2314       if (!VM_Version::supports_svebitperm()) {
 2315         return false;
 2316       }
 2317       break;
 2318     case Op_FmaF:
 2319     case Op_FmaD:
 2320     case Op_FmaVF:
 2321     case Op_FmaVD:
 2322       if (!UseFMA) {
 2323         return false;
 2324       }
 2325       break;
 2326     case Op_FmaHF:
 2327       // UseFMA flag also needs to be checked along with FEAT_FP16
 2328       if (!UseFMA || !is_feat_fp16_supported()) {
 2329         return false;
 2330       }
 2331       break;
 2332     case Op_AddHF:
 2333     case Op_SubHF:
 2334     case Op_MulHF:
 2335     case Op_DivHF:
 2336     case Op_MinHF:
 2337     case Op_MaxHF:
 2338     case Op_SqrtHF:
 2339       // Half-precision floating point scalar operations require FEAT_FP16
 2340       // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
 2341       // features are supported.
 2342       if (!is_feat_fp16_supported()) {
 2343         return false;
 2344       }
 2345       break;
 2346   }
 2347 
 2348   return true; // Per default match rules are supported.
 2349 }
 2350 
 2351 const RegMask* Matcher::predicate_reg_mask(void) {
 2352   return &_PR_REG_mask;
 2353 }
 2354 
 2355 bool Matcher::supports_vector_calling_convention(void) {
 2356   return EnableVectorSupport;
 2357 }
 2358 
 2359 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2360   assert(EnableVectorSupport, "sanity");
 2361   int lo = V0_num;
 2362   int hi = V0_H_num;
 2363   if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
 2364     hi = V0_K_num;
 2365   }
 2366   return OptoRegPair(hi, lo);
 2367 }
 2368 
 2369 // Is this branch offset short enough that a short branch can be used?
 2370 //
 2371 // NOTE: If the platform does not provide any short branch variants, then
 2372 //       this method should return false for offset 0.
 2373 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2374   // The passed offset is relative to address of the branch.
 2375 
 2376   return (-32768 <= offset && offset < 32768);
 2377 }
 2378 
 2379 // Vector width in bytes.
 2380 int Matcher::vector_width_in_bytes(BasicType bt) {
 2381   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2382   int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
 2383   // Minimum 2 values in vector
 2384   if (size < 2*type2aelembytes(bt)) size = 0;
 2385   // But never < 4
 2386   if (size < 4) size = 0;
 2387   return size;
 2388 }
 2389 
 2390 // Limits on vector size (number of elements) loaded into vector.
 2391 int Matcher::max_vector_size(const BasicType bt) {
 2392   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2393 }
 2394 
 2395 int Matcher::min_vector_size(const BasicType bt) {
 2396   // Usually, the shortest vector length supported by AArch64 ISA and
 2397   // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
 2398   // vectors in a few special cases.
 2399   int size;
 2400   switch(bt) {
 2401     case T_BOOLEAN:
 2402       // Load/store a vector mask with only 2 elements for vector types
 2403       // such as "2I/2F/2L/2D".
 2404       size = 2;
 2405       break;
 2406     case T_BYTE:
 2407       // Generate a "4B" vector, to support vector cast between "8B/16B"
 2408       // and "4S/4I/4L/4F/4D".
 2409       size = 4;
 2410       break;
 2411     case T_SHORT:
 2412       // Generate a "2S" vector, to support vector cast between "4S/8S"
 2413       // and "2I/2L/2F/2D".
 2414       size = 2;
 2415       break;
 2416     default:
 2417       // Limit the min vector length to 64-bit.
 2418       size = 8 / type2aelembytes(bt);
 2419       // The number of elements in a vector should be at least 2.
 2420       size = MAX2(size, 2);
 2421   }
 2422 
 2423   int max_size = max_vector_size(bt);
 2424   return MIN2(size, max_size);
 2425 }
 2426 
 2427 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
 2428   return Matcher::max_vector_size(bt);
 2429 }
 2430 
 2431 // Actual max scalable vector register length.
 2432 int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2433   return Matcher::max_vector_size(bt);
 2434 }
 2435 
 2436 // Vector ideal reg.
 2437 uint Matcher::vector_ideal_reg(int len) {
 2438   if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
 2439     return Op_VecA;
 2440   }
 2441   switch(len) {
 2442     // For 16-bit/32-bit mask vector, reuse VecD.
 2443     case  2:
 2444     case  4:
 2445     case  8: return Op_VecD;
 2446     case 16: return Op_VecX;
 2447   }
 2448   ShouldNotReachHere();
 2449   return 0;
 2450 }
 2451 
 2452 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
 2453   assert(Matcher::is_generic_vector(generic_opnd), "not generic");
 2454   switch (ideal_reg) {
 2455     case Op_VecA: return new vecAOper();
 2456     case Op_VecD: return new vecDOper();
 2457     case Op_VecX: return new vecXOper();
 2458   }
 2459   ShouldNotReachHere();
 2460   return nullptr;
 2461 }
 2462 
 2463 bool Matcher::is_reg2reg_move(MachNode* m) {
 2464   return false;
 2465 }
 2466 
 2467 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2468   return opnd->opcode() == VREG;
 2469 }
 2470 
 2471 // Return whether or not this register is ever used as an argument.
 2472 // This function is used on startup to build the trampoline stubs in
 2473 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2474 // call in the trampoline, and arguments in those registers not be
 2475 // available to the callee.
 2476 bool Matcher::can_be_java_arg(int reg)
 2477 {
 2478   return
 2479     reg ==  R0_num || reg == R0_H_num ||
 2480     reg ==  R1_num || reg == R1_H_num ||
 2481     reg ==  R2_num || reg == R2_H_num ||
 2482     reg ==  R3_num || reg == R3_H_num ||
 2483     reg ==  R4_num || reg == R4_H_num ||
 2484     reg ==  R5_num || reg == R5_H_num ||
 2485     reg ==  R6_num || reg == R6_H_num ||
 2486     reg ==  R7_num || reg == R7_H_num ||
 2487     reg ==  V0_num || reg == V0_H_num ||
 2488     reg ==  V1_num || reg == V1_H_num ||
 2489     reg ==  V2_num || reg == V2_H_num ||
 2490     reg ==  V3_num || reg == V3_H_num ||
 2491     reg ==  V4_num || reg == V4_H_num ||
 2492     reg ==  V5_num || reg == V5_H_num ||
 2493     reg ==  V6_num || reg == V6_H_num ||
 2494     reg ==  V7_num || reg == V7_H_num;
 2495 }
 2496 
 2497 bool Matcher::is_spillable_arg(int reg)
 2498 {
 2499   return can_be_java_arg(reg);
 2500 }
 2501 
 2502 uint Matcher::int_pressure_limit()
 2503 {
 2504   // JDK-8183543: When taking the number of available registers as int
 2505   // register pressure threshold, the jtreg test:
 2506   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2507   // failed due to C2 compilation failure with
 2508   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2509   //
 2510   // A derived pointer is live at CallNode and then is flagged by RA
 2511   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2512   // derived pointers and lastly fail to spill after reaching maximum
 2513   // number of iterations. Lowering the default pressure threshold to
 2514   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2515   // a high register pressure area of the code so that split_DEF can
 2516   // generate DefinitionSpillCopy for the derived pointer.
 2517   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2518   if (!PreserveFramePointer) {
 2519     // When PreserveFramePointer is off, frame pointer is allocatable,
 2520     // but different from other SOC registers, it is excluded from
 2521     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2522     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2523     // See check_pressure_at_fatproj().
 2524     default_int_pressure_threshold--;
 2525   }
 2526   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2527 }
 2528 
 2529 uint Matcher::float_pressure_limit()
 2530 {
 2531   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2532   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2533 }
 2534 
 2535 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2536   return false;
 2537 }
 2538 
 2539 RegMask Matcher::divI_proj_mask() {
 2540   ShouldNotReachHere();
 2541   return RegMask();
 2542 }
 2543 
 2544 // Register for MODI projection of divmodI.
 2545 RegMask Matcher::modI_proj_mask() {
 2546   ShouldNotReachHere();
 2547   return RegMask();
 2548 }
 2549 
 2550 // Register for DIVL projection of divmodL.
 2551 RegMask Matcher::divL_proj_mask() {
 2552   ShouldNotReachHere();
 2553   return RegMask();
 2554 }
 2555 
 2556 // Register for MODL projection of divmodL.
 2557 RegMask Matcher::modL_proj_mask() {
 2558   ShouldNotReachHere();
 2559   return RegMask();
 2560 }
 2561 
 2562 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2563   return FP_REG_mask();
 2564 }
 2565 
 2566 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2567   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2568     Node* u = addp->fast_out(i);
 2569     if (u->is_LoadStore()) {
 2570       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2571       // instructions) only take register indirect as an operand, so
 2572       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2573       // must fail.
 2574       return false;
 2575     }
 2576     if (u->is_Mem()) {
 2577       int opsize = u->as_Mem()->memory_size();
 2578       assert(opsize > 0, "unexpected memory operand size");
 2579       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2580         return false;
 2581       }
 2582     }
 2583   }
 2584   return true;
 2585 }
 2586 
 2587 // Convert BootTest condition to Assembler condition.
 2588 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 2589 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
 2590   Assembler::Condition result;
 2591   switch(cond) {
 2592     case BoolTest::eq:
 2593       result = Assembler::EQ; break;
 2594     case BoolTest::ne:
 2595       result = Assembler::NE; break;
 2596     case BoolTest::le:
 2597       result = Assembler::LE; break;
 2598     case BoolTest::ge:
 2599       result = Assembler::GE; break;
 2600     case BoolTest::lt:
 2601       result = Assembler::LT; break;
 2602     case BoolTest::gt:
 2603       result = Assembler::GT; break;
 2604     case BoolTest::ule:
 2605       result = Assembler::LS; break;
 2606     case BoolTest::uge:
 2607       result = Assembler::HS; break;
 2608     case BoolTest::ult:
 2609       result = Assembler::LO; break;
 2610     case BoolTest::ugt:
 2611       result = Assembler::HI; break;
 2612     case BoolTest::overflow:
 2613       result = Assembler::VS; break;
 2614     case BoolTest::no_overflow:
 2615       result = Assembler::VC; break;
 2616     default:
 2617       ShouldNotReachHere();
 2618       return Assembler::Condition(-1);
 2619   }
 2620 
 2621   // Check conversion
 2622   if (cond & BoolTest::unsigned_compare) {
 2623     assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
 2624   } else {
 2625     assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
 2626   }
 2627 
 2628   return result;
 2629 }
 2630 
 2631 // Binary src (Replicate con)
 2632 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
 2633   if (n == nullptr || m == nullptr) {
 2634     return false;
 2635   }
 2636 
 2637   if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
 2638     return false;
 2639   }
 2640 
 2641   Node* imm_node = m->in(1);
 2642   if (!imm_node->is_Con()) {
 2643     return false;
 2644   }
 2645 
 2646   const Type* t = imm_node->bottom_type();
 2647   if (!(t->isa_int() || t->isa_long())) {
 2648     return false;
 2649   }
 2650 
 2651   switch (n->Opcode()) {
 2652   case Op_AndV:
 2653   case Op_OrV:
 2654   case Op_XorV: {
 2655     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
 2656     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2657     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2658   }
 2659   case Op_AddVB:
 2660     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2661   case Op_AddVS:
 2662   case Op_AddVI:
 2663     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2664   case Op_AddVL:
 2665     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2666   default:
 2667     return false;
 2668   }
 2669 }
 2670 
 2671 // (XorV src (Replicate m1))
 2672 // (XorVMask src (MaskAll m1))
 2673 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
 2674   if (n != nullptr && m != nullptr) {
 2675     return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
 2676            VectorNode::is_all_ones_vector(m);
 2677   }
 2678   return false;
 2679 }
 2680 
 2681 // Should the matcher clone input 'm' of node 'n'?
 2682 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2683   if (is_vshift_con_pattern(n, m) ||
 2684       is_vector_bitwise_not_pattern(n, m) ||
 2685       is_valid_sve_arith_imm_pattern(n, m) ||
 2686       is_encode_and_store_pattern(n, m)) {
 2687     mstack.push(m, Visit);
 2688     return true;
 2689   }
 2690   return false;
 2691 }
 2692 
 2693 // Should the Matcher clone shifts on addressing modes, expecting them
 2694 // to be subsumed into complex addressing expressions or compute them
 2695 // into registers?
 2696 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2697 
 2698   // Loads and stores with indirect memory input (e.g., volatile loads and
 2699   // stores) do not subsume the input into complex addressing expressions. If
 2700   // the addressing expression is input to at least one such load or store, do
 2701   // not clone the addressing expression. Query needs_acquiring_load and
 2702   // needs_releasing_store as a proxy for indirect memory input, as it is not
 2703   // possible to directly query for indirect memory input at this stage.
 2704   for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
 2705     Node* n = m->fast_out(i);
 2706     if (n->is_Load() && needs_acquiring_load(n)) {
 2707       return false;
 2708     }
 2709     if (n->is_Store() && needs_releasing_store(n)) {
 2710       return false;
 2711     }
 2712   }
 2713 
 2714   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2715     return true;
 2716   }
 2717 
 2718   Node *off = m->in(AddPNode::Offset);
 2719   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2720       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2721       // Are there other uses besides address expressions?
 2722       !is_visited(off)) {
 2723     address_visited.set(off->_idx); // Flag as address_visited
 2724     mstack.push(off->in(2), Visit);
 2725     Node *conv = off->in(1);
 2726     if (conv->Opcode() == Op_ConvI2L &&
 2727         // Are there other uses besides address expressions?
 2728         !is_visited(conv)) {
 2729       address_visited.set(conv->_idx); // Flag as address_visited
 2730       mstack.push(conv->in(1), Pre_Visit);
 2731     } else {
 2732       mstack.push(conv, Pre_Visit);
 2733     }
 2734     address_visited.test_set(m->_idx); // Flag as address_visited
 2735     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2736     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2737     return true;
 2738   } else if (off->Opcode() == Op_ConvI2L &&
 2739              // Are there other uses besides address expressions?
 2740              !is_visited(off)) {
 2741     address_visited.test_set(m->_idx); // Flag as address_visited
 2742     address_visited.set(off->_idx); // Flag as address_visited
 2743     mstack.push(off->in(1), Pre_Visit);
 2744     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2745     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2746     return true;
 2747   }
 2748   return false;
 2749 }
 2750 
 2751 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2752   {                                                                     \
 2753     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2754     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2755     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2756     __ INSN(REG, as_Register(BASE));                                    \
 2757   }
 2758 
 2759 
 2760 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2761   {
 2762     Address::extend scale;
 2763 
 2764     // Hooboy, this is fugly.  We need a way to communicate to the
 2765     // encoder that the index needs to be sign extended, so we have to
 2766     // enumerate all the cases.
 2767     switch (opcode) {
 2768     case INDINDEXSCALEDI2L:
 2769     case INDINDEXSCALEDI2LN:
 2770     case INDINDEXI2L:
 2771     case INDINDEXI2LN:
 2772       scale = Address::sxtw(size);
 2773       break;
 2774     default:
 2775       scale = Address::lsl(size);
 2776     }
 2777 
 2778     if (index == -1) {
 2779       return Address(base, disp);
 2780     } else {
 2781       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2782       return Address(base, as_Register(index), scale);
 2783     }
 2784   }
 2785 
 2786 
 2787 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2788 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2789 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2790 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2791                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2792 
 2793   // Used for all non-volatile memory accesses.  The use of
 2794   // $mem->opcode() to discover whether this pattern uses sign-extended
 2795   // offsets is something of a kludge.
 2796   static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
 2797                         Register reg, int opcode,
 2798                         Register base, int index, int scale, int disp,
 2799                         int size_in_memory)
 2800   {
 2801     Address addr = mem2address(opcode, base, index, scale, disp);
 2802     if (addr.getMode() == Address::base_plus_offset) {
 2803       /* Fix up any out-of-range offsets. */
 2804       assert_different_registers(rscratch1, base);
 2805       assert_different_registers(rscratch1, reg);
 2806       addr = __ legitimize_address(addr, size_in_memory, rscratch1);
 2807     }
 2808     (masm->*insn)(reg, addr);
 2809   }
 2810 
 2811   static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
 2812                         FloatRegister reg, int opcode,
 2813                         Register base, int index, int size, int disp,
 2814                         int size_in_memory)
 2815   {
 2816     Address::extend scale;
 2817 
 2818     switch (opcode) {
 2819     case INDINDEXSCALEDI2L:
 2820     case INDINDEXSCALEDI2LN:
 2821       scale = Address::sxtw(size);
 2822       break;
 2823     default:
 2824       scale = Address::lsl(size);
 2825     }
 2826 
 2827     if (index == -1) {
 2828       // Fix up any out-of-range offsets.
 2829       assert_different_registers(rscratch1, base);
 2830       Address addr = Address(base, disp);
 2831       addr = __ legitimize_address(addr, size_in_memory, rscratch1);
 2832       (masm->*insn)(reg, addr);
 2833     } else {
 2834       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2835       (masm->*insn)(reg, Address(base, as_Register(index), scale));
 2836     }
 2837   }
 2838 
 2839   static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
 2840                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2841                         int opcode, Register base, int index, int size, int disp)
 2842   {
 2843     if (index == -1) {
 2844       (masm->*insn)(reg, T, Address(base, disp));
 2845     } else {
 2846       assert(disp == 0, "unsupported address mode");
 2847       (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2848     }
 2849   }
 2850 
 2851 %}
 2852 
 2853 
 2854 
 2855 //----------ENCODING BLOCK-----------------------------------------------------
 2856 // This block specifies the encoding classes used by the compiler to
 2857 // output byte streams.  Encoding classes are parameterized macros
 2858 // used by Machine Instruction Nodes in order to generate the bit
 2859 // encoding of the instruction.  Operands specify their base encoding
 2860 // interface with the interface keyword.  There are currently
 2861 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2862 // COND_INTER.  REG_INTER causes an operand to generate a function
 2863 // which returns its register number when queried.  CONST_INTER causes
 2864 // an operand to generate a function which returns the value of the
 2865 // constant when queried.  MEMORY_INTER causes an operand to generate
 2866 // four functions which return the Base Register, the Index Register,
 2867 // the Scale Value, and the Offset Value of the operand when queried.
 2868 // COND_INTER causes an operand to generate six functions which return
 2869 // the encoding code (ie - encoding bits for the instruction)
 2870 // associated with each basic boolean condition for a conditional
 2871 // instruction.
 2872 //
 2873 // Instructions specify two basic values for encoding.  Again, a
 2874 // function is available to check if the constant displacement is an
 2875 // oop. They use the ins_encode keyword to specify their encoding
 2876 // classes (which must be a sequence of enc_class names, and their
 2877 // parameters, specified in the encoding block), and they use the
 2878 // opcode keyword to specify, in order, their primary, secondary, and
 2879 // tertiary opcode.  Only the opcode sections which a particular
 2880 // instruction needs for encoding need to be specified.
 2881 encode %{
 2882   // Build emit functions for each basic byte or larger field in the
 2883   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2884   // from C++ code in the enc_class source block.  Emit functions will
 2885   // live in the main source block for now.  In future, we can
 2886   // generalize this by adding a syntax that specifies the sizes of
 2887   // fields in an order, so that the adlc can build the emit functions
 2888   // automagically
 2889 
 2890   // catch all for unimplemented encodings
 2891   enc_class enc_unimplemented %{
 2892     __ unimplemented("C2 catch all");
 2893   %}
 2894 
 2895   // BEGIN Non-volatile memory access
 2896 
 2897   // This encoding class is generated automatically from ad_encode.m4.
 2898   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2899   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2900     Register dst_reg = as_Register($dst$$reg);
 2901     loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2902                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2903   %}
 2904 
 2905   // This encoding class is generated automatically from ad_encode.m4.
 2906   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2907   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2908     Register dst_reg = as_Register($dst$$reg);
 2909     loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2910                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2911   %}
 2912 
 2913   // This encoding class is generated automatically from ad_encode.m4.
 2914   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2915   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2916     Register dst_reg = as_Register($dst$$reg);
 2917     loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2918                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2919   %}
 2920 
 2921   // This encoding class is generated automatically from ad_encode.m4.
 2922   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2923   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2924     Register dst_reg = as_Register($dst$$reg);
 2925     loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2926                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2927   %}
 2928 
 2929   // This encoding class is generated automatically from ad_encode.m4.
 2930   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2931   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2932     Register dst_reg = as_Register($dst$$reg);
 2933     loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2934                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2935   %}
 2936 
 2937   // This encoding class is generated automatically from ad_encode.m4.
 2938   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2939   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2940     Register dst_reg = as_Register($dst$$reg);
 2941     loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2942                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2943   %}
 2944 
 2945   // This encoding class is generated automatically from ad_encode.m4.
 2946   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2947   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2948     Register dst_reg = as_Register($dst$$reg);
 2949     loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2950                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2951   %}
 2952 
 2953   // This encoding class is generated automatically from ad_encode.m4.
 2954   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2955   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2956     Register dst_reg = as_Register($dst$$reg);
 2957     loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2958                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2959   %}
 2960 
 2961   // This encoding class is generated automatically from ad_encode.m4.
 2962   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2963   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2964     Register dst_reg = as_Register($dst$$reg);
 2965     loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2966                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2967   %}
 2968 
 2969   // This encoding class is generated automatically from ad_encode.m4.
 2970   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2971   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2972     Register dst_reg = as_Register($dst$$reg);
 2973     loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2974                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2975   %}
 2976 
 2977   // This encoding class is generated automatically from ad_encode.m4.
 2978   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2979   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2980     Register dst_reg = as_Register($dst$$reg);
 2981     loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2982                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2983   %}
 2984 
 2985   // This encoding class is generated automatically from ad_encode.m4.
 2986   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2987   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2988     Register dst_reg = as_Register($dst$$reg);
 2989     loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2990                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2991   %}
 2992 
 2993   // This encoding class is generated automatically from ad_encode.m4.
 2994   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2995   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2996     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2997     loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2998                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2999   %}
 3000 
 3001   // This encoding class is generated automatically from ad_encode.m4.
 3002   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3003   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 3004     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3005     loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 3006                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3007   %}
 3008 
 3009   // This encoding class is generated automatically from ad_encode.m4.
 3010   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3011   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 3012     Register src_reg = as_Register($src$$reg);
 3013     loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
 3014                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3015   %}
 3016 
 3017   // This encoding class is generated automatically from ad_encode.m4.
 3018   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3019   enc_class aarch64_enc_strb0(memory1 mem) %{
 3020     loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3021                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3022   %}
 3023 
 3024   // This encoding class is generated automatically from ad_encode.m4.
 3025   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3026   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 3027     Register src_reg = as_Register($src$$reg);
 3028     loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
 3029                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3030   %}
 3031 
 3032   // This encoding class is generated automatically from ad_encode.m4.
 3033   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3034   enc_class aarch64_enc_strh0(memory2 mem) %{
 3035     loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
 3036                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3037   %}
 3038 
 3039   // This encoding class is generated automatically from ad_encode.m4.
 3040   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3041   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 3042     Register src_reg = as_Register($src$$reg);
 3043     loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
 3044                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3045   %}
 3046 
 3047   // This encoding class is generated automatically from ad_encode.m4.
 3048   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3049   enc_class aarch64_enc_strw0(memory4 mem) %{
 3050     loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
 3051                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3052   %}
 3053 
 3054   // This encoding class is generated automatically from ad_encode.m4.
 3055   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3056   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 3057     Register src_reg = as_Register($src$$reg);
 3058     // we sometimes get asked to store the stack pointer into the
 3059     // current thread -- we cannot do that directly on AArch64
 3060     if (src_reg == r31_sp) {
 3061       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3062       __ mov(rscratch2, sp);
 3063       src_reg = rscratch2;
 3064     }
 3065     loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
 3066                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3067   %}
 3068 
 3069   // This encoding class is generated automatically from ad_encode.m4.
 3070   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3071   enc_class aarch64_enc_str0(memory8 mem) %{
 3072     loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
 3073                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3074   %}
 3075 
 3076   // This encoding class is generated automatically from ad_encode.m4.
 3077   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3078   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3079     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3080     loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
 3081                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3082   %}
 3083 
 3084   // This encoding class is generated automatically from ad_encode.m4.
 3085   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3086   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3087     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3088     loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
 3089                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3090   %}
 3091 
 3092   // This encoding class is generated automatically from ad_encode.m4.
 3093   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3094   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3095       __ membar(Assembler::StoreStore);
 3096       loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3097                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3098   %}
 3099 
 3100   // END Non-volatile memory access
 3101 
 3102   // Vector loads and stores
 3103   enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
 3104     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3105     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3106        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3107   %}
 3108 
 3109   enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
 3110     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3111     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3112        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3113   %}
 3114 
 3115   enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
 3116     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3117     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3118        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3119   %}
 3120 
 3121   enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
 3122     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3123     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3124        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3125   %}
 3126 
 3127   enc_class aarch64_enc_strvH(vReg src, memory mem) %{
 3128     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3129     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
 3130        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3131   %}
 3132 
 3133   enc_class aarch64_enc_strvS(vReg src, memory mem) %{
 3134     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3135     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
 3136        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3137   %}
 3138 
 3139   enc_class aarch64_enc_strvD(vReg src, memory mem) %{
 3140     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3141     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
 3142        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3143   %}
 3144 
 3145   enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
 3146     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3147     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3148        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3149   %}
 3150 
 3151   // volatile loads and stores
 3152 
 3153   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3154     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3155                  rscratch1, stlrb);
 3156   %}
 3157 
 3158   enc_class aarch64_enc_stlrb0(memory mem) %{
 3159     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3160                  rscratch1, stlrb);
 3161   %}
 3162 
 3163   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3164     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3165                  rscratch1, stlrh);
 3166   %}
 3167 
 3168   enc_class aarch64_enc_stlrh0(memory mem) %{
 3169     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3170                  rscratch1, stlrh);
 3171   %}
 3172 
 3173   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3174     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3175                  rscratch1, stlrw);
 3176   %}
 3177 
 3178   enc_class aarch64_enc_stlrw0(memory mem) %{
 3179     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3180                  rscratch1, stlrw);
 3181   %}
 3182 
 3183   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3184     Register dst_reg = as_Register($dst$$reg);
 3185     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3186              rscratch1, ldarb);
 3187     __ sxtbw(dst_reg, dst_reg);
 3188   %}
 3189 
 3190   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3191     Register dst_reg = as_Register($dst$$reg);
 3192     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3193              rscratch1, ldarb);
 3194     __ sxtb(dst_reg, dst_reg);
 3195   %}
 3196 
 3197   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3198     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3199              rscratch1, ldarb);
 3200   %}
 3201 
 3202   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3203     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3204              rscratch1, ldarb);
 3205   %}
 3206 
 3207   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3208     Register dst_reg = as_Register($dst$$reg);
 3209     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3210              rscratch1, ldarh);
 3211     __ sxthw(dst_reg, dst_reg);
 3212   %}
 3213 
 3214   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3215     Register dst_reg = as_Register($dst$$reg);
 3216     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3217              rscratch1, ldarh);
 3218     __ sxth(dst_reg, dst_reg);
 3219   %}
 3220 
 3221   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3222     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3223              rscratch1, ldarh);
 3224   %}
 3225 
 3226   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3227     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3228              rscratch1, ldarh);
 3229   %}
 3230 
 3231   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3232     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3233              rscratch1, ldarw);
 3234   %}
 3235 
 3236   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3237     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3238              rscratch1, ldarw);
 3239   %}
 3240 
 3241   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3242     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3243              rscratch1, ldar);
 3244   %}
 3245 
 3246   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3247     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3248              rscratch1, ldarw);
 3249     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3250   %}
 3251 
 3252   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3253     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3254              rscratch1, ldar);
 3255     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3256   %}
 3257 
 3258   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3259     Register src_reg = as_Register($src$$reg);
 3260     // we sometimes get asked to store the stack pointer into the
 3261     // current thread -- we cannot do that directly on AArch64
 3262     if (src_reg == r31_sp) {
 3263       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3264       __ mov(rscratch2, sp);
 3265       src_reg = rscratch2;
 3266     }
 3267     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3268                  rscratch1, stlr);
 3269   %}
 3270 
 3271   enc_class aarch64_enc_stlr0(memory mem) %{
 3272     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3273                  rscratch1, stlr);
 3274   %}
 3275 
 3276   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3277     {
 3278       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3279       __ fmovs(rscratch2, src_reg);
 3280     }
 3281     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3282                  rscratch1, stlrw);
 3283   %}
 3284 
 3285   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3286     {
 3287       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3288       __ fmovd(rscratch2, src_reg);
 3289     }
 3290     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3291                  rscratch1, stlr);
 3292   %}
 3293 
 3294   // synchronized read/update encodings
 3295 
 3296   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3297     Register dst_reg = as_Register($dst$$reg);
 3298     Register base = as_Register($mem$$base);
 3299     int index = $mem$$index;
 3300     int scale = $mem$$scale;
 3301     int disp = $mem$$disp;
 3302     if (index == -1) {
 3303        if (disp != 0) {
 3304         __ lea(rscratch1, Address(base, disp));
 3305         __ ldaxr(dst_reg, rscratch1);
 3306       } else {
 3307         // TODO
 3308         // should we ever get anything other than this case?
 3309         __ ldaxr(dst_reg, base);
 3310       }
 3311     } else {
 3312       Register index_reg = as_Register(index);
 3313       if (disp == 0) {
 3314         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3315         __ ldaxr(dst_reg, rscratch1);
 3316       } else {
 3317         __ lea(rscratch1, Address(base, disp));
 3318         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3319         __ ldaxr(dst_reg, rscratch1);
 3320       }
 3321     }
 3322   %}
 3323 
 3324   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3325     Register src_reg = as_Register($src$$reg);
 3326     Register base = as_Register($mem$$base);
 3327     int index = $mem$$index;
 3328     int scale = $mem$$scale;
 3329     int disp = $mem$$disp;
 3330     if (index == -1) {
 3331        if (disp != 0) {
 3332         __ lea(rscratch2, Address(base, disp));
 3333         __ stlxr(rscratch1, src_reg, rscratch2);
 3334       } else {
 3335         // TODO
 3336         // should we ever get anything other than this case?
 3337         __ stlxr(rscratch1, src_reg, base);
 3338       }
 3339     } else {
 3340       Register index_reg = as_Register(index);
 3341       if (disp == 0) {
 3342         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3343         __ stlxr(rscratch1, src_reg, rscratch2);
 3344       } else {
 3345         __ lea(rscratch2, Address(base, disp));
 3346         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3347         __ stlxr(rscratch1, src_reg, rscratch2);
 3348       }
 3349     }
 3350     __ cmpw(rscratch1, zr);
 3351   %}
 3352 
 3353   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3354     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3355     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3356                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3357                /*weak*/ false, noreg);
 3358   %}
 3359 
 3360   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3361     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3362     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3363                Assembler::word, /*acquire*/ false, /*release*/ true,
 3364                /*weak*/ false, noreg);
 3365   %}
 3366 
 3367   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3368     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3369     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3370                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3371                /*weak*/ false, noreg);
 3372   %}
 3373 
 3374   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3375     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3376     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3377                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3378                /*weak*/ false, noreg);
 3379   %}
 3380 
 3381 
 3382   // The only difference between aarch64_enc_cmpxchg and
 3383   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3384   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3385   // lock.
 3386   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3387     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3388     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3389                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3390                /*weak*/ false, noreg);
 3391   %}
 3392 
 3393   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3394     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3395     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3396                Assembler::word, /*acquire*/ true, /*release*/ true,
 3397                /*weak*/ false, noreg);
 3398   %}
 3399 
 3400   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3401     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3402     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3403                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3404                /*weak*/ false, noreg);
 3405   %}
 3406 
 3407   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3408     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3409     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3410                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3411                /*weak*/ false, noreg);
 3412   %}
 3413 
 3414   // auxiliary used for CompareAndSwapX to set result register
 3415   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3416     Register res_reg = as_Register($res$$reg);
 3417     __ cset(res_reg, Assembler::EQ);
 3418   %}
 3419 
 3420   // prefetch encodings
 3421 
 3422   enc_class aarch64_enc_prefetchw(memory mem) %{
 3423     Register base = as_Register($mem$$base);
 3424     int index = $mem$$index;
 3425     int scale = $mem$$scale;
 3426     int disp = $mem$$disp;
 3427     if (index == -1) {
 3428       // Fix up any out-of-range offsets.
 3429       assert_different_registers(rscratch1, base);
 3430       Address addr = Address(base, disp);
 3431       addr = __ legitimize_address(addr, 8, rscratch1);
 3432       __ prfm(addr, PSTL1KEEP);
 3433     } else {
 3434       Register index_reg = as_Register(index);
 3435       if (disp == 0) {
 3436         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3437       } else {
 3438         __ lea(rscratch1, Address(base, disp));
 3439 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3440       }
 3441     }
 3442   %}
 3443 
 3444   // mov encodings
 3445 
 3446   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3447     uint32_t con = (uint32_t)$src$$constant;
 3448     Register dst_reg = as_Register($dst$$reg);
 3449     if (con == 0) {
 3450       __ movw(dst_reg, zr);
 3451     } else {
 3452       __ movw(dst_reg, con);
 3453     }
 3454   %}
 3455 
 3456   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3457     Register dst_reg = as_Register($dst$$reg);
 3458     uint64_t con = (uint64_t)$src$$constant;
 3459     if (con == 0) {
 3460       __ mov(dst_reg, zr);
 3461     } else {
 3462       __ mov(dst_reg, con);
 3463     }
 3464   %}
 3465 
 3466   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3467     Register dst_reg = as_Register($dst$$reg);
 3468     address con = (address)$src$$constant;
 3469     if (con == nullptr || con == (address)1) {
 3470       ShouldNotReachHere();
 3471     } else {
 3472       relocInfo::relocType rtype = $src->constant_reloc();
 3473       if (rtype == relocInfo::oop_type) {
 3474         __ movoop(dst_reg, (jobject)con);
 3475       } else if (rtype == relocInfo::metadata_type) {
 3476         __ mov_metadata(dst_reg, (Metadata*)con);
 3477       } else {
 3478         assert(rtype == relocInfo::none, "unexpected reloc type");
 3479         if (! __ is_valid_AArch64_address(con) ||
 3480             con < (address)(uintptr_t)os::vm_page_size()) {
 3481           __ mov(dst_reg, con);
 3482         } else {
 3483           uint64_t offset;
 3484           __ adrp(dst_reg, con, offset);
 3485           __ add(dst_reg, dst_reg, offset);
 3486         }
 3487       }
 3488     }
 3489   %}
 3490 
 3491   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3492     Register dst_reg = as_Register($dst$$reg);
 3493     __ mov(dst_reg, zr);
 3494   %}
 3495 
 3496   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3497     Register dst_reg = as_Register($dst$$reg);
 3498     __ mov(dst_reg, (uint64_t)1);
 3499   %}
 3500 
 3501   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3502     Register dst_reg = as_Register($dst$$reg);
 3503     address con = (address)$src$$constant;
 3504     if (con == nullptr) {
 3505       ShouldNotReachHere();
 3506     } else {
 3507       relocInfo::relocType rtype = $src->constant_reloc();
 3508       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3509       __ set_narrow_oop(dst_reg, (jobject)con);
 3510     }
 3511   %}
 3512 
 3513   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3514     Register dst_reg = as_Register($dst$$reg);
 3515     __ mov(dst_reg, zr);
 3516   %}
 3517 
 3518   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3519     Register dst_reg = as_Register($dst$$reg);
 3520     address con = (address)$src$$constant;
 3521     if (con == nullptr) {
 3522       ShouldNotReachHere();
 3523     } else {
 3524       relocInfo::relocType rtype = $src->constant_reloc();
 3525       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3526       __ set_narrow_klass(dst_reg, (Klass *)con);
 3527     }
 3528   %}
 3529 
 3530   // arithmetic encodings
 3531 
 3532   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3533     Register dst_reg = as_Register($dst$$reg);
 3534     Register src_reg = as_Register($src1$$reg);
 3535     int32_t con = (int32_t)$src2$$constant;
 3536     // add has primary == 0, subtract has primary == 1
 3537     if ($primary) { con = -con; }
 3538     if (con < 0) {
 3539       __ subw(dst_reg, src_reg, -con);
 3540     } else {
 3541       __ addw(dst_reg, src_reg, con);
 3542     }
 3543   %}
 3544 
 3545   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3546     Register dst_reg = as_Register($dst$$reg);
 3547     Register src_reg = as_Register($src1$$reg);
 3548     int32_t con = (int32_t)$src2$$constant;
 3549     // add has primary == 0, subtract has primary == 1
 3550     if ($primary) { con = -con; }
 3551     if (con < 0) {
 3552       __ sub(dst_reg, src_reg, -con);
 3553     } else {
 3554       __ add(dst_reg, src_reg, con);
 3555     }
 3556   %}
 3557 
 3558   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3559    Register dst_reg = as_Register($dst$$reg);
 3560    Register src1_reg = as_Register($src1$$reg);
 3561    Register src2_reg = as_Register($src2$$reg);
 3562     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3563   %}
 3564 
 3565   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3566    Register dst_reg = as_Register($dst$$reg);
 3567    Register src1_reg = as_Register($src1$$reg);
 3568    Register src2_reg = as_Register($src2$$reg);
 3569     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3570   %}
 3571 
 3572   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3573    Register dst_reg = as_Register($dst$$reg);
 3574    Register src1_reg = as_Register($src1$$reg);
 3575    Register src2_reg = as_Register($src2$$reg);
 3576     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3577   %}
 3578 
 3579   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3580    Register dst_reg = as_Register($dst$$reg);
 3581    Register src1_reg = as_Register($src1$$reg);
 3582    Register src2_reg = as_Register($src2$$reg);
 3583     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3584   %}
 3585 
 3586   // compare instruction encodings
 3587 
 3588   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3589     Register reg1 = as_Register($src1$$reg);
 3590     Register reg2 = as_Register($src2$$reg);
 3591     __ cmpw(reg1, reg2);
 3592   %}
 3593 
 3594   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3595     Register reg = as_Register($src1$$reg);
 3596     int32_t val = $src2$$constant;
 3597     if (val >= 0) {
 3598       __ subsw(zr, reg, val);
 3599     } else {
 3600       __ addsw(zr, reg, -val);
 3601     }
 3602   %}
 3603 
 3604   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3605     Register reg1 = as_Register($src1$$reg);
 3606     uint32_t val = (uint32_t)$src2$$constant;
 3607     __ movw(rscratch1, val);
 3608     __ cmpw(reg1, rscratch1);
 3609   %}
 3610 
 3611   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3612     Register reg1 = as_Register($src1$$reg);
 3613     Register reg2 = as_Register($src2$$reg);
 3614     __ cmp(reg1, reg2);
 3615   %}
 3616 
 3617   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3618     Register reg = as_Register($src1$$reg);
 3619     int64_t val = $src2$$constant;
 3620     if (val >= 0) {
 3621       __ subs(zr, reg, val);
 3622     } else if (val != -val) {
 3623       __ adds(zr, reg, -val);
 3624     } else {
 3625     // aargh, Long.MIN_VALUE is a special case
 3626       __ orr(rscratch1, zr, (uint64_t)val);
 3627       __ subs(zr, reg, rscratch1);
 3628     }
 3629   %}
 3630 
 3631   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3632     Register reg1 = as_Register($src1$$reg);
 3633     uint64_t val = (uint64_t)$src2$$constant;
 3634     __ mov(rscratch1, val);
 3635     __ cmp(reg1, rscratch1);
 3636   %}
 3637 
 3638   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3639     Register reg1 = as_Register($src1$$reg);
 3640     Register reg2 = as_Register($src2$$reg);
 3641     __ cmp(reg1, reg2);
 3642   %}
 3643 
 3644   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3645     Register reg1 = as_Register($src1$$reg);
 3646     Register reg2 = as_Register($src2$$reg);
 3647     __ cmpw(reg1, reg2);
 3648   %}
 3649 
 3650   enc_class aarch64_enc_testp(iRegP src) %{
 3651     Register reg = as_Register($src$$reg);
 3652     __ cmp(reg, zr);
 3653   %}
 3654 
 3655   enc_class aarch64_enc_testn(iRegN src) %{
 3656     Register reg = as_Register($src$$reg);
 3657     __ cmpw(reg, zr);
 3658   %}
 3659 
 3660   enc_class aarch64_enc_b(label lbl) %{
 3661     Label *L = $lbl$$label;
 3662     __ b(*L);
 3663   %}
 3664 
 3665   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3666     Label *L = $lbl$$label;
 3667     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3668   %}
 3669 
 3670   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3671     Label *L = $lbl$$label;
 3672     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3673   %}
 3674 
 3675   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3676   %{
 3677      Register sub_reg = as_Register($sub$$reg);
 3678      Register super_reg = as_Register($super$$reg);
 3679      Register temp_reg = as_Register($temp$$reg);
 3680      Register result_reg = as_Register($result$$reg);
 3681 
 3682      Label miss;
 3683      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3684                                      nullptr, &miss,
 3685                                      /*set_cond_codes:*/ true);
 3686      if ($primary) {
 3687        __ mov(result_reg, zr);
 3688      }
 3689      __ bind(miss);
 3690   %}
 3691 
 3692   enc_class aarch64_enc_java_static_call(method meth) %{
 3693     address addr = (address)$meth$$method;
 3694     address call;
 3695     if (!_method) {
 3696       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3697       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
 3698       if (call == nullptr) {
 3699         ciEnv::current()->record_failure("CodeCache is full");
 3700         return;
 3701       }
 3702     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 3703       // The NOP here is purely to ensure that eliding a call to
 3704       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 3705       __ nop();
 3706       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 3707     } else {
 3708       int method_index = resolved_method_index(masm);
 3709       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3710                                                   : static_call_Relocation::spec(method_index);
 3711       call = __ trampoline_call(Address(addr, rspec));
 3712       if (call == nullptr) {
 3713         ciEnv::current()->record_failure("CodeCache is full");
 3714         return;
 3715       }
 3716       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 3717         // Calls of the same statically bound method can share
 3718         // a stub to the interpreter.
 3719         __ code()->shared_stub_to_interp_for(_method, call - __ begin());
 3720       } else {
 3721         // Emit stub for static call
 3722         address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
 3723         if (stub == nullptr) {
 3724           ciEnv::current()->record_failure("CodeCache is full");
 3725           return;
 3726         }
 3727       }
 3728     }
 3729 
 3730     __ post_call_nop();
 3731 
 3732     // Only non uncommon_trap calls need to reinitialize ptrue.
 3733     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3734       __ reinitialize_ptrue();
 3735     }
 3736   %}
 3737 
 3738   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3739     int method_index = resolved_method_index(masm);
 3740     address call = __ ic_call((address)$meth$$method, method_index);
 3741     if (call == nullptr) {
 3742       ciEnv::current()->record_failure("CodeCache is full");
 3743       return;
 3744     }
 3745     __ post_call_nop();
 3746     if (Compile::current()->max_vector_size() > 0) {
 3747       __ reinitialize_ptrue();
 3748     }
 3749   %}
 3750 
 3751   enc_class aarch64_enc_call_epilog() %{
 3752     if (VerifyStackAtCalls) {
 3753       // Check that stack depth is unchanged: find majik cookie on stack
 3754       __ call_Unimplemented();
 3755     }
 3756     if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic() && _method->return_type()->is_loaded()) {
 3757       // The last return value is not set by the callee but used to pass the null marker to compiled code.
 3758       // Search for the corresponding projection, get the register and emit code that initialized it.
 3759       uint con = (tf()->range_cc()->cnt() - 1);
 3760       for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
 3761         ProjNode* proj = fast_out(i)->as_Proj();
 3762         if (proj->_con == con) {
 3763           // Set null marker if r0 is non-null (a non-null value is returned buffered or scalarized)
 3764           OptoReg::Name optoReg = ra_->get_reg_first(proj);
 3765           VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
 3766           Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
 3767           __ cmp(r0, zr);
 3768           __ cset(toReg, Assembler::NE);
 3769           if (reg->is_stack()) {
 3770             int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
 3771             __ str(toReg, Address(sp, st_off));
 3772           }
 3773           break;
 3774         }
 3775       }
 3776       if (return_value_is_used()) {
 3777         // An inline type is returned as fields in multiple registers.
 3778         // R0 either contains an oop if the inline type is buffered or a pointer
 3779         // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
 3780         // if the lowest bit is set to allow C2 to use the oop after null checking.
 3781         // r0 &= (r0 & 1) - 1
 3782         __ andr(rscratch1, r0, 0x1);
 3783         __ sub(rscratch1, rscratch1, 0x1);
 3784         __ andr(r0, r0, rscratch1);
 3785       }
 3786     }
 3787   %}
 3788 
 3789   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3790     // some calls to generated routines (arraycopy code) are scheduled
 3791     // by C2 as runtime calls. if so we can call them using a br (they
 3792     // will be in a reachable segment) otherwise we have to use a blr
 3793     // which loads the absolute address into a register.
 3794     address entry = (address)$meth$$method;
 3795     CodeBlob *cb = CodeCache::find_blob(entry);
 3796     if (cb) {
 3797       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3798       if (call == nullptr) {
 3799         ciEnv::current()->record_failure("CodeCache is full");
 3800         return;
 3801       }
 3802       __ post_call_nop();
 3803     } else {
 3804       Label retaddr;
 3805       // Make the anchor frame walkable
 3806       __ adr(rscratch2, retaddr);
 3807       __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
 3808       __ lea(rscratch1, RuntimeAddress(entry));
 3809       __ blr(rscratch1);
 3810       __ bind(retaddr);
 3811       __ post_call_nop();
 3812     }
 3813     if (Compile::current()->max_vector_size() > 0) {
 3814       __ reinitialize_ptrue();
 3815     }
 3816   %}
 3817 
 3818   enc_class aarch64_enc_rethrow() %{
 3819     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3820   %}
 3821 
 3822   enc_class aarch64_enc_ret() %{
 3823 #ifdef ASSERT
 3824     if (Compile::current()->max_vector_size() > 0) {
 3825       __ verify_ptrue();
 3826     }
 3827 #endif
 3828     __ ret(lr);
 3829   %}
 3830 
 3831   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3832     Register target_reg = as_Register($jump_target$$reg);
 3833     __ br(target_reg);
 3834   %}
 3835 
 3836   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3837     Register target_reg = as_Register($jump_target$$reg);
 3838     // exception oop should be in r0
 3839     // ret addr has been popped into lr
 3840     // callee expects it in r3
 3841     __ mov(r3, lr);
 3842     __ br(target_reg);
 3843   %}
 3844 
 3845 %}
 3846 
 3847 //----------FRAME--------------------------------------------------------------
 3848 // Definition of frame structure and management information.
 3849 //
 3850 //  S T A C K   L A Y O U T    Allocators stack-slot number
 3851 //                             |   (to get allocators register number
 3852 //  G  Owned by    |        |  v    add OptoReg::stack0())
 3853 //  r   CALLER     |        |
 3854 //  o     |        +--------+      pad to even-align allocators stack-slot
 3855 //  w     V        |  pad0  |        numbers; owned by CALLER
 3856 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 3857 //  h     ^        |   in   |  5
 3858 //        |        |  args  |  4   Holes in incoming args owned by SELF
 3859 //  |     |        |        |  3
 3860 //  |     |        +--------+
 3861 //  V     |        | old out|      Empty on Intel, window on Sparc
 3862 //        |    old |preserve|      Must be even aligned.
 3863 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 3864 //        |        |   in   |  3   area for Intel ret address
 3865 //     Owned by    |preserve|      Empty on Sparc.
 3866 //       SELF      +--------+
 3867 //        |        |  pad2  |  2   pad to align old SP
 3868 //        |        +--------+  1
 3869 //        |        | locks  |  0
 3870 //        |        +--------+----> OptoReg::stack0(), even aligned
 3871 //        |        |  pad1  | 11   pad to align new SP
 3872 //        |        +--------+
 3873 //        |        |        | 10
 3874 //        |        | spills |  9   spills
 3875 //        V        |        |  8   (pad0 slot for callee)
 3876 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 3877 //        ^        |  out   |  7
 3878 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 3879 //     Owned by    +--------+
 3880 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 3881 //        |    new |preserve|      Must be even-aligned.
 3882 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 3883 //        |        |        |
 3884 //
 3885 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 3886 //         known from SELF's arguments and the Java calling convention.
 3887 //         Region 6-7 is determined per call site.
 3888 // Note 2: If the calling convention leaves holes in the incoming argument
 3889 //         area, those holes are owned by SELF.  Holes in the outgoing area
 3890 //         are owned by the CALLEE.  Holes should not be necessary in the
 3891 //         incoming area, as the Java calling convention is completely under
 3892 //         the control of the AD file.  Doubles can be sorted and packed to
 3893 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 3894 //         varargs C calling conventions.
 3895 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 3896 //         even aligned with pad0 as needed.
 3897 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 3898 //           (the latter is true on Intel but is it false on AArch64?)
 3899 //         region 6-11 is even aligned; it may be padded out more so that
 3900 //         the region from SP to FP meets the minimum stack alignment.
 3901 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 3902 //         alignment.  Region 11, pad1, may be dynamically extended so that
 3903 //         SP meets the minimum alignment.
 3904 
 3905 frame %{
 3906   // These three registers define part of the calling convention
 3907   // between compiled code and the interpreter.
 3908 
 3909   // Inline Cache Register or Method for I2C.
 3910   inline_cache_reg(R12);
 3911 
 3912   // Number of stack slots consumed by locking an object
 3913   sync_stack_slots(2);
 3914 
 3915   // Compiled code's Frame Pointer
 3916   frame_pointer(R31);
 3917 
 3918   // Interpreter stores its frame pointer in a register which is
 3919   // stored to the stack by I2CAdaptors.
 3920   // I2CAdaptors convert from interpreted java to compiled java.
 3921   interpreter_frame_pointer(R29);
 3922 
 3923   // Stack alignment requirement
 3924   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 3925 
 3926   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 3927   // for calls to C.  Supports the var-args backing area for register parms.
 3928   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 3929 
 3930   // The after-PROLOG location of the return address.  Location of
 3931   // return address specifies a type (REG or STACK) and a number
 3932   // representing the register number (i.e. - use a register name) or
 3933   // stack slot.
 3934   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 3935   // Otherwise, it is above the locks and verification slot and alignment word
 3936   // TODO this may well be correct but need to check why that - 2 is there
 3937   // ppc port uses 0 but we definitely need to allow for fixed_slots
 3938   // which folds in the space used for monitors
 3939   return_addr(STACK - 2 +
 3940               align_up((Compile::current()->in_preserve_stack_slots() +
 3941                         Compile::current()->fixed_slots()),
 3942                        stack_alignment_in_slots()));
 3943 
 3944   // Location of compiled Java return values.  Same as C for now.
 3945   return_value
 3946   %{
 3947     // TODO do we allow ideal_reg == Op_RegN???
 3948     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 3949            "only return normal values");
 3950 
 3951     static const int lo[Op_RegL + 1] = { // enum name
 3952       0,                                 // Op_Node
 3953       0,                                 // Op_Set
 3954       R0_num,                            // Op_RegN
 3955       R0_num,                            // Op_RegI
 3956       R0_num,                            // Op_RegP
 3957       V0_num,                            // Op_RegF
 3958       V0_num,                            // Op_RegD
 3959       R0_num                             // Op_RegL
 3960     };
 3961 
 3962     static const int hi[Op_RegL + 1] = { // enum name
 3963       0,                                 // Op_Node
 3964       0,                                 // Op_Set
 3965       OptoReg::Bad,                      // Op_RegN
 3966       OptoReg::Bad,                      // Op_RegI
 3967       R0_H_num,                          // Op_RegP
 3968       OptoReg::Bad,                      // Op_RegF
 3969       V0_H_num,                          // Op_RegD
 3970       R0_H_num                           // Op_RegL
 3971     };
 3972 
 3973     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 3974   %}
 3975 %}
 3976 
 3977 //----------ATTRIBUTES---------------------------------------------------------
 3978 //----------Operand Attributes-------------------------------------------------
 3979 op_attrib op_cost(1);        // Required cost attribute
 3980 
 3981 //----------Instruction Attributes---------------------------------------------
 3982 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 3983 ins_attrib ins_size(32);        // Required size attribute (in bits)
 3984 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 3985                                 // a non-matching short branch variant
 3986                                 // of some long branch?
 3987 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 3988                                 // be a power of 2) specifies the
 3989                                 // alignment that some part of the
 3990                                 // instruction (not necessarily the
 3991                                 // start) requires.  If > 1, a
 3992                                 // compute_padding() function must be
 3993                                 // provided for the instruction
 3994 
 3995 // Whether this node is expanded during code emission into a sequence of
 3996 // instructions and the first instruction can perform an implicit null check.
 3997 ins_attrib ins_is_late_expanded_null_check_candidate(false);
 3998 
 3999 //----------OPERANDS-----------------------------------------------------------
 4000 // Operand definitions must precede instruction definitions for correct parsing
 4001 // in the ADLC because operands constitute user defined types which are used in
 4002 // instruction definitions.
 4003 
 4004 //----------Simple Operands----------------------------------------------------
 4005 
 4006 // Integer operands 32 bit
 4007 // 32 bit immediate
 4008 operand immI()
 4009 %{
 4010   match(ConI);
 4011 
 4012   op_cost(0);
 4013   format %{ %}
 4014   interface(CONST_INTER);
 4015 %}
 4016 
 4017 // 32 bit zero
 4018 operand immI0()
 4019 %{
 4020   predicate(n->get_int() == 0);
 4021   match(ConI);
 4022 
 4023   op_cost(0);
 4024   format %{ %}
 4025   interface(CONST_INTER);
 4026 %}
 4027 
 4028 // 32 bit unit increment
 4029 operand immI_1()
 4030 %{
 4031   predicate(n->get_int() == 1);
 4032   match(ConI);
 4033 
 4034   op_cost(0);
 4035   format %{ %}
 4036   interface(CONST_INTER);
 4037 %}
 4038 
 4039 // 32 bit unit decrement
 4040 operand immI_M1()
 4041 %{
 4042   predicate(n->get_int() == -1);
 4043   match(ConI);
 4044 
 4045   op_cost(0);
 4046   format %{ %}
 4047   interface(CONST_INTER);
 4048 %}
 4049 
 4050 // Shift values for add/sub extension shift
 4051 operand immIExt()
 4052 %{
 4053   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4054   match(ConI);
 4055 
 4056   op_cost(0);
 4057   format %{ %}
 4058   interface(CONST_INTER);
 4059 %}
 4060 
 4061 operand immI_gt_1()
 4062 %{
 4063   predicate(n->get_int() > 1);
 4064   match(ConI);
 4065 
 4066   op_cost(0);
 4067   format %{ %}
 4068   interface(CONST_INTER);
 4069 %}
 4070 
 4071 operand immI_le_4()
 4072 %{
 4073   predicate(n->get_int() <= 4);
 4074   match(ConI);
 4075 
 4076   op_cost(0);
 4077   format %{ %}
 4078   interface(CONST_INTER);
 4079 %}
 4080 
 4081 operand immI_16()
 4082 %{
 4083   predicate(n->get_int() == 16);
 4084   match(ConI);
 4085 
 4086   op_cost(0);
 4087   format %{ %}
 4088   interface(CONST_INTER);
 4089 %}
 4090 
 4091 operand immI_24()
 4092 %{
 4093   predicate(n->get_int() == 24);
 4094   match(ConI);
 4095 
 4096   op_cost(0);
 4097   format %{ %}
 4098   interface(CONST_INTER);
 4099 %}
 4100 
 4101 operand immI_32()
 4102 %{
 4103   predicate(n->get_int() == 32);
 4104   match(ConI);
 4105 
 4106   op_cost(0);
 4107   format %{ %}
 4108   interface(CONST_INTER);
 4109 %}
 4110 
 4111 operand immI_48()
 4112 %{
 4113   predicate(n->get_int() == 48);
 4114   match(ConI);
 4115 
 4116   op_cost(0);
 4117   format %{ %}
 4118   interface(CONST_INTER);
 4119 %}
 4120 
 4121 operand immI_56()
 4122 %{
 4123   predicate(n->get_int() == 56);
 4124   match(ConI);
 4125 
 4126   op_cost(0);
 4127   format %{ %}
 4128   interface(CONST_INTER);
 4129 %}
 4130 
 4131 operand immI_255()
 4132 %{
 4133   predicate(n->get_int() == 255);
 4134   match(ConI);
 4135 
 4136   op_cost(0);
 4137   format %{ %}
 4138   interface(CONST_INTER);
 4139 %}
 4140 
 4141 operand immI_65535()
 4142 %{
 4143   predicate(n->get_int() == 65535);
 4144   match(ConI);
 4145 
 4146   op_cost(0);
 4147   format %{ %}
 4148   interface(CONST_INTER);
 4149 %}
 4150 
 4151 operand immI_positive()
 4152 %{
 4153   predicate(n->get_int() > 0);
 4154   match(ConI);
 4155 
 4156   op_cost(0);
 4157   format %{ %}
 4158   interface(CONST_INTER);
 4159 %}
 4160 
 4161 // BoolTest condition for signed compare
 4162 operand immI_cmp_cond()
 4163 %{
 4164   predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
 4165   match(ConI);
 4166 
 4167   op_cost(0);
 4168   format %{ %}
 4169   interface(CONST_INTER);
 4170 %}
 4171 
 4172 // BoolTest condition for unsigned compare
 4173 operand immI_cmpU_cond()
 4174 %{
 4175   predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
 4176   match(ConI);
 4177 
 4178   op_cost(0);
 4179   format %{ %}
 4180   interface(CONST_INTER);
 4181 %}
 4182 
 4183 operand immL_255()
 4184 %{
 4185   predicate(n->get_long() == 255L);
 4186   match(ConL);
 4187 
 4188   op_cost(0);
 4189   format %{ %}
 4190   interface(CONST_INTER);
 4191 %}
 4192 
 4193 operand immL_65535()
 4194 %{
 4195   predicate(n->get_long() == 65535L);
 4196   match(ConL);
 4197 
 4198   op_cost(0);
 4199   format %{ %}
 4200   interface(CONST_INTER);
 4201 %}
 4202 
 4203 operand immL_4294967295()
 4204 %{
 4205   predicate(n->get_long() == 4294967295L);
 4206   match(ConL);
 4207 
 4208   op_cost(0);
 4209   format %{ %}
 4210   interface(CONST_INTER);
 4211 %}
 4212 
 4213 operand immL_bitmask()
 4214 %{
 4215   predicate((n->get_long() != 0)
 4216             && ((n->get_long() & 0xc000000000000000l) == 0)
 4217             && is_power_of_2(n->get_long() + 1));
 4218   match(ConL);
 4219 
 4220   op_cost(0);
 4221   format %{ %}
 4222   interface(CONST_INTER);
 4223 %}
 4224 
 4225 operand immI_bitmask()
 4226 %{
 4227   predicate((n->get_int() != 0)
 4228             && ((n->get_int() & 0xc0000000) == 0)
 4229             && is_power_of_2(n->get_int() + 1));
 4230   match(ConI);
 4231 
 4232   op_cost(0);
 4233   format %{ %}
 4234   interface(CONST_INTER);
 4235 %}
 4236 
 4237 operand immL_positive_bitmaskI()
 4238 %{
 4239   predicate((n->get_long() != 0)
 4240             && ((julong)n->get_long() < 0x80000000ULL)
 4241             && is_power_of_2(n->get_long() + 1));
 4242   match(ConL);
 4243 
 4244   op_cost(0);
 4245   format %{ %}
 4246   interface(CONST_INTER);
 4247 %}
 4248 
 4249 // Scale values for scaled offset addressing modes (up to long but not quad)
 4250 operand immIScale()
 4251 %{
 4252   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4253   match(ConI);
 4254 
 4255   op_cost(0);
 4256   format %{ %}
 4257   interface(CONST_INTER);
 4258 %}
 4259 
 4260 // 5 bit signed integer
 4261 operand immI5()
 4262 %{
 4263   predicate(Assembler::is_simm(n->get_int(), 5));
 4264   match(ConI);
 4265 
 4266   op_cost(0);
 4267   format %{ %}
 4268   interface(CONST_INTER);
 4269 %}
 4270 
 4271 // 7 bit unsigned integer
 4272 operand immIU7()
 4273 %{
 4274   predicate(Assembler::is_uimm(n->get_int(), 7));
 4275   match(ConI);
 4276 
 4277   op_cost(0);
 4278   format %{ %}
 4279   interface(CONST_INTER);
 4280 %}
 4281 
 4282 // Offset for scaled or unscaled immediate loads and stores
 4283 operand immIOffset()
 4284 %{
 4285   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4286   match(ConI);
 4287 
 4288   op_cost(0);
 4289   format %{ %}
 4290   interface(CONST_INTER);
 4291 %}
 4292 
 4293 operand immIOffset1()
 4294 %{
 4295   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4296   match(ConI);
 4297 
 4298   op_cost(0);
 4299   format %{ %}
 4300   interface(CONST_INTER);
 4301 %}
 4302 
 4303 operand immIOffset2()
 4304 %{
 4305   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4306   match(ConI);
 4307 
 4308   op_cost(0);
 4309   format %{ %}
 4310   interface(CONST_INTER);
 4311 %}
 4312 
 4313 operand immIOffset4()
 4314 %{
 4315   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4316   match(ConI);
 4317 
 4318   op_cost(0);
 4319   format %{ %}
 4320   interface(CONST_INTER);
 4321 %}
 4322 
 4323 operand immIOffset8()
 4324 %{
 4325   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4326   match(ConI);
 4327 
 4328   op_cost(0);
 4329   format %{ %}
 4330   interface(CONST_INTER);
 4331 %}
 4332 
 4333 operand immIOffset16()
 4334 %{
 4335   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4336   match(ConI);
 4337 
 4338   op_cost(0);
 4339   format %{ %}
 4340   interface(CONST_INTER);
 4341 %}
 4342 
 4343 operand immLOffset()
 4344 %{
 4345   predicate(n->get_long() >= -256 && n->get_long() <= 65520);
 4346   match(ConL);
 4347 
 4348   op_cost(0);
 4349   format %{ %}
 4350   interface(CONST_INTER);
 4351 %}
 4352 
 4353 operand immLoffset1()
 4354 %{
 4355   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4356   match(ConL);
 4357 
 4358   op_cost(0);
 4359   format %{ %}
 4360   interface(CONST_INTER);
 4361 %}
 4362 
 4363 operand immLoffset2()
 4364 %{
 4365   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4366   match(ConL);
 4367 
 4368   op_cost(0);
 4369   format %{ %}
 4370   interface(CONST_INTER);
 4371 %}
 4372 
 4373 operand immLoffset4()
 4374 %{
 4375   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4376   match(ConL);
 4377 
 4378   op_cost(0);
 4379   format %{ %}
 4380   interface(CONST_INTER);
 4381 %}
 4382 
 4383 operand immLoffset8()
 4384 %{
 4385   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4386   match(ConL);
 4387 
 4388   op_cost(0);
 4389   format %{ %}
 4390   interface(CONST_INTER);
 4391 %}
 4392 
 4393 operand immLoffset16()
 4394 %{
 4395   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4396   match(ConL);
 4397 
 4398   op_cost(0);
 4399   format %{ %}
 4400   interface(CONST_INTER);
 4401 %}
 4402 
 4403 // 5 bit signed long integer
 4404 operand immL5()
 4405 %{
 4406   predicate(Assembler::is_simm(n->get_long(), 5));
 4407   match(ConL);
 4408 
 4409   op_cost(0);
 4410   format %{ %}
 4411   interface(CONST_INTER);
 4412 %}
 4413 
 4414 // 7 bit unsigned long integer
 4415 operand immLU7()
 4416 %{
 4417   predicate(Assembler::is_uimm(n->get_long(), 7));
 4418   match(ConL);
 4419 
 4420   op_cost(0);
 4421   format %{ %}
 4422   interface(CONST_INTER);
 4423 %}
 4424 
 4425 // 8 bit signed value.
 4426 operand immI8()
 4427 %{
 4428   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4429   match(ConI);
 4430 
 4431   op_cost(0);
 4432   format %{ %}
 4433   interface(CONST_INTER);
 4434 %}
 4435 
 4436 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4437 operand immIDupV()
 4438 %{
 4439   predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
 4440   match(ConI);
 4441 
 4442   op_cost(0);
 4443   format %{ %}
 4444   interface(CONST_INTER);
 4445 %}
 4446 
 4447 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4448 operand immLDupV()
 4449 %{
 4450   predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
 4451   match(ConL);
 4452 
 4453   op_cost(0);
 4454   format %{ %}
 4455   interface(CONST_INTER);
 4456 %}
 4457 
 4458 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4459 operand immHDupV()
 4460 %{
 4461   predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
 4462   match(ConH);
 4463 
 4464   op_cost(0);
 4465   format %{ %}
 4466   interface(CONST_INTER);
 4467 %}
 4468 
 4469 // 8 bit integer valid for vector add sub immediate
 4470 operand immBAddSubV()
 4471 %{
 4472   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4473   match(ConI);
 4474 
 4475   op_cost(0);
 4476   format %{ %}
 4477   interface(CONST_INTER);
 4478 %}
 4479 
 4480 // 32 bit integer valid for add sub immediate
 4481 operand immIAddSub()
 4482 %{
 4483   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4484   match(ConI);
 4485   op_cost(0);
 4486   format %{ %}
 4487   interface(CONST_INTER);
 4488 %}
 4489 
 4490 // 32 bit integer valid for vector add sub immediate
 4491 operand immIAddSubV()
 4492 %{
 4493   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4494   match(ConI);
 4495 
 4496   op_cost(0);
 4497   format %{ %}
 4498   interface(CONST_INTER);
 4499 %}
 4500 
 4501 // 32 bit unsigned integer valid for logical immediate
 4502 
 4503 operand immBLog()
 4504 %{
 4505   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4506   match(ConI);
 4507 
 4508   op_cost(0);
 4509   format %{ %}
 4510   interface(CONST_INTER);
 4511 %}
 4512 
 4513 operand immSLog()
 4514 %{
 4515   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4516   match(ConI);
 4517 
 4518   op_cost(0);
 4519   format %{ %}
 4520   interface(CONST_INTER);
 4521 %}
 4522 
 4523 operand immILog()
 4524 %{
 4525   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4526   match(ConI);
 4527 
 4528   op_cost(0);
 4529   format %{ %}
 4530   interface(CONST_INTER);
 4531 %}
 4532 
 4533 // Integer operands 64 bit
 4534 // 64 bit immediate
 4535 operand immL()
 4536 %{
 4537   match(ConL);
 4538 
 4539   op_cost(0);
 4540   format %{ %}
 4541   interface(CONST_INTER);
 4542 %}
 4543 
 4544 // 64 bit zero
 4545 operand immL0()
 4546 %{
 4547   predicate(n->get_long() == 0);
 4548   match(ConL);
 4549 
 4550   op_cost(0);
 4551   format %{ %}
 4552   interface(CONST_INTER);
 4553 %}
 4554 
 4555 // 64 bit unit decrement
 4556 operand immL_M1()
 4557 %{
 4558   predicate(n->get_long() == -1);
 4559   match(ConL);
 4560 
 4561   op_cost(0);
 4562   format %{ %}
 4563   interface(CONST_INTER);
 4564 %}
 4565 
 4566 // 64 bit integer valid for add sub immediate
 4567 operand immLAddSub()
 4568 %{
 4569   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4570   match(ConL);
 4571   op_cost(0);
 4572   format %{ %}
 4573   interface(CONST_INTER);
 4574 %}
 4575 
 4576 // 64 bit integer valid for addv subv immediate
 4577 operand immLAddSubV()
 4578 %{
 4579   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4580   match(ConL);
 4581 
 4582   op_cost(0);
 4583   format %{ %}
 4584   interface(CONST_INTER);
 4585 %}
 4586 
 4587 // 64 bit integer valid for logical immediate
 4588 operand immLLog()
 4589 %{
 4590   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4591   match(ConL);
 4592   op_cost(0);
 4593   format %{ %}
 4594   interface(CONST_INTER);
 4595 %}
 4596 
 4597 // Long Immediate: low 32-bit mask
 4598 operand immL_32bits()
 4599 %{
 4600   predicate(n->get_long() == 0xFFFFFFFFL);
 4601   match(ConL);
 4602   op_cost(0);
 4603   format %{ %}
 4604   interface(CONST_INTER);
 4605 %}
 4606 
 4607 // Pointer operands
 4608 // Pointer Immediate
 4609 operand immP()
 4610 %{
 4611   match(ConP);
 4612 
 4613   op_cost(0);
 4614   format %{ %}
 4615   interface(CONST_INTER);
 4616 %}
 4617 
 4618 // nullptr Pointer Immediate
 4619 operand immP0()
 4620 %{
 4621   predicate(n->get_ptr() == 0);
 4622   match(ConP);
 4623 
 4624   op_cost(0);
 4625   format %{ %}
 4626   interface(CONST_INTER);
 4627 %}
 4628 
 4629 // Pointer Immediate One
 4630 // this is used in object initialization (initial object header)
 4631 operand immP_1()
 4632 %{
 4633   predicate(n->get_ptr() == 1);
 4634   match(ConP);
 4635 
 4636   op_cost(0);
 4637   format %{ %}
 4638   interface(CONST_INTER);
 4639 %}
 4640 
 4641 // Float and Double operands
 4642 // Double Immediate
 4643 operand immD()
 4644 %{
 4645   match(ConD);
 4646   op_cost(0);
 4647   format %{ %}
 4648   interface(CONST_INTER);
 4649 %}
 4650 
 4651 // Double Immediate: +0.0d
 4652 operand immD0()
 4653 %{
 4654   predicate(jlong_cast(n->getd()) == 0);
 4655   match(ConD);
 4656 
 4657   op_cost(0);
 4658   format %{ %}
 4659   interface(CONST_INTER);
 4660 %}
 4661 
 4662 // constant 'double +0.0'.
 4663 operand immDPacked()
 4664 %{
 4665   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4666   match(ConD);
 4667   op_cost(0);
 4668   format %{ %}
 4669   interface(CONST_INTER);
 4670 %}
 4671 
 4672 // Float Immediate
 4673 operand immF()
 4674 %{
 4675   match(ConF);
 4676   op_cost(0);
 4677   format %{ %}
 4678   interface(CONST_INTER);
 4679 %}
 4680 
 4681 // Float Immediate: +0.0f.
 4682 operand immF0()
 4683 %{
 4684   predicate(jint_cast(n->getf()) == 0);
 4685   match(ConF);
 4686 
 4687   op_cost(0);
 4688   format %{ %}
 4689   interface(CONST_INTER);
 4690 %}
 4691 
 4692 // Half Float (FP16) Immediate
 4693 operand immH()
 4694 %{
 4695   match(ConH);
 4696   op_cost(0);
 4697   format %{ %}
 4698   interface(CONST_INTER);
 4699 %}
 4700 
 4701 //
 4702 operand immFPacked()
 4703 %{
 4704   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4705   match(ConF);
 4706   op_cost(0);
 4707   format %{ %}
 4708   interface(CONST_INTER);
 4709 %}
 4710 
 4711 // Narrow pointer operands
 4712 // Narrow Pointer Immediate
 4713 operand immN()
 4714 %{
 4715   match(ConN);
 4716 
 4717   op_cost(0);
 4718   format %{ %}
 4719   interface(CONST_INTER);
 4720 %}
 4721 
 4722 // Narrow nullptr Pointer Immediate
 4723 operand immN0()
 4724 %{
 4725   predicate(n->get_narrowcon() == 0);
 4726   match(ConN);
 4727 
 4728   op_cost(0);
 4729   format %{ %}
 4730   interface(CONST_INTER);
 4731 %}
 4732 
 4733 operand immNKlass()
 4734 %{
 4735   match(ConNKlass);
 4736 
 4737   op_cost(0);
 4738   format %{ %}
 4739   interface(CONST_INTER);
 4740 %}
 4741 
 4742 // Integer 32 bit Register Operands
 4743 // Integer 32 bitRegister (excludes SP)
 4744 operand iRegI()
 4745 %{
 4746   constraint(ALLOC_IN_RC(any_reg32));
 4747   match(RegI);
 4748   match(iRegINoSp);
 4749   op_cost(0);
 4750   format %{ %}
 4751   interface(REG_INTER);
 4752 %}
 4753 
 4754 // Integer 32 bit Register not Special
 4755 operand iRegINoSp()
 4756 %{
 4757   constraint(ALLOC_IN_RC(no_special_reg32));
 4758   match(RegI);
 4759   op_cost(0);
 4760   format %{ %}
 4761   interface(REG_INTER);
 4762 %}
 4763 
 4764 // Integer 64 bit Register Operands
 4765 // Integer 64 bit Register (includes SP)
 4766 operand iRegL()
 4767 %{
 4768   constraint(ALLOC_IN_RC(any_reg));
 4769   match(RegL);
 4770   match(iRegLNoSp);
 4771   op_cost(0);
 4772   format %{ %}
 4773   interface(REG_INTER);
 4774 %}
 4775 
 4776 // Integer 64 bit Register not Special
 4777 operand iRegLNoSp()
 4778 %{
 4779   constraint(ALLOC_IN_RC(no_special_reg));
 4780   match(RegL);
 4781   match(iRegL_R0);
 4782   format %{ %}
 4783   interface(REG_INTER);
 4784 %}
 4785 
 4786 // Pointer Register Operands
 4787 // Pointer Register
 4788 operand iRegP()
 4789 %{
 4790   constraint(ALLOC_IN_RC(ptr_reg));
 4791   match(RegP);
 4792   match(iRegPNoSp);
 4793   match(iRegP_R0);
 4794   //match(iRegP_R2);
 4795   //match(iRegP_R4);
 4796   match(iRegP_R5);
 4797   match(thread_RegP);
 4798   op_cost(0);
 4799   format %{ %}
 4800   interface(REG_INTER);
 4801 %}
 4802 
 4803 // Pointer 64 bit Register not Special
 4804 operand iRegPNoSp()
 4805 %{
 4806   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4807   match(RegP);
 4808   // match(iRegP);
 4809   // match(iRegP_R0);
 4810   // match(iRegP_R2);
 4811   // match(iRegP_R4);
 4812   // match(iRegP_R5);
 4813   // match(thread_RegP);
 4814   op_cost(0);
 4815   format %{ %}
 4816   interface(REG_INTER);
 4817 %}
 4818 
 4819 // This operand is not allowed to use rfp even if
 4820 // rfp is not used to hold the frame pointer.
 4821 operand iRegPNoSpNoRfp()
 4822 %{
 4823   constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
 4824   match(RegP);
 4825   match(iRegPNoSp);
 4826   op_cost(0);
 4827   format %{ %}
 4828   interface(REG_INTER);
 4829 %}
 4830 
 4831 // Pointer 64 bit Register R0 only
 4832 operand iRegP_R0()
 4833 %{
 4834   constraint(ALLOC_IN_RC(r0_reg));
 4835   match(RegP);
 4836   // match(iRegP);
 4837   match(iRegPNoSp);
 4838   op_cost(0);
 4839   format %{ %}
 4840   interface(REG_INTER);
 4841 %}
 4842 
 4843 // Pointer 64 bit Register R1 only
 4844 operand iRegP_R1()
 4845 %{
 4846   constraint(ALLOC_IN_RC(r1_reg));
 4847   match(RegP);
 4848   // match(iRegP);
 4849   match(iRegPNoSp);
 4850   op_cost(0);
 4851   format %{ %}
 4852   interface(REG_INTER);
 4853 %}
 4854 
 4855 // Pointer 64 bit Register R2 only
 4856 operand iRegP_R2()
 4857 %{
 4858   constraint(ALLOC_IN_RC(r2_reg));
 4859   match(RegP);
 4860   // match(iRegP);
 4861   match(iRegPNoSp);
 4862   op_cost(0);
 4863   format %{ %}
 4864   interface(REG_INTER);
 4865 %}
 4866 
 4867 // Pointer 64 bit Register R3 only
 4868 operand iRegP_R3()
 4869 %{
 4870   constraint(ALLOC_IN_RC(r3_reg));
 4871   match(RegP);
 4872   // match(iRegP);
 4873   match(iRegPNoSp);
 4874   op_cost(0);
 4875   format %{ %}
 4876   interface(REG_INTER);
 4877 %}
 4878 
 4879 // Pointer 64 bit Register R4 only
 4880 operand iRegP_R4()
 4881 %{
 4882   constraint(ALLOC_IN_RC(r4_reg));
 4883   match(RegP);
 4884   // match(iRegP);
 4885   match(iRegPNoSp);
 4886   op_cost(0);
 4887   format %{ %}
 4888   interface(REG_INTER);
 4889 %}
 4890 
 4891 // Pointer 64 bit Register R5 only
 4892 operand iRegP_R5()
 4893 %{
 4894   constraint(ALLOC_IN_RC(r5_reg));
 4895   match(RegP);
 4896   // match(iRegP);
 4897   match(iRegPNoSp);
 4898   op_cost(0);
 4899   format %{ %}
 4900   interface(REG_INTER);
 4901 %}
 4902 
 4903 // Pointer 64 bit Register R10 only
 4904 operand iRegP_R10()
 4905 %{
 4906   constraint(ALLOC_IN_RC(r10_reg));
 4907   match(RegP);
 4908   // match(iRegP);
 4909   match(iRegPNoSp);
 4910   op_cost(0);
 4911   format %{ %}
 4912   interface(REG_INTER);
 4913 %}
 4914 
 4915 // Long 64 bit Register R0 only
 4916 operand iRegL_R0()
 4917 %{
 4918   constraint(ALLOC_IN_RC(r0_reg));
 4919   match(RegL);
 4920   match(iRegLNoSp);
 4921   op_cost(0);
 4922   format %{ %}
 4923   interface(REG_INTER);
 4924 %}
 4925 
 4926 // Long 64 bit Register R11 only
 4927 operand iRegL_R11()
 4928 %{
 4929   constraint(ALLOC_IN_RC(r11_reg));
 4930   match(RegL);
 4931   match(iRegLNoSp);
 4932   op_cost(0);
 4933   format %{ %}
 4934   interface(REG_INTER);
 4935 %}
 4936 
 4937 // Register R0 only
 4938 operand iRegI_R0()
 4939 %{
 4940   constraint(ALLOC_IN_RC(int_r0_reg));
 4941   match(RegI);
 4942   match(iRegINoSp);
 4943   op_cost(0);
 4944   format %{ %}
 4945   interface(REG_INTER);
 4946 %}
 4947 
 4948 // Register R2 only
 4949 operand iRegI_R2()
 4950 %{
 4951   constraint(ALLOC_IN_RC(int_r2_reg));
 4952   match(RegI);
 4953   match(iRegINoSp);
 4954   op_cost(0);
 4955   format %{ %}
 4956   interface(REG_INTER);
 4957 %}
 4958 
 4959 // Register R3 only
 4960 operand iRegI_R3()
 4961 %{
 4962   constraint(ALLOC_IN_RC(int_r3_reg));
 4963   match(RegI);
 4964   match(iRegINoSp);
 4965   op_cost(0);
 4966   format %{ %}
 4967   interface(REG_INTER);
 4968 %}
 4969 
 4970 
 4971 // Register R4 only
 4972 operand iRegI_R4()
 4973 %{
 4974   constraint(ALLOC_IN_RC(int_r4_reg));
 4975   match(RegI);
 4976   match(iRegINoSp);
 4977   op_cost(0);
 4978   format %{ %}
 4979   interface(REG_INTER);
 4980 %}
 4981 
 4982 
 4983 // Pointer Register Operands
 4984 // Narrow Pointer Register
 4985 operand iRegN()
 4986 %{
 4987   constraint(ALLOC_IN_RC(any_reg32));
 4988   match(RegN);
 4989   match(iRegNNoSp);
 4990   op_cost(0);
 4991   format %{ %}
 4992   interface(REG_INTER);
 4993 %}
 4994 
 4995 // Integer 64 bit Register not Special
 4996 operand iRegNNoSp()
 4997 %{
 4998   constraint(ALLOC_IN_RC(no_special_reg32));
 4999   match(RegN);
 5000   op_cost(0);
 5001   format %{ %}
 5002   interface(REG_INTER);
 5003 %}
 5004 
 5005 // Float Register
 5006 // Float register operands
 5007 operand vRegF()
 5008 %{
 5009   constraint(ALLOC_IN_RC(float_reg));
 5010   match(RegF);
 5011 
 5012   op_cost(0);
 5013   format %{ %}
 5014   interface(REG_INTER);
 5015 %}
 5016 
 5017 // Double Register
 5018 // Double register operands
 5019 operand vRegD()
 5020 %{
 5021   constraint(ALLOC_IN_RC(double_reg));
 5022   match(RegD);
 5023 
 5024   op_cost(0);
 5025   format %{ %}
 5026   interface(REG_INTER);
 5027 %}
 5028 
 5029 // Generic vector class. This will be used for
 5030 // all vector operands, including NEON and SVE.
 5031 operand vReg()
 5032 %{
 5033   constraint(ALLOC_IN_RC(dynamic));
 5034   match(VecA);
 5035   match(VecD);
 5036   match(VecX);
 5037 
 5038   op_cost(0);
 5039   format %{ %}
 5040   interface(REG_INTER);
 5041 %}
 5042 
 5043 operand vReg_V10()
 5044 %{
 5045   constraint(ALLOC_IN_RC(v10_veca_reg));
 5046   match(vReg);
 5047 
 5048   op_cost(0);
 5049   format %{ %}
 5050   interface(REG_INTER);
 5051 %}
 5052 
 5053 operand vReg_V11()
 5054 %{
 5055   constraint(ALLOC_IN_RC(v11_veca_reg));
 5056   match(vReg);
 5057 
 5058   op_cost(0);
 5059   format %{ %}
 5060   interface(REG_INTER);
 5061 %}
 5062 
 5063 operand vReg_V12()
 5064 %{
 5065   constraint(ALLOC_IN_RC(v12_veca_reg));
 5066   match(vReg);
 5067 
 5068   op_cost(0);
 5069   format %{ %}
 5070   interface(REG_INTER);
 5071 %}
 5072 
 5073 operand vReg_V13()
 5074 %{
 5075   constraint(ALLOC_IN_RC(v13_veca_reg));
 5076   match(vReg);
 5077 
 5078   op_cost(0);
 5079   format %{ %}
 5080   interface(REG_INTER);
 5081 %}
 5082 
 5083 operand vReg_V17()
 5084 %{
 5085   constraint(ALLOC_IN_RC(v17_veca_reg));
 5086   match(vReg);
 5087 
 5088   op_cost(0);
 5089   format %{ %}
 5090   interface(REG_INTER);
 5091 %}
 5092 
 5093 operand vReg_V18()
 5094 %{
 5095   constraint(ALLOC_IN_RC(v18_veca_reg));
 5096   match(vReg);
 5097 
 5098   op_cost(0);
 5099   format %{ %}
 5100   interface(REG_INTER);
 5101 %}
 5102 
 5103 operand vReg_V23()
 5104 %{
 5105   constraint(ALLOC_IN_RC(v23_veca_reg));
 5106   match(vReg);
 5107 
 5108   op_cost(0);
 5109   format %{ %}
 5110   interface(REG_INTER);
 5111 %}
 5112 
 5113 operand vReg_V24()
 5114 %{
 5115   constraint(ALLOC_IN_RC(v24_veca_reg));
 5116   match(vReg);
 5117 
 5118   op_cost(0);
 5119   format %{ %}
 5120   interface(REG_INTER);
 5121 %}
 5122 
 5123 operand vecA()
 5124 %{
 5125   constraint(ALLOC_IN_RC(vectora_reg));
 5126   match(VecA);
 5127 
 5128   op_cost(0);
 5129   format %{ %}
 5130   interface(REG_INTER);
 5131 %}
 5132 
 5133 operand vecD()
 5134 %{
 5135   constraint(ALLOC_IN_RC(vectord_reg));
 5136   match(VecD);
 5137 
 5138   op_cost(0);
 5139   format %{ %}
 5140   interface(REG_INTER);
 5141 %}
 5142 
 5143 operand vecX()
 5144 %{
 5145   constraint(ALLOC_IN_RC(vectorx_reg));
 5146   match(VecX);
 5147 
 5148   op_cost(0);
 5149   format %{ %}
 5150   interface(REG_INTER);
 5151 %}
 5152 
 5153 operand vRegD_V0()
 5154 %{
 5155   constraint(ALLOC_IN_RC(v0_reg));
 5156   match(RegD);
 5157   op_cost(0);
 5158   format %{ %}
 5159   interface(REG_INTER);
 5160 %}
 5161 
 5162 operand vRegD_V1()
 5163 %{
 5164   constraint(ALLOC_IN_RC(v1_reg));
 5165   match(RegD);
 5166   op_cost(0);
 5167   format %{ %}
 5168   interface(REG_INTER);
 5169 %}
 5170 
 5171 operand vRegD_V2()
 5172 %{
 5173   constraint(ALLOC_IN_RC(v2_reg));
 5174   match(RegD);
 5175   op_cost(0);
 5176   format %{ %}
 5177   interface(REG_INTER);
 5178 %}
 5179 
 5180 operand vRegD_V3()
 5181 %{
 5182   constraint(ALLOC_IN_RC(v3_reg));
 5183   match(RegD);
 5184   op_cost(0);
 5185   format %{ %}
 5186   interface(REG_INTER);
 5187 %}
 5188 
 5189 operand vRegD_V4()
 5190 %{
 5191   constraint(ALLOC_IN_RC(v4_reg));
 5192   match(RegD);
 5193   op_cost(0);
 5194   format %{ %}
 5195   interface(REG_INTER);
 5196 %}
 5197 
 5198 operand vRegD_V5()
 5199 %{
 5200   constraint(ALLOC_IN_RC(v5_reg));
 5201   match(RegD);
 5202   op_cost(0);
 5203   format %{ %}
 5204   interface(REG_INTER);
 5205 %}
 5206 
 5207 operand vRegD_V6()
 5208 %{
 5209   constraint(ALLOC_IN_RC(v6_reg));
 5210   match(RegD);
 5211   op_cost(0);
 5212   format %{ %}
 5213   interface(REG_INTER);
 5214 %}
 5215 
 5216 operand vRegD_V7()
 5217 %{
 5218   constraint(ALLOC_IN_RC(v7_reg));
 5219   match(RegD);
 5220   op_cost(0);
 5221   format %{ %}
 5222   interface(REG_INTER);
 5223 %}
 5224 
 5225 operand vRegD_V12()
 5226 %{
 5227   constraint(ALLOC_IN_RC(v12_reg));
 5228   match(RegD);
 5229   op_cost(0);
 5230   format %{ %}
 5231   interface(REG_INTER);
 5232 %}
 5233 
 5234 operand vRegD_V13()
 5235 %{
 5236   constraint(ALLOC_IN_RC(v13_reg));
 5237   match(RegD);
 5238   op_cost(0);
 5239   format %{ %}
 5240   interface(REG_INTER);
 5241 %}
 5242 
 5243 operand pReg()
 5244 %{
 5245   constraint(ALLOC_IN_RC(pr_reg));
 5246   match(RegVectMask);
 5247   match(pRegGov);
 5248   op_cost(0);
 5249   format %{ %}
 5250   interface(REG_INTER);
 5251 %}
 5252 
 5253 operand pRegGov()
 5254 %{
 5255   constraint(ALLOC_IN_RC(gov_pr));
 5256   match(RegVectMask);
 5257   match(pReg);
 5258   op_cost(0);
 5259   format %{ %}
 5260   interface(REG_INTER);
 5261 %}
 5262 
 5263 operand pRegGov_P0()
 5264 %{
 5265   constraint(ALLOC_IN_RC(p0_reg));
 5266   match(RegVectMask);
 5267   op_cost(0);
 5268   format %{ %}
 5269   interface(REG_INTER);
 5270 %}
 5271 
 5272 operand pRegGov_P1()
 5273 %{
 5274   constraint(ALLOC_IN_RC(p1_reg));
 5275   match(RegVectMask);
 5276   op_cost(0);
 5277   format %{ %}
 5278   interface(REG_INTER);
 5279 %}
 5280 
 5281 // Flags register, used as output of signed compare instructions
 5282 
 5283 // note that on AArch64 we also use this register as the output for
 5284 // for floating point compare instructions (CmpF CmpD). this ensures
 5285 // that ordered inequality tests use GT, GE, LT or LE none of which
 5286 // pass through cases where the result is unordered i.e. one or both
 5287 // inputs to the compare is a NaN. this means that the ideal code can
 5288 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5289 // (where the comparison should always fail). EQ and NE tests are
 5290 // always generated in ideal code so that unordered folds into the NE
 5291 // case, matching the behaviour of AArch64 NE.
 5292 //
 5293 // This differs from x86 where the outputs of FP compares use a
 5294 // special FP flags registers and where compares based on this
 5295 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5296 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5297 // to explicitly handle the unordered case in branches. x86 also has
 5298 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5299 
 5300 operand rFlagsReg()
 5301 %{
 5302   constraint(ALLOC_IN_RC(int_flags));
 5303   match(RegFlags);
 5304 
 5305   op_cost(0);
 5306   format %{ "RFLAGS" %}
 5307   interface(REG_INTER);
 5308 %}
 5309 
 5310 // Flags register, used as output of unsigned compare instructions
 5311 operand rFlagsRegU()
 5312 %{
 5313   constraint(ALLOC_IN_RC(int_flags));
 5314   match(RegFlags);
 5315 
 5316   op_cost(0);
 5317   format %{ "RFLAGSU" %}
 5318   interface(REG_INTER);
 5319 %}
 5320 
 5321 // Special Registers
 5322 
 5323 // Method Register
 5324 operand inline_cache_RegP(iRegP reg)
 5325 %{
 5326   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5327   match(reg);
 5328   match(iRegPNoSp);
 5329   op_cost(0);
 5330   format %{ %}
 5331   interface(REG_INTER);
 5332 %}
 5333 
 5334 // Thread Register
 5335 operand thread_RegP(iRegP reg)
 5336 %{
 5337   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5338   match(reg);
 5339   op_cost(0);
 5340   format %{ %}
 5341   interface(REG_INTER);
 5342 %}
 5343 
 5344 //----------Memory Operands----------------------------------------------------
 5345 
 5346 operand indirect(iRegP reg)
 5347 %{
 5348   constraint(ALLOC_IN_RC(ptr_reg));
 5349   match(reg);
 5350   op_cost(0);
 5351   format %{ "[$reg]" %}
 5352   interface(MEMORY_INTER) %{
 5353     base($reg);
 5354     index(0xffffffff);
 5355     scale(0x0);
 5356     disp(0x0);
 5357   %}
 5358 %}
 5359 
 5360 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5361 %{
 5362   constraint(ALLOC_IN_RC(ptr_reg));
 5363   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5364   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5365   op_cost(0);
 5366   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5367   interface(MEMORY_INTER) %{
 5368     base($reg);
 5369     index($ireg);
 5370     scale($scale);
 5371     disp(0x0);
 5372   %}
 5373 %}
 5374 
 5375 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5376 %{
 5377   constraint(ALLOC_IN_RC(ptr_reg));
 5378   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5379   match(AddP reg (LShiftL lreg scale));
 5380   op_cost(0);
 5381   format %{ "$reg, $lreg lsl($scale)" %}
 5382   interface(MEMORY_INTER) %{
 5383     base($reg);
 5384     index($lreg);
 5385     scale($scale);
 5386     disp(0x0);
 5387   %}
 5388 %}
 5389 
 5390 operand indIndexI2L(iRegP reg, iRegI ireg)
 5391 %{
 5392   constraint(ALLOC_IN_RC(ptr_reg));
 5393   match(AddP reg (ConvI2L ireg));
 5394   op_cost(0);
 5395   format %{ "$reg, $ireg, 0, I2L" %}
 5396   interface(MEMORY_INTER) %{
 5397     base($reg);
 5398     index($ireg);
 5399     scale(0x0);
 5400     disp(0x0);
 5401   %}
 5402 %}
 5403 
 5404 operand indIndex(iRegP reg, iRegL lreg)
 5405 %{
 5406   constraint(ALLOC_IN_RC(ptr_reg));
 5407   match(AddP reg lreg);
 5408   op_cost(0);
 5409   format %{ "$reg, $lreg" %}
 5410   interface(MEMORY_INTER) %{
 5411     base($reg);
 5412     index($lreg);
 5413     scale(0x0);
 5414     disp(0x0);
 5415   %}
 5416 %}
 5417 
 5418 operand indOffI1(iRegP reg, immIOffset1 off)
 5419 %{
 5420   constraint(ALLOC_IN_RC(ptr_reg));
 5421   match(AddP reg off);
 5422   op_cost(0);
 5423   format %{ "[$reg, $off]" %}
 5424   interface(MEMORY_INTER) %{
 5425     base($reg);
 5426     index(0xffffffff);
 5427     scale(0x0);
 5428     disp($off);
 5429   %}
 5430 %}
 5431 
 5432 operand indOffI2(iRegP reg, immIOffset2 off)
 5433 %{
 5434   constraint(ALLOC_IN_RC(ptr_reg));
 5435   match(AddP reg off);
 5436   op_cost(0);
 5437   format %{ "[$reg, $off]" %}
 5438   interface(MEMORY_INTER) %{
 5439     base($reg);
 5440     index(0xffffffff);
 5441     scale(0x0);
 5442     disp($off);
 5443   %}
 5444 %}
 5445 
 5446 operand indOffI4(iRegP reg, immIOffset4 off)
 5447 %{
 5448   constraint(ALLOC_IN_RC(ptr_reg));
 5449   match(AddP reg off);
 5450   op_cost(0);
 5451   format %{ "[$reg, $off]" %}
 5452   interface(MEMORY_INTER) %{
 5453     base($reg);
 5454     index(0xffffffff);
 5455     scale(0x0);
 5456     disp($off);
 5457   %}
 5458 %}
 5459 
 5460 operand indOffI8(iRegP reg, immIOffset8 off)
 5461 %{
 5462   constraint(ALLOC_IN_RC(ptr_reg));
 5463   match(AddP reg off);
 5464   op_cost(0);
 5465   format %{ "[$reg, $off]" %}
 5466   interface(MEMORY_INTER) %{
 5467     base($reg);
 5468     index(0xffffffff);
 5469     scale(0x0);
 5470     disp($off);
 5471   %}
 5472 %}
 5473 
 5474 operand indOffI16(iRegP reg, immIOffset16 off)
 5475 %{
 5476   constraint(ALLOC_IN_RC(ptr_reg));
 5477   match(AddP reg off);
 5478   op_cost(0);
 5479   format %{ "[$reg, $off]" %}
 5480   interface(MEMORY_INTER) %{
 5481     base($reg);
 5482     index(0xffffffff);
 5483     scale(0x0);
 5484     disp($off);
 5485   %}
 5486 %}
 5487 
 5488 operand indOffL1(iRegP reg, immLoffset1 off)
 5489 %{
 5490   constraint(ALLOC_IN_RC(ptr_reg));
 5491   match(AddP reg off);
 5492   op_cost(0);
 5493   format %{ "[$reg, $off]" %}
 5494   interface(MEMORY_INTER) %{
 5495     base($reg);
 5496     index(0xffffffff);
 5497     scale(0x0);
 5498     disp($off);
 5499   %}
 5500 %}
 5501 
 5502 operand indOffL2(iRegP reg, immLoffset2 off)
 5503 %{
 5504   constraint(ALLOC_IN_RC(ptr_reg));
 5505   match(AddP reg off);
 5506   op_cost(0);
 5507   format %{ "[$reg, $off]" %}
 5508   interface(MEMORY_INTER) %{
 5509     base($reg);
 5510     index(0xffffffff);
 5511     scale(0x0);
 5512     disp($off);
 5513   %}
 5514 %}
 5515 
 5516 operand indOffL4(iRegP reg, immLoffset4 off)
 5517 %{
 5518   constraint(ALLOC_IN_RC(ptr_reg));
 5519   match(AddP reg off);
 5520   op_cost(0);
 5521   format %{ "[$reg, $off]" %}
 5522   interface(MEMORY_INTER) %{
 5523     base($reg);
 5524     index(0xffffffff);
 5525     scale(0x0);
 5526     disp($off);
 5527   %}
 5528 %}
 5529 
 5530 operand indOffL8(iRegP reg, immLoffset8 off)
 5531 %{
 5532   constraint(ALLOC_IN_RC(ptr_reg));
 5533   match(AddP reg off);
 5534   op_cost(0);
 5535   format %{ "[$reg, $off]" %}
 5536   interface(MEMORY_INTER) %{
 5537     base($reg);
 5538     index(0xffffffff);
 5539     scale(0x0);
 5540     disp($off);
 5541   %}
 5542 %}
 5543 
 5544 operand indOffL16(iRegP reg, immLoffset16 off)
 5545 %{
 5546   constraint(ALLOC_IN_RC(ptr_reg));
 5547   match(AddP reg off);
 5548   op_cost(0);
 5549   format %{ "[$reg, $off]" %}
 5550   interface(MEMORY_INTER) %{
 5551     base($reg);
 5552     index(0xffffffff);
 5553     scale(0x0);
 5554     disp($off);
 5555   %}
 5556 %}
 5557 
 5558 operand indirectX2P(iRegL reg)
 5559 %{
 5560   constraint(ALLOC_IN_RC(ptr_reg));
 5561   match(CastX2P reg);
 5562   op_cost(0);
 5563   format %{ "[$reg]\t# long -> ptr" %}
 5564   interface(MEMORY_INTER) %{
 5565     base($reg);
 5566     index(0xffffffff);
 5567     scale(0x0);
 5568     disp(0x0);
 5569   %}
 5570 %}
 5571 
 5572 operand indOffX2P(iRegL reg, immLOffset off)
 5573 %{
 5574   constraint(ALLOC_IN_RC(ptr_reg));
 5575   match(AddP (CastX2P reg) off);
 5576   op_cost(0);
 5577   format %{ "[$reg, $off]\t# long -> ptr" %}
 5578   interface(MEMORY_INTER) %{
 5579     base($reg);
 5580     index(0xffffffff);
 5581     scale(0x0);
 5582     disp($off);
 5583   %}
 5584 %}
 5585 
 5586 operand indirectN(iRegN reg)
 5587 %{
 5588   predicate(CompressedOops::shift() == 0);
 5589   constraint(ALLOC_IN_RC(ptr_reg));
 5590   match(DecodeN reg);
 5591   op_cost(0);
 5592   format %{ "[$reg]\t# narrow" %}
 5593   interface(MEMORY_INTER) %{
 5594     base($reg);
 5595     index(0xffffffff);
 5596     scale(0x0);
 5597     disp(0x0);
 5598   %}
 5599 %}
 5600 
 5601 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5602 %{
 5603   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5604   constraint(ALLOC_IN_RC(ptr_reg));
 5605   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5606   op_cost(0);
 5607   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5608   interface(MEMORY_INTER) %{
 5609     base($reg);
 5610     index($ireg);
 5611     scale($scale);
 5612     disp(0x0);
 5613   %}
 5614 %}
 5615 
 5616 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5617 %{
 5618   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5619   constraint(ALLOC_IN_RC(ptr_reg));
 5620   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5621   op_cost(0);
 5622   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5623   interface(MEMORY_INTER) %{
 5624     base($reg);
 5625     index($lreg);
 5626     scale($scale);
 5627     disp(0x0);
 5628   %}
 5629 %}
 5630 
 5631 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5632 %{
 5633   predicate(CompressedOops::shift() == 0);
 5634   constraint(ALLOC_IN_RC(ptr_reg));
 5635   match(AddP (DecodeN reg) (ConvI2L ireg));
 5636   op_cost(0);
 5637   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5638   interface(MEMORY_INTER) %{
 5639     base($reg);
 5640     index($ireg);
 5641     scale(0x0);
 5642     disp(0x0);
 5643   %}
 5644 %}
 5645 
 5646 operand indIndexN(iRegN reg, iRegL lreg)
 5647 %{
 5648   predicate(CompressedOops::shift() == 0);
 5649   constraint(ALLOC_IN_RC(ptr_reg));
 5650   match(AddP (DecodeN reg) lreg);
 5651   op_cost(0);
 5652   format %{ "$reg, $lreg\t# narrow" %}
 5653   interface(MEMORY_INTER) %{
 5654     base($reg);
 5655     index($lreg);
 5656     scale(0x0);
 5657     disp(0x0);
 5658   %}
 5659 %}
 5660 
 5661 operand indOffIN(iRegN reg, immIOffset off)
 5662 %{
 5663   predicate(CompressedOops::shift() == 0);
 5664   constraint(ALLOC_IN_RC(ptr_reg));
 5665   match(AddP (DecodeN reg) off);
 5666   op_cost(0);
 5667   format %{ "[$reg, $off]\t# narrow" %}
 5668   interface(MEMORY_INTER) %{
 5669     base($reg);
 5670     index(0xffffffff);
 5671     scale(0x0);
 5672     disp($off);
 5673   %}
 5674 %}
 5675 
 5676 operand indOffLN(iRegN reg, immLOffset off)
 5677 %{
 5678   predicate(CompressedOops::shift() == 0);
 5679   constraint(ALLOC_IN_RC(ptr_reg));
 5680   match(AddP (DecodeN reg) off);
 5681   op_cost(0);
 5682   format %{ "[$reg, $off]\t# narrow" %}
 5683   interface(MEMORY_INTER) %{
 5684     base($reg);
 5685     index(0xffffffff);
 5686     scale(0x0);
 5687     disp($off);
 5688   %}
 5689 %}
 5690 
 5691 
 5692 //----------Special Memory Operands--------------------------------------------
 5693 // Stack Slot Operand - This operand is used for loading and storing temporary
 5694 //                      values on the stack where a match requires a value to
 5695 //                      flow through memory.
 5696 operand stackSlotP(sRegP reg)
 5697 %{
 5698   constraint(ALLOC_IN_RC(stack_slots));
 5699   op_cost(100);
 5700   // No match rule because this operand is only generated in matching
 5701   // match(RegP);
 5702   format %{ "[$reg]" %}
 5703   interface(MEMORY_INTER) %{
 5704     base(0x1e);  // RSP
 5705     index(0x0);  // No Index
 5706     scale(0x0);  // No Scale
 5707     disp($reg);  // Stack Offset
 5708   %}
 5709 %}
 5710 
 5711 operand stackSlotI(sRegI reg)
 5712 %{
 5713   constraint(ALLOC_IN_RC(stack_slots));
 5714   // No match rule because this operand is only generated in matching
 5715   // match(RegI);
 5716   format %{ "[$reg]" %}
 5717   interface(MEMORY_INTER) %{
 5718     base(0x1e);  // RSP
 5719     index(0x0);  // No Index
 5720     scale(0x0);  // No Scale
 5721     disp($reg);  // Stack Offset
 5722   %}
 5723 %}
 5724 
 5725 operand stackSlotF(sRegF reg)
 5726 %{
 5727   constraint(ALLOC_IN_RC(stack_slots));
 5728   // No match rule because this operand is only generated in matching
 5729   // match(RegF);
 5730   format %{ "[$reg]" %}
 5731   interface(MEMORY_INTER) %{
 5732     base(0x1e);  // RSP
 5733     index(0x0);  // No Index
 5734     scale(0x0);  // No Scale
 5735     disp($reg);  // Stack Offset
 5736   %}
 5737 %}
 5738 
 5739 operand stackSlotD(sRegD reg)
 5740 %{
 5741   constraint(ALLOC_IN_RC(stack_slots));
 5742   // No match rule because this operand is only generated in matching
 5743   // match(RegD);
 5744   format %{ "[$reg]" %}
 5745   interface(MEMORY_INTER) %{
 5746     base(0x1e);  // RSP
 5747     index(0x0);  // No Index
 5748     scale(0x0);  // No Scale
 5749     disp($reg);  // Stack Offset
 5750   %}
 5751 %}
 5752 
 5753 operand stackSlotL(sRegL reg)
 5754 %{
 5755   constraint(ALLOC_IN_RC(stack_slots));
 5756   // No match rule because this operand is only generated in matching
 5757   // match(RegL);
 5758   format %{ "[$reg]" %}
 5759   interface(MEMORY_INTER) %{
 5760     base(0x1e);  // RSP
 5761     index(0x0);  // No Index
 5762     scale(0x0);  // No Scale
 5763     disp($reg);  // Stack Offset
 5764   %}
 5765 %}
 5766 
 5767 // Operands for expressing Control Flow
 5768 // NOTE: Label is a predefined operand which should not be redefined in
 5769 //       the AD file. It is generically handled within the ADLC.
 5770 
 5771 //----------Conditional Branch Operands----------------------------------------
 5772 // Comparison Op  - This is the operation of the comparison, and is limited to
 5773 //                  the following set of codes:
 5774 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 5775 //
 5776 // Other attributes of the comparison, such as unsignedness, are specified
 5777 // by the comparison instruction that sets a condition code flags register.
 5778 // That result is represented by a flags operand whose subtype is appropriate
 5779 // to the unsignedness (etc.) of the comparison.
 5780 //
 5781 // Later, the instruction which matches both the Comparison Op (a Bool) and
 5782 // the flags (produced by the Cmp) specifies the coding of the comparison op
 5783 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 5784 
 5785 // used for signed integral comparisons and fp comparisons
 5786 
 5787 operand cmpOp()
 5788 %{
 5789   match(Bool);
 5790 
 5791   format %{ "" %}
 5792   interface(COND_INTER) %{
 5793     equal(0x0, "eq");
 5794     not_equal(0x1, "ne");
 5795     less(0xb, "lt");
 5796     greater_equal(0xa, "ge");
 5797     less_equal(0xd, "le");
 5798     greater(0xc, "gt");
 5799     overflow(0x6, "vs");
 5800     no_overflow(0x7, "vc");
 5801   %}
 5802 %}
 5803 
 5804 // used for unsigned integral comparisons
 5805 
 5806 operand cmpOpU()
 5807 %{
 5808   match(Bool);
 5809 
 5810   format %{ "" %}
 5811   interface(COND_INTER) %{
 5812     equal(0x0, "eq");
 5813     not_equal(0x1, "ne");
 5814     less(0x3, "lo");
 5815     greater_equal(0x2, "hs");
 5816     less_equal(0x9, "ls");
 5817     greater(0x8, "hi");
 5818     overflow(0x6, "vs");
 5819     no_overflow(0x7, "vc");
 5820   %}
 5821 %}
 5822 
 5823 // used for certain integral comparisons which can be
 5824 // converted to cbxx or tbxx instructions
 5825 
 5826 operand cmpOpEqNe()
 5827 %{
 5828   match(Bool);
 5829   op_cost(0);
 5830   predicate(n->as_Bool()->_test._test == BoolTest::ne
 5831             || n->as_Bool()->_test._test == BoolTest::eq);
 5832 
 5833   format %{ "" %}
 5834   interface(COND_INTER) %{
 5835     equal(0x0, "eq");
 5836     not_equal(0x1, "ne");
 5837     less(0xb, "lt");
 5838     greater_equal(0xa, "ge");
 5839     less_equal(0xd, "le");
 5840     greater(0xc, "gt");
 5841     overflow(0x6, "vs");
 5842     no_overflow(0x7, "vc");
 5843   %}
 5844 %}
 5845 
 5846 // used for certain integral comparisons which can be
 5847 // converted to cbxx or tbxx instructions
 5848 
 5849 operand cmpOpLtGe()
 5850 %{
 5851   match(Bool);
 5852   op_cost(0);
 5853 
 5854   predicate(n->as_Bool()->_test._test == BoolTest::lt
 5855             || n->as_Bool()->_test._test == BoolTest::ge);
 5856 
 5857   format %{ "" %}
 5858   interface(COND_INTER) %{
 5859     equal(0x0, "eq");
 5860     not_equal(0x1, "ne");
 5861     less(0xb, "lt");
 5862     greater_equal(0xa, "ge");
 5863     less_equal(0xd, "le");
 5864     greater(0xc, "gt");
 5865     overflow(0x6, "vs");
 5866     no_overflow(0x7, "vc");
 5867   %}
 5868 %}
 5869 
 5870 // used for certain unsigned integral comparisons which can be
 5871 // converted to cbxx or tbxx instructions
 5872 
 5873 operand cmpOpUEqNeLeGt()
 5874 %{
 5875   match(Bool);
 5876   op_cost(0);
 5877 
 5878   predicate(n->as_Bool()->_test._test == BoolTest::eq ||
 5879             n->as_Bool()->_test._test == BoolTest::ne ||
 5880             n->as_Bool()->_test._test == BoolTest::le ||
 5881             n->as_Bool()->_test._test == BoolTest::gt);
 5882 
 5883   format %{ "" %}
 5884   interface(COND_INTER) %{
 5885     equal(0x0, "eq");
 5886     not_equal(0x1, "ne");
 5887     less(0x3, "lo");
 5888     greater_equal(0x2, "hs");
 5889     less_equal(0x9, "ls");
 5890     greater(0x8, "hi");
 5891     overflow(0x6, "vs");
 5892     no_overflow(0x7, "vc");
 5893   %}
 5894 %}
 5895 
 5896 // Special operand allowing long args to int ops to be truncated for free
 5897 
 5898 operand iRegL2I(iRegL reg) %{
 5899 
 5900   op_cost(0);
 5901 
 5902   match(ConvL2I reg);
 5903 
 5904   format %{ "l2i($reg)" %}
 5905 
 5906   interface(REG_INTER)
 5907 %}
 5908 
 5909 operand iRegL2P(iRegL reg) %{
 5910 
 5911   op_cost(0);
 5912 
 5913   match(CastX2P reg);
 5914 
 5915   format %{ "l2p($reg)" %}
 5916 
 5917   interface(REG_INTER)
 5918 %}
 5919 
 5920 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 5921 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 5922 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 5923 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 5924 
 5925 //----------OPERAND CLASSES----------------------------------------------------
 5926 // Operand Classes are groups of operands that are used as to simplify
 5927 // instruction definitions by not requiring the AD writer to specify
 5928 // separate instructions for every form of operand when the
 5929 // instruction accepts multiple operand types with the same basic
 5930 // encoding and format. The classic case of this is memory operands.
 5931 
 5932 // memory is used to define read/write location for load/store
 5933 // instruction defs. we can turn a memory op into an Address
 5934 
 5935 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 5936                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
 5937 
 5938 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 5939                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
 5940 
 5941 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 5942                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
 5943 
 5944 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 5945                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
 5946 
 5947 // All of the memory operands. For the pipeline description.
 5948 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 5949                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 5950                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
 5951 
 5952 
 5953 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 5954 // operations. it allows the src to be either an iRegI or a (ConvL2I
 5955 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 5956 // can be elided because the 32-bit instruction will just employ the
 5957 // lower 32 bits anyway.
 5958 //
 5959 // n.b. this does not elide all L2I conversions. if the truncated
 5960 // value is consumed by more than one operation then the ConvL2I
 5961 // cannot be bundled into the consuming nodes so an l2i gets planted
 5962 // (actually a movw $dst $src) and the downstream instructions consume
 5963 // the result of the l2i as an iRegI input. That's a shame since the
 5964 // movw is actually redundant but its not too costly.
 5965 
 5966 opclass iRegIorL2I(iRegI, iRegL2I);
 5967 opclass iRegPorL2P(iRegP, iRegL2P);
 5968 
 5969 //----------PIPELINE-----------------------------------------------------------
 5970 // Rules which define the behavior of the target architectures pipeline.
 5971 
 5972 // For specific pipelines, eg A53, define the stages of that pipeline
 5973 //pipe_desc(ISS, EX1, EX2, WR);
 5974 #define ISS S0
 5975 #define EX1 S1
 5976 #define EX2 S2
 5977 #define WR  S3
 5978 
 5979 // Integer ALU reg operation
 5980 pipeline %{
 5981 
 5982 attributes %{
 5983   // ARM instructions are of fixed length
 5984   fixed_size_instructions;        // Fixed size instructions TODO does
 5985   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 5986   // ARM instructions come in 32-bit word units
 5987   instruction_unit_size = 4;         // An instruction is 4 bytes long
 5988   instruction_fetch_unit_size = 64;  // The processor fetches one line
 5989   instruction_fetch_units = 1;       // of 64 bytes
 5990 %}
 5991 
 5992 // We don't use an actual pipeline model so don't care about resources
 5993 // or description. we do use pipeline classes to introduce fixed
 5994 // latencies
 5995 
 5996 //----------RESOURCES----------------------------------------------------------
 5997 // Resources are the functional units available to the machine
 5998 
 5999 resources( INS0, INS1, INS01 = INS0 | INS1,
 6000            ALU0, ALU1, ALU = ALU0 | ALU1,
 6001            MAC,
 6002            DIV,
 6003            BRANCH,
 6004            LDST,
 6005            NEON_FP);
 6006 
 6007 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6008 // Pipeline Description specifies the stages in the machine's pipeline
 6009 
 6010 // Define the pipeline as a generic 6 stage pipeline
 6011 pipe_desc(S0, S1, S2, S3, S4, S5);
 6012 
 6013 //----------PIPELINE CLASSES---------------------------------------------------
 6014 // Pipeline Classes describe the stages in which input and output are
 6015 // referenced by the hardware pipeline.
 6016 
 6017 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6018 %{
 6019   single_instruction;
 6020   src1   : S1(read);
 6021   src2   : S2(read);
 6022   dst    : S5(write);
 6023   INS01  : ISS;
 6024   NEON_FP : S5;
 6025 %}
 6026 
 6027 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6028 %{
 6029   single_instruction;
 6030   src1   : S1(read);
 6031   src2   : S2(read);
 6032   dst    : S5(write);
 6033   INS01  : ISS;
 6034   NEON_FP : S5;
 6035 %}
 6036 
 6037 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6038 %{
 6039   single_instruction;
 6040   src    : S1(read);
 6041   dst    : S5(write);
 6042   INS01  : ISS;
 6043   NEON_FP : S5;
 6044 %}
 6045 
 6046 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6047 %{
 6048   single_instruction;
 6049   src    : S1(read);
 6050   dst    : S5(write);
 6051   INS01  : ISS;
 6052   NEON_FP : S5;
 6053 %}
 6054 
 6055 pipe_class fp_d2f(vRegF dst, vRegD src)
 6056 %{
 6057   single_instruction;
 6058   src    : S1(read);
 6059   dst    : S5(write);
 6060   INS01  : ISS;
 6061   NEON_FP : S5;
 6062 %}
 6063 
 6064 pipe_class fp_f2d(vRegD dst, vRegF src)
 6065 %{
 6066   single_instruction;
 6067   src    : S1(read);
 6068   dst    : S5(write);
 6069   INS01  : ISS;
 6070   NEON_FP : S5;
 6071 %}
 6072 
 6073 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6074 %{
 6075   single_instruction;
 6076   src    : S1(read);
 6077   dst    : S5(write);
 6078   INS01  : ISS;
 6079   NEON_FP : S5;
 6080 %}
 6081 
 6082 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6083 %{
 6084   single_instruction;
 6085   src    : S1(read);
 6086   dst    : S5(write);
 6087   INS01  : ISS;
 6088   NEON_FP : S5;
 6089 %}
 6090 
 6091 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6092 %{
 6093   single_instruction;
 6094   src    : S1(read);
 6095   dst    : S5(write);
 6096   INS01  : ISS;
 6097   NEON_FP : S5;
 6098 %}
 6099 
 6100 pipe_class fp_l2f(vRegF dst, iRegL src)
 6101 %{
 6102   single_instruction;
 6103   src    : S1(read);
 6104   dst    : S5(write);
 6105   INS01  : ISS;
 6106   NEON_FP : S5;
 6107 %}
 6108 
 6109 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6110 %{
 6111   single_instruction;
 6112   src    : S1(read);
 6113   dst    : S5(write);
 6114   INS01  : ISS;
 6115   NEON_FP : S5;
 6116 %}
 6117 
 6118 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6119 %{
 6120   single_instruction;
 6121   src    : S1(read);
 6122   dst    : S5(write);
 6123   INS01  : ISS;
 6124   NEON_FP : S5;
 6125 %}
 6126 
 6127 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6128 %{
 6129   single_instruction;
 6130   src    : S1(read);
 6131   dst    : S5(write);
 6132   INS01  : ISS;
 6133   NEON_FP : S5;
 6134 %}
 6135 
 6136 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6137 %{
 6138   single_instruction;
 6139   src    : S1(read);
 6140   dst    : S5(write);
 6141   INS01  : ISS;
 6142   NEON_FP : S5;
 6143 %}
 6144 
 6145 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6146 %{
 6147   single_instruction;
 6148   src1   : S1(read);
 6149   src2   : S2(read);
 6150   dst    : S5(write);
 6151   INS0   : ISS;
 6152   NEON_FP : S5;
 6153 %}
 6154 
 6155 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6156 %{
 6157   single_instruction;
 6158   src1   : S1(read);
 6159   src2   : S2(read);
 6160   dst    : S5(write);
 6161   INS0   : ISS;
 6162   NEON_FP : S5;
 6163 %}
 6164 
 6165 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6166 %{
 6167   single_instruction;
 6168   cr     : S1(read);
 6169   src1   : S1(read);
 6170   src2   : S1(read);
 6171   dst    : S3(write);
 6172   INS01  : ISS;
 6173   NEON_FP : S3;
 6174 %}
 6175 
 6176 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6177 %{
 6178   single_instruction;
 6179   cr     : S1(read);
 6180   src1   : S1(read);
 6181   src2   : S1(read);
 6182   dst    : S3(write);
 6183   INS01  : ISS;
 6184   NEON_FP : S3;
 6185 %}
 6186 
 6187 pipe_class fp_imm_s(vRegF dst)
 6188 %{
 6189   single_instruction;
 6190   dst    : S3(write);
 6191   INS01  : ISS;
 6192   NEON_FP : S3;
 6193 %}
 6194 
 6195 pipe_class fp_imm_d(vRegD dst)
 6196 %{
 6197   single_instruction;
 6198   dst    : S3(write);
 6199   INS01  : ISS;
 6200   NEON_FP : S3;
 6201 %}
 6202 
 6203 pipe_class fp_load_constant_s(vRegF dst)
 6204 %{
 6205   single_instruction;
 6206   dst    : S4(write);
 6207   INS01  : ISS;
 6208   NEON_FP : S4;
 6209 %}
 6210 
 6211 pipe_class fp_load_constant_d(vRegD dst)
 6212 %{
 6213   single_instruction;
 6214   dst    : S4(write);
 6215   INS01  : ISS;
 6216   NEON_FP : S4;
 6217 %}
 6218 
 6219 //------- Integer ALU operations --------------------------
 6220 
 6221 // Integer ALU reg-reg operation
 6222 // Operands needed in EX1, result generated in EX2
 6223 // Eg.  ADD     x0, x1, x2
 6224 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6225 %{
 6226   single_instruction;
 6227   dst    : EX2(write);
 6228   src1   : EX1(read);
 6229   src2   : EX1(read);
 6230   INS01  : ISS; // Dual issue as instruction 0 or 1
 6231   ALU    : EX2;
 6232 %}
 6233 
 6234 // Integer ALU reg-reg operation with constant shift
 6235 // Shifted register must be available in LATE_ISS instead of EX1
 6236 // Eg.  ADD     x0, x1, x2, LSL #2
 6237 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6238 %{
 6239   single_instruction;
 6240   dst    : EX2(write);
 6241   src1   : EX1(read);
 6242   src2   : ISS(read);
 6243   INS01  : ISS;
 6244   ALU    : EX2;
 6245 %}
 6246 
 6247 // Integer ALU reg operation with constant shift
 6248 // Eg.  LSL     x0, x1, #shift
 6249 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6250 %{
 6251   single_instruction;
 6252   dst    : EX2(write);
 6253   src1   : ISS(read);
 6254   INS01  : ISS;
 6255   ALU    : EX2;
 6256 %}
 6257 
 6258 // Integer ALU reg-reg operation with variable shift
 6259 // Both operands must be available in LATE_ISS instead of EX1
 6260 // Result is available in EX1 instead of EX2
 6261 // Eg.  LSLV    x0, x1, x2
 6262 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6263 %{
 6264   single_instruction;
 6265   dst    : EX1(write);
 6266   src1   : ISS(read);
 6267   src2   : ISS(read);
 6268   INS01  : ISS;
 6269   ALU    : EX1;
 6270 %}
 6271 
 6272 // Integer ALU reg-reg operation with extract
 6273 // As for _vshift above, but result generated in EX2
 6274 // Eg.  EXTR    x0, x1, x2, #N
 6275 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6276 %{
 6277   single_instruction;
 6278   dst    : EX2(write);
 6279   src1   : ISS(read);
 6280   src2   : ISS(read);
 6281   INS1   : ISS; // Can only dual issue as Instruction 1
 6282   ALU    : EX1;
 6283 %}
 6284 
 6285 // Integer ALU reg operation
 6286 // Eg.  NEG     x0, x1
 6287 pipe_class ialu_reg(iRegI dst, iRegI src)
 6288 %{
 6289   single_instruction;
 6290   dst    : EX2(write);
 6291   src    : EX1(read);
 6292   INS01  : ISS;
 6293   ALU    : EX2;
 6294 %}
 6295 
 6296 // Integer ALU reg mmediate operation
 6297 // Eg.  ADD     x0, x1, #N
 6298 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6299 %{
 6300   single_instruction;
 6301   dst    : EX2(write);
 6302   src1   : EX1(read);
 6303   INS01  : ISS;
 6304   ALU    : EX2;
 6305 %}
 6306 
 6307 // Integer ALU immediate operation (no source operands)
 6308 // Eg.  MOV     x0, #N
 6309 pipe_class ialu_imm(iRegI dst)
 6310 %{
 6311   single_instruction;
 6312   dst    : EX1(write);
 6313   INS01  : ISS;
 6314   ALU    : EX1;
 6315 %}
 6316 
 6317 //------- Compare operation -------------------------------
 6318 
 6319 // Compare reg-reg
 6320 // Eg.  CMP     x0, x1
 6321 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6322 %{
 6323   single_instruction;
 6324 //  fixed_latency(16);
 6325   cr     : EX2(write);
 6326   op1    : EX1(read);
 6327   op2    : EX1(read);
 6328   INS01  : ISS;
 6329   ALU    : EX2;
 6330 %}
 6331 
 6332 // Compare reg-reg
 6333 // Eg.  CMP     x0, #N
 6334 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6335 %{
 6336   single_instruction;
 6337 //  fixed_latency(16);
 6338   cr     : EX2(write);
 6339   op1    : EX1(read);
 6340   INS01  : ISS;
 6341   ALU    : EX2;
 6342 %}
 6343 
 6344 //------- Conditional instructions ------------------------
 6345 
 6346 // Conditional no operands
 6347 // Eg.  CSINC   x0, zr, zr, <cond>
 6348 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6349 %{
 6350   single_instruction;
 6351   cr     : EX1(read);
 6352   dst    : EX2(write);
 6353   INS01  : ISS;
 6354   ALU    : EX2;
 6355 %}
 6356 
 6357 // Conditional 2 operand
 6358 // EG.  CSEL    X0, X1, X2, <cond>
 6359 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6360 %{
 6361   single_instruction;
 6362   cr     : EX1(read);
 6363   src1   : EX1(read);
 6364   src2   : EX1(read);
 6365   dst    : EX2(write);
 6366   INS01  : ISS;
 6367   ALU    : EX2;
 6368 %}
 6369 
 6370 // Conditional 2 operand
 6371 // EG.  CSEL    X0, X1, X2, <cond>
 6372 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6373 %{
 6374   single_instruction;
 6375   cr     : EX1(read);
 6376   src    : EX1(read);
 6377   dst    : EX2(write);
 6378   INS01  : ISS;
 6379   ALU    : EX2;
 6380 %}
 6381 
 6382 //------- Multiply pipeline operations --------------------
 6383 
 6384 // Multiply reg-reg
 6385 // Eg.  MUL     w0, w1, w2
 6386 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6387 %{
 6388   single_instruction;
 6389   dst    : WR(write);
 6390   src1   : ISS(read);
 6391   src2   : ISS(read);
 6392   INS01  : ISS;
 6393   MAC    : WR;
 6394 %}
 6395 
 6396 // Multiply accumulate
 6397 // Eg.  MADD    w0, w1, w2, w3
 6398 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6399 %{
 6400   single_instruction;
 6401   dst    : WR(write);
 6402   src1   : ISS(read);
 6403   src2   : ISS(read);
 6404   src3   : ISS(read);
 6405   INS01  : ISS;
 6406   MAC    : WR;
 6407 %}
 6408 
 6409 // Eg.  MUL     w0, w1, w2
 6410 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6411 %{
 6412   single_instruction;
 6413   fixed_latency(3); // Maximum latency for 64 bit mul
 6414   dst    : WR(write);
 6415   src1   : ISS(read);
 6416   src2   : ISS(read);
 6417   INS01  : ISS;
 6418   MAC    : WR;
 6419 %}
 6420 
 6421 // Multiply accumulate
 6422 // Eg.  MADD    w0, w1, w2, w3
 6423 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6424 %{
 6425   single_instruction;
 6426   fixed_latency(3); // Maximum latency for 64 bit mul
 6427   dst    : WR(write);
 6428   src1   : ISS(read);
 6429   src2   : ISS(read);
 6430   src3   : ISS(read);
 6431   INS01  : ISS;
 6432   MAC    : WR;
 6433 %}
 6434 
 6435 //------- Divide pipeline operations --------------------
 6436 
 6437 // Eg.  SDIV    w0, w1, w2
 6438 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6439 %{
 6440   single_instruction;
 6441   fixed_latency(8); // Maximum latency for 32 bit divide
 6442   dst    : WR(write);
 6443   src1   : ISS(read);
 6444   src2   : ISS(read);
 6445   INS0   : ISS; // Can only dual issue as instruction 0
 6446   DIV    : WR;
 6447 %}
 6448 
 6449 // Eg.  SDIV    x0, x1, x2
 6450 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6451 %{
 6452   single_instruction;
 6453   fixed_latency(16); // Maximum latency for 64 bit divide
 6454   dst    : WR(write);
 6455   src1   : ISS(read);
 6456   src2   : ISS(read);
 6457   INS0   : ISS; // Can only dual issue as instruction 0
 6458   DIV    : WR;
 6459 %}
 6460 
 6461 //------- Load pipeline operations ------------------------
 6462 
 6463 // Load - prefetch
 6464 // Eg.  PFRM    <mem>
 6465 pipe_class iload_prefetch(memory mem)
 6466 %{
 6467   single_instruction;
 6468   mem    : ISS(read);
 6469   INS01  : ISS;
 6470   LDST   : WR;
 6471 %}
 6472 
 6473 // Load - reg, mem
 6474 // Eg.  LDR     x0, <mem>
 6475 pipe_class iload_reg_mem(iRegI dst, memory mem)
 6476 %{
 6477   single_instruction;
 6478   dst    : WR(write);
 6479   mem    : ISS(read);
 6480   INS01  : ISS;
 6481   LDST   : WR;
 6482 %}
 6483 
 6484 // Load - reg, reg
 6485 // Eg.  LDR     x0, [sp, x1]
 6486 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 6487 %{
 6488   single_instruction;
 6489   dst    : WR(write);
 6490   src    : ISS(read);
 6491   INS01  : ISS;
 6492   LDST   : WR;
 6493 %}
 6494 
 6495 //------- Store pipeline operations -----------------------
 6496 
 6497 // Store - zr, mem
 6498 // Eg.  STR     zr, <mem>
 6499 pipe_class istore_mem(memory mem)
 6500 %{
 6501   single_instruction;
 6502   mem    : ISS(read);
 6503   INS01  : ISS;
 6504   LDST   : WR;
 6505 %}
 6506 
 6507 // Store - reg, mem
 6508 // Eg.  STR     x0, <mem>
 6509 pipe_class istore_reg_mem(iRegI src, memory mem)
 6510 %{
 6511   single_instruction;
 6512   mem    : ISS(read);
 6513   src    : EX2(read);
 6514   INS01  : ISS;
 6515   LDST   : WR;
 6516 %}
 6517 
 6518 // Store - reg, reg
 6519 // Eg. STR      x0, [sp, x1]
 6520 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 6521 %{
 6522   single_instruction;
 6523   dst    : ISS(read);
 6524   src    : EX2(read);
 6525   INS01  : ISS;
 6526   LDST   : WR;
 6527 %}
 6528 
 6529 //------- Store pipeline operations -----------------------
 6530 
 6531 // Branch
 6532 pipe_class pipe_branch()
 6533 %{
 6534   single_instruction;
 6535   INS01  : ISS;
 6536   BRANCH : EX1;
 6537 %}
 6538 
 6539 // Conditional branch
 6540 pipe_class pipe_branch_cond(rFlagsReg cr)
 6541 %{
 6542   single_instruction;
 6543   cr     : EX1(read);
 6544   INS01  : ISS;
 6545   BRANCH : EX1;
 6546 %}
 6547 
 6548 // Compare & Branch
 6549 // EG.  CBZ/CBNZ
 6550 pipe_class pipe_cmp_branch(iRegI op1)
 6551 %{
 6552   single_instruction;
 6553   op1    : EX1(read);
 6554   INS01  : ISS;
 6555   BRANCH : EX1;
 6556 %}
 6557 
 6558 //------- Synchronisation operations ----------------------
 6559 
 6560 // Any operation requiring serialization.
 6561 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 6562 pipe_class pipe_serial()
 6563 %{
 6564   single_instruction;
 6565   force_serialization;
 6566   fixed_latency(16);
 6567   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6568   LDST   : WR;
 6569 %}
 6570 
 6571 // Generic big/slow expanded idiom - also serialized
 6572 pipe_class pipe_slow()
 6573 %{
 6574   instruction_count(10);
 6575   multiple_bundles;
 6576   force_serialization;
 6577   fixed_latency(16);
 6578   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6579   LDST   : WR;
 6580 %}
 6581 
 6582 // Empty pipeline class
 6583 pipe_class pipe_class_empty()
 6584 %{
 6585   single_instruction;
 6586   fixed_latency(0);
 6587 %}
 6588 
 6589 // Default pipeline class.
 6590 pipe_class pipe_class_default()
 6591 %{
 6592   single_instruction;
 6593   fixed_latency(2);
 6594 %}
 6595 
 6596 // Pipeline class for compares.
 6597 pipe_class pipe_class_compare()
 6598 %{
 6599   single_instruction;
 6600   fixed_latency(16);
 6601 %}
 6602 
 6603 // Pipeline class for memory operations.
 6604 pipe_class pipe_class_memory()
 6605 %{
 6606   single_instruction;
 6607   fixed_latency(16);
 6608 %}
 6609 
 6610 // Pipeline class for call.
 6611 pipe_class pipe_class_call()
 6612 %{
 6613   single_instruction;
 6614   fixed_latency(100);
 6615 %}
 6616 
 6617 // Define the class for the Nop node.
 6618 define %{
 6619    MachNop = pipe_class_empty;
 6620 %}
 6621 
 6622 %}
 6623 //----------INSTRUCTIONS-------------------------------------------------------
 6624 //
 6625 // match      -- States which machine-independent subtree may be replaced
 6626 //               by this instruction.
 6627 // ins_cost   -- The estimated cost of this instruction is used by instruction
 6628 //               selection to identify a minimum cost tree of machine
 6629 //               instructions that matches a tree of machine-independent
 6630 //               instructions.
 6631 // format     -- A string providing the disassembly for this instruction.
 6632 //               The value of an instruction's operand may be inserted
 6633 //               by referring to it with a '$' prefix.
 6634 // opcode     -- Three instruction opcodes may be provided.  These are referred
 6635 //               to within an encode class as $primary, $secondary, and $tertiary
 6636 //               rrspectively.  The primary opcode is commonly used to
 6637 //               indicate the type of machine instruction, while secondary
 6638 //               and tertiary are often used for prefix options or addressing
 6639 //               modes.
 6640 // ins_encode -- A list of encode classes with parameters. The encode class
 6641 //               name must have been defined in an 'enc_class' specification
 6642 //               in the encode section of the architecture description.
 6643 
 6644 // ============================================================================
 6645 // Memory (Load/Store) Instructions
 6646 
 6647 // Load Instructions
 6648 
 6649 // Load Byte (8 bit signed)
 6650 instruct loadB(iRegINoSp dst, memory1 mem)
 6651 %{
 6652   match(Set dst (LoadB mem));
 6653   predicate(!needs_acquiring_load(n));
 6654 
 6655   ins_cost(4 * INSN_COST);
 6656   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 6657 
 6658   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 6659 
 6660   ins_pipe(iload_reg_mem);
 6661 %}
 6662 
 6663 // Load Byte (8 bit signed) into long
 6664 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 6665 %{
 6666   match(Set dst (ConvI2L (LoadB mem)));
 6667   predicate(!needs_acquiring_load(n->in(1)));
 6668 
 6669   ins_cost(4 * INSN_COST);
 6670   format %{ "ldrsb  $dst, $mem\t# byte" %}
 6671 
 6672   ins_encode(aarch64_enc_ldrsb(dst, mem));
 6673 
 6674   ins_pipe(iload_reg_mem);
 6675 %}
 6676 
 6677 // Load Byte (8 bit unsigned)
 6678 instruct loadUB(iRegINoSp dst, memory1 mem)
 6679 %{
 6680   match(Set dst (LoadUB mem));
 6681   predicate(!needs_acquiring_load(n));
 6682 
 6683   ins_cost(4 * INSN_COST);
 6684   format %{ "ldrbw  $dst, $mem\t# byte" %}
 6685 
 6686   ins_encode(aarch64_enc_ldrb(dst, mem));
 6687 
 6688   ins_pipe(iload_reg_mem);
 6689 %}
 6690 
 6691 // Load Byte (8 bit unsigned) into long
 6692 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 6693 %{
 6694   match(Set dst (ConvI2L (LoadUB mem)));
 6695   predicate(!needs_acquiring_load(n->in(1)));
 6696 
 6697   ins_cost(4 * INSN_COST);
 6698   format %{ "ldrb  $dst, $mem\t# byte" %}
 6699 
 6700   ins_encode(aarch64_enc_ldrb(dst, mem));
 6701 
 6702   ins_pipe(iload_reg_mem);
 6703 %}
 6704 
 6705 // Load Short (16 bit signed)
 6706 instruct loadS(iRegINoSp dst, memory2 mem)
 6707 %{
 6708   match(Set dst (LoadS mem));
 6709   predicate(!needs_acquiring_load(n));
 6710 
 6711   ins_cost(4 * INSN_COST);
 6712   format %{ "ldrshw  $dst, $mem\t# short" %}
 6713 
 6714   ins_encode(aarch64_enc_ldrshw(dst, mem));
 6715 
 6716   ins_pipe(iload_reg_mem);
 6717 %}
 6718 
 6719 // Load Short (16 bit signed) into long
 6720 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 6721 %{
 6722   match(Set dst (ConvI2L (LoadS mem)));
 6723   predicate(!needs_acquiring_load(n->in(1)));
 6724 
 6725   ins_cost(4 * INSN_COST);
 6726   format %{ "ldrsh  $dst, $mem\t# short" %}
 6727 
 6728   ins_encode(aarch64_enc_ldrsh(dst, mem));
 6729 
 6730   ins_pipe(iload_reg_mem);
 6731 %}
 6732 
 6733 // Load Char (16 bit unsigned)
 6734 instruct loadUS(iRegINoSp dst, memory2 mem)
 6735 %{
 6736   match(Set dst (LoadUS mem));
 6737   predicate(!needs_acquiring_load(n));
 6738 
 6739   ins_cost(4 * INSN_COST);
 6740   format %{ "ldrh  $dst, $mem\t# short" %}
 6741 
 6742   ins_encode(aarch64_enc_ldrh(dst, mem));
 6743 
 6744   ins_pipe(iload_reg_mem);
 6745 %}
 6746 
 6747 // Load Short/Char (16 bit unsigned) into long
 6748 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 6749 %{
 6750   match(Set dst (ConvI2L (LoadUS mem)));
 6751   predicate(!needs_acquiring_load(n->in(1)));
 6752 
 6753   ins_cost(4 * INSN_COST);
 6754   format %{ "ldrh  $dst, $mem\t# short" %}
 6755 
 6756   ins_encode(aarch64_enc_ldrh(dst, mem));
 6757 
 6758   ins_pipe(iload_reg_mem);
 6759 %}
 6760 
 6761 // Load Integer (32 bit signed)
 6762 instruct loadI(iRegINoSp dst, memory4 mem)
 6763 %{
 6764   match(Set dst (LoadI mem));
 6765   predicate(!needs_acquiring_load(n));
 6766 
 6767   ins_cost(4 * INSN_COST);
 6768   format %{ "ldrw  $dst, $mem\t# int" %}
 6769 
 6770   ins_encode(aarch64_enc_ldrw(dst, mem));
 6771 
 6772   ins_pipe(iload_reg_mem);
 6773 %}
 6774 
 6775 // Load Integer (32 bit signed) into long
 6776 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 6777 %{
 6778   match(Set dst (ConvI2L (LoadI mem)));
 6779   predicate(!needs_acquiring_load(n->in(1)));
 6780 
 6781   ins_cost(4 * INSN_COST);
 6782   format %{ "ldrsw  $dst, $mem\t# int" %}
 6783 
 6784   ins_encode(aarch64_enc_ldrsw(dst, mem));
 6785 
 6786   ins_pipe(iload_reg_mem);
 6787 %}
 6788 
 6789 // Load Integer (32 bit unsigned) into long
 6790 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 6791 %{
 6792   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 6793   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 6794 
 6795   ins_cost(4 * INSN_COST);
 6796   format %{ "ldrw  $dst, $mem\t# int" %}
 6797 
 6798   ins_encode(aarch64_enc_ldrw(dst, mem));
 6799 
 6800   ins_pipe(iload_reg_mem);
 6801 %}
 6802 
 6803 // Load Long (64 bit signed)
 6804 instruct loadL(iRegLNoSp dst, memory8 mem)
 6805 %{
 6806   match(Set dst (LoadL mem));
 6807   predicate(!needs_acquiring_load(n));
 6808 
 6809   ins_cost(4 * INSN_COST);
 6810   format %{ "ldr  $dst, $mem\t# int" %}
 6811 
 6812   ins_encode(aarch64_enc_ldr(dst, mem));
 6813 
 6814   ins_pipe(iload_reg_mem);
 6815 %}
 6816 
 6817 // Load Range
 6818 instruct loadRange(iRegINoSp dst, memory4 mem)
 6819 %{
 6820   match(Set dst (LoadRange mem));
 6821 
 6822   ins_cost(4 * INSN_COST);
 6823   format %{ "ldrw  $dst, $mem\t# range" %}
 6824 
 6825   ins_encode(aarch64_enc_ldrw(dst, mem));
 6826 
 6827   ins_pipe(iload_reg_mem);
 6828 %}
 6829 
 6830 // Load Pointer
 6831 instruct loadP(iRegPNoSp dst, memory8 mem)
 6832 %{
 6833   match(Set dst (LoadP mem));
 6834   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 6835 
 6836   ins_cost(4 * INSN_COST);
 6837   format %{ "ldr  $dst, $mem\t# ptr" %}
 6838 
 6839   ins_encode(aarch64_enc_ldr(dst, mem));
 6840 
 6841   ins_pipe(iload_reg_mem);
 6842 %}
 6843 
 6844 // Load Compressed Pointer
 6845 instruct loadN(iRegNNoSp dst, memory4 mem)
 6846 %{
 6847   match(Set dst (LoadN mem));
 6848   predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
 6849 
 6850   ins_cost(4 * INSN_COST);
 6851   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 6852 
 6853   ins_encode(aarch64_enc_ldrw(dst, mem));
 6854 
 6855   ins_pipe(iload_reg_mem);
 6856 %}
 6857 
 6858 // Load Klass Pointer
 6859 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 6860 %{
 6861   match(Set dst (LoadKlass mem));
 6862   predicate(!needs_acquiring_load(n));
 6863 
 6864   ins_cost(4 * INSN_COST);
 6865   format %{ "ldr  $dst, $mem\t# class" %}
 6866 
 6867   ins_encode(aarch64_enc_ldr(dst, mem));
 6868 
 6869   ins_pipe(iload_reg_mem);
 6870 %}
 6871 
 6872 // Load Narrow Klass Pointer
 6873 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 6874 %{
 6875   match(Set dst (LoadNKlass mem));
 6876   predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
 6877 
 6878   ins_cost(4 * INSN_COST);
 6879   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 6880 
 6881   ins_encode(aarch64_enc_ldrw(dst, mem));
 6882 
 6883   ins_pipe(iload_reg_mem);
 6884 %}
 6885 
 6886 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
 6887 %{
 6888   match(Set dst (LoadNKlass mem));
 6889   predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
 6890 
 6891   ins_cost(4 * INSN_COST);
 6892   format %{
 6893     "ldrw  $dst, $mem\t# compressed class ptr, shifted\n\t"
 6894     "lsrw  $dst, $dst, markWord::klass_shift_at_offset"
 6895   %}
 6896   ins_encode %{
 6897     // inlined aarch64_enc_ldrw
 6898     loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
 6899               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 6900     __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
 6901   %}
 6902   ins_pipe(iload_reg_mem);
 6903 %}
 6904 
 6905 // Load Float
 6906 instruct loadF(vRegF dst, memory4 mem)
 6907 %{
 6908   match(Set dst (LoadF mem));
 6909   predicate(!needs_acquiring_load(n));
 6910 
 6911   ins_cost(4 * INSN_COST);
 6912   format %{ "ldrs  $dst, $mem\t# float" %}
 6913 
 6914   ins_encode( aarch64_enc_ldrs(dst, mem) );
 6915 
 6916   ins_pipe(pipe_class_memory);
 6917 %}
 6918 
 6919 // Load Double
 6920 instruct loadD(vRegD dst, memory8 mem)
 6921 %{
 6922   match(Set dst (LoadD mem));
 6923   predicate(!needs_acquiring_load(n));
 6924 
 6925   ins_cost(4 * INSN_COST);
 6926   format %{ "ldrd  $dst, $mem\t# double" %}
 6927 
 6928   ins_encode( aarch64_enc_ldrd(dst, mem) );
 6929 
 6930   ins_pipe(pipe_class_memory);
 6931 %}
 6932 
 6933 
 6934 // Load Int Constant
 6935 instruct loadConI(iRegINoSp dst, immI src)
 6936 %{
 6937   match(Set dst src);
 6938 
 6939   ins_cost(INSN_COST);
 6940   format %{ "mov $dst, $src\t# int" %}
 6941 
 6942   ins_encode( aarch64_enc_movw_imm(dst, src) );
 6943 
 6944   ins_pipe(ialu_imm);
 6945 %}
 6946 
 6947 // Load Long Constant
 6948 instruct loadConL(iRegLNoSp dst, immL src)
 6949 %{
 6950   match(Set dst src);
 6951 
 6952   ins_cost(INSN_COST);
 6953   format %{ "mov $dst, $src\t# long" %}
 6954 
 6955   ins_encode( aarch64_enc_mov_imm(dst, src) );
 6956 
 6957   ins_pipe(ialu_imm);
 6958 %}
 6959 
 6960 // Load Pointer Constant
 6961 
 6962 instruct loadConP(iRegPNoSp dst, immP con)
 6963 %{
 6964   match(Set dst con);
 6965 
 6966   ins_cost(INSN_COST * 4);
 6967   format %{
 6968     "mov  $dst, $con\t# ptr"
 6969   %}
 6970 
 6971   ins_encode(aarch64_enc_mov_p(dst, con));
 6972 
 6973   ins_pipe(ialu_imm);
 6974 %}
 6975 
 6976 // Load Null Pointer Constant
 6977 
 6978 instruct loadConP0(iRegPNoSp dst, immP0 con)
 6979 %{
 6980   match(Set dst con);
 6981 
 6982   ins_cost(INSN_COST);
 6983   format %{ "mov  $dst, $con\t# nullptr ptr" %}
 6984 
 6985   ins_encode(aarch64_enc_mov_p0(dst, con));
 6986 
 6987   ins_pipe(ialu_imm);
 6988 %}
 6989 
 6990 // Load Pointer Constant One
 6991 
 6992 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 6993 %{
 6994   match(Set dst con);
 6995 
 6996   ins_cost(INSN_COST);
 6997   format %{ "mov  $dst, $con\t# nullptr ptr" %}
 6998 
 6999   ins_encode(aarch64_enc_mov_p1(dst, con));
 7000 
 7001   ins_pipe(ialu_imm);
 7002 %}
 7003 
 7004 // Load Narrow Pointer Constant
 7005 
 7006 instruct loadConN(iRegNNoSp dst, immN con)
 7007 %{
 7008   match(Set dst con);
 7009 
 7010   ins_cost(INSN_COST * 4);
 7011   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7012 
 7013   ins_encode(aarch64_enc_mov_n(dst, con));
 7014 
 7015   ins_pipe(ialu_imm);
 7016 %}
 7017 
 7018 // Load Narrow Null Pointer Constant
 7019 
 7020 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7021 %{
 7022   match(Set dst con);
 7023 
 7024   ins_cost(INSN_COST);
 7025   format %{ "mov  $dst, $con\t# compressed nullptr ptr" %}
 7026 
 7027   ins_encode(aarch64_enc_mov_n0(dst, con));
 7028 
 7029   ins_pipe(ialu_imm);
 7030 %}
 7031 
 7032 // Load Narrow Klass Constant
 7033 
 7034 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7035 %{
 7036   match(Set dst con);
 7037 
 7038   ins_cost(INSN_COST);
 7039   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7040 
 7041   ins_encode(aarch64_enc_mov_nk(dst, con));
 7042 
 7043   ins_pipe(ialu_imm);
 7044 %}
 7045 
 7046 // Load Packed Float Constant
 7047 
 7048 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7049   match(Set dst con);
 7050   ins_cost(INSN_COST * 4);
 7051   format %{ "fmovs  $dst, $con"%}
 7052   ins_encode %{
 7053     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7054   %}
 7055 
 7056   ins_pipe(fp_imm_s);
 7057 %}
 7058 
 7059 // Load Float Constant
 7060 
 7061 instruct loadConF(vRegF dst, immF con) %{
 7062   match(Set dst con);
 7063 
 7064   ins_cost(INSN_COST * 4);
 7065 
 7066   format %{
 7067     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7068   %}
 7069 
 7070   ins_encode %{
 7071     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7072   %}
 7073 
 7074   ins_pipe(fp_load_constant_s);
 7075 %}
 7076 
 7077 // Load Packed Double Constant
 7078 
 7079 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7080   match(Set dst con);
 7081   ins_cost(INSN_COST);
 7082   format %{ "fmovd  $dst, $con"%}
 7083   ins_encode %{
 7084     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7085   %}
 7086 
 7087   ins_pipe(fp_imm_d);
 7088 %}
 7089 
 7090 // Load Double Constant
 7091 
 7092 instruct loadConD(vRegD dst, immD con) %{
 7093   match(Set dst con);
 7094 
 7095   ins_cost(INSN_COST * 5);
 7096   format %{
 7097     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7098   %}
 7099 
 7100   ins_encode %{
 7101     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7102   %}
 7103 
 7104   ins_pipe(fp_load_constant_d);
 7105 %}
 7106 
 7107 // Load Half Float Constant
 7108 instruct loadConH(vRegF dst, immH con) %{
 7109   match(Set dst con);
 7110   format %{ "mov    rscratch1, $con\n\t"
 7111             "fmov   $dst, rscratch1"
 7112          %}
 7113   ins_encode %{
 7114     __ movw(rscratch1, (uint32_t)$con$$constant);
 7115     __ fmovs($dst$$FloatRegister, rscratch1);
 7116   %}
 7117   ins_pipe(pipe_class_default);
 7118 %}
 7119 
 7120 // Store Instructions
 7121 
 7122 // Store Byte
 7123 instruct storeB(iRegIorL2I src, memory1 mem)
 7124 %{
 7125   match(Set mem (StoreB mem src));
 7126   predicate(!needs_releasing_store(n));
 7127 
 7128   ins_cost(INSN_COST);
 7129   format %{ "strb  $src, $mem\t# byte" %}
 7130 
 7131   ins_encode(aarch64_enc_strb(src, mem));
 7132 
 7133   ins_pipe(istore_reg_mem);
 7134 %}
 7135 
 7136 
 7137 instruct storeimmB0(immI0 zero, memory1 mem)
 7138 %{
 7139   match(Set mem (StoreB mem zero));
 7140   predicate(!needs_releasing_store(n));
 7141 
 7142   ins_cost(INSN_COST);
 7143   format %{ "strb rscractch2, $mem\t# byte" %}
 7144 
 7145   ins_encode(aarch64_enc_strb0(mem));
 7146 
 7147   ins_pipe(istore_mem);
 7148 %}
 7149 
 7150 // Store Char/Short
 7151 instruct storeC(iRegIorL2I src, memory2 mem)
 7152 %{
 7153   match(Set mem (StoreC mem src));
 7154   predicate(!needs_releasing_store(n));
 7155 
 7156   ins_cost(INSN_COST);
 7157   format %{ "strh  $src, $mem\t# short" %}
 7158 
 7159   ins_encode(aarch64_enc_strh(src, mem));
 7160 
 7161   ins_pipe(istore_reg_mem);
 7162 %}
 7163 
 7164 instruct storeimmC0(immI0 zero, memory2 mem)
 7165 %{
 7166   match(Set mem (StoreC mem zero));
 7167   predicate(!needs_releasing_store(n));
 7168 
 7169   ins_cost(INSN_COST);
 7170   format %{ "strh  zr, $mem\t# short" %}
 7171 
 7172   ins_encode(aarch64_enc_strh0(mem));
 7173 
 7174   ins_pipe(istore_mem);
 7175 %}
 7176 
 7177 // Store Integer
 7178 
 7179 instruct storeI(iRegIorL2I src, memory4 mem)
 7180 %{
 7181   match(Set mem(StoreI mem src));
 7182   predicate(!needs_releasing_store(n));
 7183 
 7184   ins_cost(INSN_COST);
 7185   format %{ "strw  $src, $mem\t# int" %}
 7186 
 7187   ins_encode(aarch64_enc_strw(src, mem));
 7188 
 7189   ins_pipe(istore_reg_mem);
 7190 %}
 7191 
 7192 instruct storeimmI0(immI0 zero, memory4 mem)
 7193 %{
 7194   match(Set mem(StoreI mem zero));
 7195   predicate(!needs_releasing_store(n));
 7196 
 7197   ins_cost(INSN_COST);
 7198   format %{ "strw  zr, $mem\t# int" %}
 7199 
 7200   ins_encode(aarch64_enc_strw0(mem));
 7201 
 7202   ins_pipe(istore_mem);
 7203 %}
 7204 
 7205 // Store Long (64 bit signed)
 7206 instruct storeL(iRegL src, memory8 mem)
 7207 %{
 7208   match(Set mem (StoreL mem src));
 7209   predicate(!needs_releasing_store(n));
 7210 
 7211   ins_cost(INSN_COST);
 7212   format %{ "str  $src, $mem\t# int" %}
 7213 
 7214   ins_encode(aarch64_enc_str(src, mem));
 7215 
 7216   ins_pipe(istore_reg_mem);
 7217 %}
 7218 
 7219 // Store Long (64 bit signed)
 7220 instruct storeimmL0(immL0 zero, memory8 mem)
 7221 %{
 7222   match(Set mem (StoreL mem zero));
 7223   predicate(!needs_releasing_store(n));
 7224 
 7225   ins_cost(INSN_COST);
 7226   format %{ "str  zr, $mem\t# int" %}
 7227 
 7228   ins_encode(aarch64_enc_str0(mem));
 7229 
 7230   ins_pipe(istore_mem);
 7231 %}
 7232 
 7233 // Store Pointer
 7234 instruct storeP(iRegP src, memory8 mem)
 7235 %{
 7236   match(Set mem (StoreP mem src));
 7237   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7238 
 7239   ins_cost(INSN_COST);
 7240   format %{ "str  $src, $mem\t# ptr" %}
 7241 
 7242   ins_encode(aarch64_enc_str(src, mem));
 7243 
 7244   ins_pipe(istore_reg_mem);
 7245 %}
 7246 
 7247 // Store Pointer
 7248 instruct storeimmP0(immP0 zero, memory8 mem)
 7249 %{
 7250   match(Set mem (StoreP mem zero));
 7251   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7252 
 7253   ins_cost(INSN_COST);
 7254   format %{ "str zr, $mem\t# ptr" %}
 7255 
 7256   ins_encode(aarch64_enc_str0(mem));
 7257 
 7258   ins_pipe(istore_mem);
 7259 %}
 7260 
 7261 // Store Compressed Pointer
 7262 instruct storeN(iRegN src, memory4 mem)
 7263 %{
 7264   match(Set mem (StoreN mem src));
 7265   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7266 
 7267   ins_cost(INSN_COST);
 7268   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7269 
 7270   ins_encode(aarch64_enc_strw(src, mem));
 7271 
 7272   ins_pipe(istore_reg_mem);
 7273 %}
 7274 
 7275 instruct storeImmN0(immN0 zero, memory4 mem)
 7276 %{
 7277   match(Set mem (StoreN mem zero));
 7278   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7279 
 7280   ins_cost(INSN_COST);
 7281   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7282 
 7283   ins_encode(aarch64_enc_strw0(mem));
 7284 
 7285   ins_pipe(istore_mem);
 7286 %}
 7287 
 7288 // Store Float
 7289 instruct storeF(vRegF src, memory4 mem)
 7290 %{
 7291   match(Set mem (StoreF mem src));
 7292   predicate(!needs_releasing_store(n));
 7293 
 7294   ins_cost(INSN_COST);
 7295   format %{ "strs  $src, $mem\t# float" %}
 7296 
 7297   ins_encode( aarch64_enc_strs(src, mem) );
 7298 
 7299   ins_pipe(pipe_class_memory);
 7300 %}
 7301 
 7302 // TODO
 7303 // implement storeImmF0 and storeFImmPacked
 7304 
 7305 // Store Double
 7306 instruct storeD(vRegD src, memory8 mem)
 7307 %{
 7308   match(Set mem (StoreD mem src));
 7309   predicate(!needs_releasing_store(n));
 7310 
 7311   ins_cost(INSN_COST);
 7312   format %{ "strd  $src, $mem\t# double" %}
 7313 
 7314   ins_encode( aarch64_enc_strd(src, mem) );
 7315 
 7316   ins_pipe(pipe_class_memory);
 7317 %}
 7318 
 7319 // Store Compressed Klass Pointer
 7320 instruct storeNKlass(iRegN src, memory4 mem)
 7321 %{
 7322   predicate(!needs_releasing_store(n));
 7323   match(Set mem (StoreNKlass mem src));
 7324 
 7325   ins_cost(INSN_COST);
 7326   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7327 
 7328   ins_encode(aarch64_enc_strw(src, mem));
 7329 
 7330   ins_pipe(istore_reg_mem);
 7331 %}
 7332 
 7333 // TODO
 7334 // implement storeImmD0 and storeDImmPacked
 7335 
 7336 // prefetch instructions
 7337 // Must be safe to execute with invalid address (cannot fault).
 7338 
 7339 instruct prefetchalloc( memory8 mem ) %{
 7340   match(PrefetchAllocation mem);
 7341 
 7342   ins_cost(INSN_COST);
 7343   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7344 
 7345   ins_encode( aarch64_enc_prefetchw(mem) );
 7346 
 7347   ins_pipe(iload_prefetch);
 7348 %}
 7349 
 7350 //  ---------------- volatile loads and stores ----------------
 7351 
 7352 // Load Byte (8 bit signed)
 7353 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7354 %{
 7355   match(Set dst (LoadB mem));
 7356 
 7357   ins_cost(VOLATILE_REF_COST);
 7358   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7359 
 7360   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7361 
 7362   ins_pipe(pipe_serial);
 7363 %}
 7364 
 7365 // Load Byte (8 bit signed) into long
 7366 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7367 %{
 7368   match(Set dst (ConvI2L (LoadB mem)));
 7369 
 7370   ins_cost(VOLATILE_REF_COST);
 7371   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7372 
 7373   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7374 
 7375   ins_pipe(pipe_serial);
 7376 %}
 7377 
 7378 // Load Byte (8 bit unsigned)
 7379 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7380 %{
 7381   match(Set dst (LoadUB mem));
 7382 
 7383   ins_cost(VOLATILE_REF_COST);
 7384   format %{ "ldarb  $dst, $mem\t# byte" %}
 7385 
 7386   ins_encode(aarch64_enc_ldarb(dst, mem));
 7387 
 7388   ins_pipe(pipe_serial);
 7389 %}
 7390 
 7391 // Load Byte (8 bit unsigned) into long
 7392 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7393 %{
 7394   match(Set dst (ConvI2L (LoadUB mem)));
 7395 
 7396   ins_cost(VOLATILE_REF_COST);
 7397   format %{ "ldarb  $dst, $mem\t# byte" %}
 7398 
 7399   ins_encode(aarch64_enc_ldarb(dst, mem));
 7400 
 7401   ins_pipe(pipe_serial);
 7402 %}
 7403 
 7404 // Load Short (16 bit signed)
 7405 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7406 %{
 7407   match(Set dst (LoadS mem));
 7408 
 7409   ins_cost(VOLATILE_REF_COST);
 7410   format %{ "ldarshw  $dst, $mem\t# short" %}
 7411 
 7412   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7413 
 7414   ins_pipe(pipe_serial);
 7415 %}
 7416 
 7417 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7418 %{
 7419   match(Set dst (LoadUS mem));
 7420 
 7421   ins_cost(VOLATILE_REF_COST);
 7422   format %{ "ldarhw  $dst, $mem\t# short" %}
 7423 
 7424   ins_encode(aarch64_enc_ldarhw(dst, mem));
 7425 
 7426   ins_pipe(pipe_serial);
 7427 %}
 7428 
 7429 // Load Short/Char (16 bit unsigned) into long
 7430 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7431 %{
 7432   match(Set dst (ConvI2L (LoadUS mem)));
 7433 
 7434   ins_cost(VOLATILE_REF_COST);
 7435   format %{ "ldarh  $dst, $mem\t# short" %}
 7436 
 7437   ins_encode(aarch64_enc_ldarh(dst, mem));
 7438 
 7439   ins_pipe(pipe_serial);
 7440 %}
 7441 
 7442 // Load Short/Char (16 bit signed) into long
 7443 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7444 %{
 7445   match(Set dst (ConvI2L (LoadS mem)));
 7446 
 7447   ins_cost(VOLATILE_REF_COST);
 7448   format %{ "ldarh  $dst, $mem\t# short" %}
 7449 
 7450   ins_encode(aarch64_enc_ldarsh(dst, mem));
 7451 
 7452   ins_pipe(pipe_serial);
 7453 %}
 7454 
 7455 // Load Integer (32 bit signed)
 7456 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7457 %{
 7458   match(Set dst (LoadI mem));
 7459 
 7460   ins_cost(VOLATILE_REF_COST);
 7461   format %{ "ldarw  $dst, $mem\t# int" %}
 7462 
 7463   ins_encode(aarch64_enc_ldarw(dst, mem));
 7464 
 7465   ins_pipe(pipe_serial);
 7466 %}
 7467 
 7468 // Load Integer (32 bit unsigned) into long
 7469 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 7470 %{
 7471   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7472 
 7473   ins_cost(VOLATILE_REF_COST);
 7474   format %{ "ldarw  $dst, $mem\t# int" %}
 7475 
 7476   ins_encode(aarch64_enc_ldarw(dst, mem));
 7477 
 7478   ins_pipe(pipe_serial);
 7479 %}
 7480 
 7481 // Load Long (64 bit signed)
 7482 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7483 %{
 7484   match(Set dst (LoadL mem));
 7485 
 7486   ins_cost(VOLATILE_REF_COST);
 7487   format %{ "ldar  $dst, $mem\t# int" %}
 7488 
 7489   ins_encode(aarch64_enc_ldar(dst, mem));
 7490 
 7491   ins_pipe(pipe_serial);
 7492 %}
 7493 
 7494 // Load Pointer
 7495 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 7496 %{
 7497   match(Set dst (LoadP mem));
 7498   predicate(n->as_Load()->barrier_data() == 0);
 7499 
 7500   ins_cost(VOLATILE_REF_COST);
 7501   format %{ "ldar  $dst, $mem\t# ptr" %}
 7502 
 7503   ins_encode(aarch64_enc_ldar(dst, mem));
 7504 
 7505   ins_pipe(pipe_serial);
 7506 %}
 7507 
 7508 // Load Compressed Pointer
 7509 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 7510 %{
 7511   match(Set dst (LoadN mem));
 7512   predicate(n->as_Load()->barrier_data() == 0);
 7513 
 7514   ins_cost(VOLATILE_REF_COST);
 7515   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 7516 
 7517   ins_encode(aarch64_enc_ldarw(dst, mem));
 7518 
 7519   ins_pipe(pipe_serial);
 7520 %}
 7521 
 7522 // Load Float
 7523 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 7524 %{
 7525   match(Set dst (LoadF mem));
 7526 
 7527   ins_cost(VOLATILE_REF_COST);
 7528   format %{ "ldars  $dst, $mem\t# float" %}
 7529 
 7530   ins_encode( aarch64_enc_fldars(dst, mem) );
 7531 
 7532   ins_pipe(pipe_serial);
 7533 %}
 7534 
 7535 // Load Double
 7536 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 7537 %{
 7538   match(Set dst (LoadD mem));
 7539 
 7540   ins_cost(VOLATILE_REF_COST);
 7541   format %{ "ldard  $dst, $mem\t# double" %}
 7542 
 7543   ins_encode( aarch64_enc_fldard(dst, mem) );
 7544 
 7545   ins_pipe(pipe_serial);
 7546 %}
 7547 
 7548 // Store Byte
 7549 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7550 %{
 7551   match(Set mem (StoreB mem src));
 7552 
 7553   ins_cost(VOLATILE_REF_COST);
 7554   format %{ "stlrb  $src, $mem\t# byte" %}
 7555 
 7556   ins_encode(aarch64_enc_stlrb(src, mem));
 7557 
 7558   ins_pipe(pipe_class_memory);
 7559 %}
 7560 
 7561 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7562 %{
 7563   match(Set mem (StoreB mem zero));
 7564 
 7565   ins_cost(VOLATILE_REF_COST);
 7566   format %{ "stlrb  zr, $mem\t# byte" %}
 7567 
 7568   ins_encode(aarch64_enc_stlrb0(mem));
 7569 
 7570   ins_pipe(pipe_class_memory);
 7571 %}
 7572 
 7573 // Store Char/Short
 7574 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7575 %{
 7576   match(Set mem (StoreC mem src));
 7577 
 7578   ins_cost(VOLATILE_REF_COST);
 7579   format %{ "stlrh  $src, $mem\t# short" %}
 7580 
 7581   ins_encode(aarch64_enc_stlrh(src, mem));
 7582 
 7583   ins_pipe(pipe_class_memory);
 7584 %}
 7585 
 7586 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7587 %{
 7588   match(Set mem (StoreC mem zero));
 7589 
 7590   ins_cost(VOLATILE_REF_COST);
 7591   format %{ "stlrh  zr, $mem\t# short" %}
 7592 
 7593   ins_encode(aarch64_enc_stlrh0(mem));
 7594 
 7595   ins_pipe(pipe_class_memory);
 7596 %}
 7597 
 7598 // Store Integer
 7599 
 7600 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7601 %{
 7602   match(Set mem(StoreI mem src));
 7603 
 7604   ins_cost(VOLATILE_REF_COST);
 7605   format %{ "stlrw  $src, $mem\t# int" %}
 7606 
 7607   ins_encode(aarch64_enc_stlrw(src, mem));
 7608 
 7609   ins_pipe(pipe_class_memory);
 7610 %}
 7611 
 7612 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7613 %{
 7614   match(Set mem(StoreI mem zero));
 7615 
 7616   ins_cost(VOLATILE_REF_COST);
 7617   format %{ "stlrw  zr, $mem\t# int" %}
 7618 
 7619   ins_encode(aarch64_enc_stlrw0(mem));
 7620 
 7621   ins_pipe(pipe_class_memory);
 7622 %}
 7623 
 7624 // Store Long (64 bit signed)
 7625 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 7626 %{
 7627   match(Set mem (StoreL mem src));
 7628 
 7629   ins_cost(VOLATILE_REF_COST);
 7630   format %{ "stlr  $src, $mem\t# int" %}
 7631 
 7632   ins_encode(aarch64_enc_stlr(src, mem));
 7633 
 7634   ins_pipe(pipe_class_memory);
 7635 %}
 7636 
 7637 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 7638 %{
 7639   match(Set mem (StoreL mem zero));
 7640 
 7641   ins_cost(VOLATILE_REF_COST);
 7642   format %{ "stlr  zr, $mem\t# int" %}
 7643 
 7644   ins_encode(aarch64_enc_stlr0(mem));
 7645 
 7646   ins_pipe(pipe_class_memory);
 7647 %}
 7648 
 7649 // Store Pointer
 7650 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 7651 %{
 7652   match(Set mem (StoreP mem src));
 7653   predicate(n->as_Store()->barrier_data() == 0);
 7654 
 7655   ins_cost(VOLATILE_REF_COST);
 7656   format %{ "stlr  $src, $mem\t# ptr" %}
 7657 
 7658   ins_encode(aarch64_enc_stlr(src, mem));
 7659 
 7660   ins_pipe(pipe_class_memory);
 7661 %}
 7662 
 7663 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 7664 %{
 7665   match(Set mem (StoreP mem zero));
 7666   predicate(n->as_Store()->barrier_data() == 0);
 7667 
 7668   ins_cost(VOLATILE_REF_COST);
 7669   format %{ "stlr  zr, $mem\t# ptr" %}
 7670 
 7671   ins_encode(aarch64_enc_stlr0(mem));
 7672 
 7673   ins_pipe(pipe_class_memory);
 7674 %}
 7675 
 7676 // Store Compressed Pointer
 7677 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 7678 %{
 7679   match(Set mem (StoreN mem src));
 7680   predicate(n->as_Store()->barrier_data() == 0);
 7681 
 7682   ins_cost(VOLATILE_REF_COST);
 7683   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 7684 
 7685   ins_encode(aarch64_enc_stlrw(src, mem));
 7686 
 7687   ins_pipe(pipe_class_memory);
 7688 %}
 7689 
 7690 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 7691 %{
 7692   match(Set mem (StoreN mem zero));
 7693   predicate(n->as_Store()->barrier_data() == 0);
 7694 
 7695   ins_cost(VOLATILE_REF_COST);
 7696   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 7697 
 7698   ins_encode(aarch64_enc_stlrw0(mem));
 7699 
 7700   ins_pipe(pipe_class_memory);
 7701 %}
 7702 
 7703 // Store Float
 7704 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 7705 %{
 7706   match(Set mem (StoreF mem src));
 7707 
 7708   ins_cost(VOLATILE_REF_COST);
 7709   format %{ "stlrs  $src, $mem\t# float" %}
 7710 
 7711   ins_encode( aarch64_enc_fstlrs(src, mem) );
 7712 
 7713   ins_pipe(pipe_class_memory);
 7714 %}
 7715 
 7716 // TODO
 7717 // implement storeImmF0 and storeFImmPacked
 7718 
 7719 // Store Double
 7720 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 7721 %{
 7722   match(Set mem (StoreD mem src));
 7723 
 7724   ins_cost(VOLATILE_REF_COST);
 7725   format %{ "stlrd  $src, $mem\t# double" %}
 7726 
 7727   ins_encode( aarch64_enc_fstlrd(src, mem) );
 7728 
 7729   ins_pipe(pipe_class_memory);
 7730 %}
 7731 
 7732 //  ---------------- end of volatile loads and stores ----------------
 7733 
 7734 instruct cacheWB(indirect addr)
 7735 %{
 7736   predicate(VM_Version::supports_data_cache_line_flush());
 7737   match(CacheWB addr);
 7738 
 7739   ins_cost(100);
 7740   format %{"cache wb $addr" %}
 7741   ins_encode %{
 7742     assert($addr->index_position() < 0, "should be");
 7743     assert($addr$$disp == 0, "should be");
 7744     __ cache_wb(Address($addr$$base$$Register, 0));
 7745   %}
 7746   ins_pipe(pipe_slow); // XXX
 7747 %}
 7748 
 7749 instruct cacheWBPreSync()
 7750 %{
 7751   predicate(VM_Version::supports_data_cache_line_flush());
 7752   match(CacheWBPreSync);
 7753 
 7754   ins_cost(100);
 7755   format %{"cache wb presync" %}
 7756   ins_encode %{
 7757     __ cache_wbsync(true);
 7758   %}
 7759   ins_pipe(pipe_slow); // XXX
 7760 %}
 7761 
 7762 instruct cacheWBPostSync()
 7763 %{
 7764   predicate(VM_Version::supports_data_cache_line_flush());
 7765   match(CacheWBPostSync);
 7766 
 7767   ins_cost(100);
 7768   format %{"cache wb postsync" %}
 7769   ins_encode %{
 7770     __ cache_wbsync(false);
 7771   %}
 7772   ins_pipe(pipe_slow); // XXX
 7773 %}
 7774 
 7775 // ============================================================================
 7776 // BSWAP Instructions
 7777 
 7778 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 7779   match(Set dst (ReverseBytesI src));
 7780 
 7781   ins_cost(INSN_COST);
 7782   format %{ "revw  $dst, $src" %}
 7783 
 7784   ins_encode %{
 7785     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 7786   %}
 7787 
 7788   ins_pipe(ialu_reg);
 7789 %}
 7790 
 7791 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 7792   match(Set dst (ReverseBytesL src));
 7793 
 7794   ins_cost(INSN_COST);
 7795   format %{ "rev  $dst, $src" %}
 7796 
 7797   ins_encode %{
 7798     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 7799   %}
 7800 
 7801   ins_pipe(ialu_reg);
 7802 %}
 7803 
 7804 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 7805   match(Set dst (ReverseBytesUS src));
 7806 
 7807   ins_cost(INSN_COST);
 7808   format %{ "rev16w  $dst, $src" %}
 7809 
 7810   ins_encode %{
 7811     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 7812   %}
 7813 
 7814   ins_pipe(ialu_reg);
 7815 %}
 7816 
 7817 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 7818   match(Set dst (ReverseBytesS src));
 7819 
 7820   ins_cost(INSN_COST);
 7821   format %{ "rev16w  $dst, $src\n\t"
 7822             "sbfmw $dst, $dst, #0, #15" %}
 7823 
 7824   ins_encode %{
 7825     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 7826     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 7827   %}
 7828 
 7829   ins_pipe(ialu_reg);
 7830 %}
 7831 
 7832 // ============================================================================
 7833 // Zero Count Instructions
 7834 
 7835 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 7836   match(Set dst (CountLeadingZerosI src));
 7837 
 7838   ins_cost(INSN_COST);
 7839   format %{ "clzw  $dst, $src" %}
 7840   ins_encode %{
 7841     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 7842   %}
 7843 
 7844   ins_pipe(ialu_reg);
 7845 %}
 7846 
 7847 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 7848   match(Set dst (CountLeadingZerosL src));
 7849 
 7850   ins_cost(INSN_COST);
 7851   format %{ "clz   $dst, $src" %}
 7852   ins_encode %{
 7853     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 7854   %}
 7855 
 7856   ins_pipe(ialu_reg);
 7857 %}
 7858 
 7859 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 7860   match(Set dst (CountTrailingZerosI src));
 7861 
 7862   ins_cost(INSN_COST * 2);
 7863   format %{ "rbitw  $dst, $src\n\t"
 7864             "clzw   $dst, $dst" %}
 7865   ins_encode %{
 7866     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 7867     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 7868   %}
 7869 
 7870   ins_pipe(ialu_reg);
 7871 %}
 7872 
 7873 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 7874   match(Set dst (CountTrailingZerosL src));
 7875 
 7876   ins_cost(INSN_COST * 2);
 7877   format %{ "rbit   $dst, $src\n\t"
 7878             "clz    $dst, $dst" %}
 7879   ins_encode %{
 7880     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 7881     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 7882   %}
 7883 
 7884   ins_pipe(ialu_reg);
 7885 %}
 7886 
 7887 //---------- Population Count Instructions -------------------------------------
 7888 //
 7889 
 7890 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 7891   match(Set dst (PopCountI src));
 7892   effect(TEMP tmp);
 7893   ins_cost(INSN_COST * 13);
 7894 
 7895   format %{ "fmovs  $tmp, $src\t# vector (1S)\n\t"
 7896             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7897             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7898             "mov    $dst, $tmp\t# vector (1D)" %}
 7899   ins_encode %{
 7900     __ fmovs($tmp$$FloatRegister, $src$$Register);
 7901     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7902     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7903     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7904   %}
 7905 
 7906   ins_pipe(pipe_class_default);
 7907 %}
 7908 
 7909 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 7910   match(Set dst (PopCountI (LoadI mem)));
 7911   effect(TEMP tmp);
 7912   ins_cost(INSN_COST * 13);
 7913 
 7914   format %{ "ldrs   $tmp, $mem\n\t"
 7915             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7916             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7917             "mov    $dst, $tmp\t# vector (1D)" %}
 7918   ins_encode %{
 7919     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 7920     loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 7921               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 7922     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7923     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7924     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7925   %}
 7926 
 7927   ins_pipe(pipe_class_default);
 7928 %}
 7929 
 7930 // Note: Long.bitCount(long) returns an int.
 7931 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 7932   match(Set dst (PopCountL src));
 7933   effect(TEMP tmp);
 7934   ins_cost(INSN_COST * 13);
 7935 
 7936   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 7937             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7938             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7939             "mov    $dst, $tmp\t# vector (1D)" %}
 7940   ins_encode %{
 7941     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 7942     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7943     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7944     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7945   %}
 7946 
 7947   ins_pipe(pipe_class_default);
 7948 %}
 7949 
 7950 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 7951   match(Set dst (PopCountL (LoadL mem)));
 7952   effect(TEMP tmp);
 7953   ins_cost(INSN_COST * 13);
 7954 
 7955   format %{ "ldrd   $tmp, $mem\n\t"
 7956             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7957             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7958             "mov    $dst, $tmp\t# vector (1D)" %}
 7959   ins_encode %{
 7960     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 7961     loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 7962               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 7963     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7964     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7965     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7966   %}
 7967 
 7968   ins_pipe(pipe_class_default);
 7969 %}
 7970 
 7971 // ============================================================================
 7972 // VerifyVectorAlignment Instruction
 7973 
 7974 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
 7975   match(Set addr (VerifyVectorAlignment addr mask));
 7976   effect(KILL cr);
 7977   format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
 7978   ins_encode %{
 7979     Label Lskip;
 7980     // check if masked bits of addr are zero
 7981     __ tst($addr$$Register, $mask$$constant);
 7982     __ br(Assembler::EQ, Lskip);
 7983     __ stop("verify_vector_alignment found a misaligned vector memory access");
 7984     __ bind(Lskip);
 7985   %}
 7986   ins_pipe(pipe_slow);
 7987 %}
 7988 
 7989 // ============================================================================
 7990 // MemBar Instruction
 7991 
 7992 instruct load_fence() %{
 7993   match(LoadFence);
 7994   ins_cost(VOLATILE_REF_COST);
 7995 
 7996   format %{ "load_fence" %}
 7997 
 7998   ins_encode %{
 7999     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8000   %}
 8001   ins_pipe(pipe_serial);
 8002 %}
 8003 
 8004 instruct unnecessary_membar_acquire() %{
 8005   predicate(unnecessary_acquire(n));
 8006   match(MemBarAcquire);
 8007   ins_cost(0);
 8008 
 8009   format %{ "membar_acquire (elided)" %}
 8010 
 8011   ins_encode %{
 8012     __ block_comment("membar_acquire (elided)");
 8013   %}
 8014 
 8015   ins_pipe(pipe_class_empty);
 8016 %}
 8017 
 8018 instruct membar_acquire() %{
 8019   match(MemBarAcquire);
 8020   ins_cost(VOLATILE_REF_COST);
 8021 
 8022   format %{ "membar_acquire\n\t"
 8023             "dmb ishld" %}
 8024 
 8025   ins_encode %{
 8026     __ block_comment("membar_acquire");
 8027     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8028   %}
 8029 
 8030   ins_pipe(pipe_serial);
 8031 %}
 8032 
 8033 
 8034 instruct membar_acquire_lock() %{
 8035   match(MemBarAcquireLock);
 8036   ins_cost(VOLATILE_REF_COST);
 8037 
 8038   format %{ "membar_acquire_lock (elided)" %}
 8039 
 8040   ins_encode %{
 8041     __ block_comment("membar_acquire_lock (elided)");
 8042   %}
 8043 
 8044   ins_pipe(pipe_serial);
 8045 %}
 8046 
 8047 instruct store_fence() %{
 8048   match(StoreFence);
 8049   ins_cost(VOLATILE_REF_COST);
 8050 
 8051   format %{ "store_fence" %}
 8052 
 8053   ins_encode %{
 8054     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8055   %}
 8056   ins_pipe(pipe_serial);
 8057 %}
 8058 
 8059 instruct unnecessary_membar_release() %{
 8060   predicate(unnecessary_release(n));
 8061   match(MemBarRelease);
 8062   ins_cost(0);
 8063 
 8064   format %{ "membar_release (elided)" %}
 8065 
 8066   ins_encode %{
 8067     __ block_comment("membar_release (elided)");
 8068   %}
 8069   ins_pipe(pipe_serial);
 8070 %}
 8071 
 8072 instruct membar_release() %{
 8073   match(MemBarRelease);
 8074   ins_cost(VOLATILE_REF_COST);
 8075 
 8076   format %{ "membar_release\n\t"
 8077             "dmb ishst\n\tdmb ishld" %}
 8078 
 8079   ins_encode %{
 8080     __ block_comment("membar_release");
 8081     // These will be merged if AlwaysMergeDMB is enabled.
 8082     __ membar(Assembler::StoreStore);
 8083     __ membar(Assembler::LoadStore);
 8084   %}
 8085   ins_pipe(pipe_serial);
 8086 %}
 8087 
 8088 instruct membar_storestore() %{
 8089   match(MemBarStoreStore);
 8090   match(StoreStoreFence);
 8091   ins_cost(VOLATILE_REF_COST);
 8092 
 8093   format %{ "MEMBAR-store-store" %}
 8094 
 8095   ins_encode %{
 8096     __ membar(Assembler::StoreStore);
 8097   %}
 8098   ins_pipe(pipe_serial);
 8099 %}
 8100 
 8101 instruct membar_release_lock() %{
 8102   match(MemBarReleaseLock);
 8103   ins_cost(VOLATILE_REF_COST);
 8104 
 8105   format %{ "membar_release_lock (elided)" %}
 8106 
 8107   ins_encode %{
 8108     __ block_comment("membar_release_lock (elided)");
 8109   %}
 8110 
 8111   ins_pipe(pipe_serial);
 8112 %}
 8113 
 8114 instruct unnecessary_membar_volatile() %{
 8115   predicate(unnecessary_volatile(n));
 8116   match(MemBarVolatile);
 8117   ins_cost(0);
 8118 
 8119   format %{ "membar_volatile (elided)" %}
 8120 
 8121   ins_encode %{
 8122     __ block_comment("membar_volatile (elided)");
 8123   %}
 8124 
 8125   ins_pipe(pipe_serial);
 8126 %}
 8127 
 8128 instruct membar_volatile() %{
 8129   match(MemBarVolatile);
 8130   ins_cost(VOLATILE_REF_COST*100);
 8131 
 8132   format %{ "membar_volatile\n\t"
 8133              "dmb ish"%}
 8134 
 8135   ins_encode %{
 8136     __ block_comment("membar_volatile");
 8137     __ membar(Assembler::StoreLoad);
 8138   %}
 8139 
 8140   ins_pipe(pipe_serial);
 8141 %}
 8142 
 8143 // ============================================================================
 8144 // Cast/Convert Instructions
 8145 
 8146 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8147   match(Set dst (CastX2P src));
 8148 
 8149   ins_cost(INSN_COST);
 8150   format %{ "mov $dst, $src\t# long -> ptr" %}
 8151 
 8152   ins_encode %{
 8153     if ($dst$$reg != $src$$reg) {
 8154       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8155     }
 8156   %}
 8157 
 8158   ins_pipe(ialu_reg);
 8159 %}
 8160 
 8161 instruct castI2N(iRegNNoSp dst, iRegI src) %{
 8162   match(Set dst (CastI2N src));
 8163 
 8164   ins_cost(INSN_COST);
 8165   format %{ "mov $dst, $src\t# int -> narrow ptr" %}
 8166 
 8167   ins_encode %{
 8168     if ($dst$$reg != $src$$reg) {
 8169       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8170     }
 8171   %}
 8172 
 8173   ins_pipe(ialu_reg);
 8174 %}
 8175 
 8176 instruct castN2X(iRegLNoSp dst, iRegN src) %{
 8177   match(Set dst (CastP2X src));
 8178 
 8179   ins_cost(INSN_COST);
 8180   format %{ "mov $dst, $src\t# ptr -> long" %}
 8181 
 8182   ins_encode %{
 8183     if ($dst$$reg != $src$$reg) {
 8184       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8185     }
 8186   %}
 8187 
 8188   ins_pipe(ialu_reg);
 8189 %}
 8190 
 8191 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8192   match(Set dst (CastP2X src));
 8193 
 8194   ins_cost(INSN_COST);
 8195   format %{ "mov $dst, $src\t# ptr -> long" %}
 8196 
 8197   ins_encode %{
 8198     if ($dst$$reg != $src$$reg) {
 8199       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8200     }
 8201   %}
 8202 
 8203   ins_pipe(ialu_reg);
 8204 %}
 8205 
 8206 // Convert oop into int for vectors alignment masking
 8207 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8208   match(Set dst (ConvL2I (CastP2X src)));
 8209 
 8210   ins_cost(INSN_COST);
 8211   format %{ "movw $dst, $src\t# ptr -> int" %}
 8212   ins_encode %{
 8213     __ movw($dst$$Register, $src$$Register);
 8214   %}
 8215 
 8216   ins_pipe(ialu_reg);
 8217 %}
 8218 
 8219 // Convert compressed oop into int for vectors alignment masking
 8220 // in case of 32bit oops (heap < 4Gb).
 8221 instruct convN2I(iRegINoSp dst, iRegN src)
 8222 %{
 8223   predicate(CompressedOops::shift() == 0);
 8224   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8225 
 8226   ins_cost(INSN_COST);
 8227   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8228   ins_encode %{
 8229     __ movw($dst$$Register, $src$$Register);
 8230   %}
 8231 
 8232   ins_pipe(ialu_reg);
 8233 %}
 8234 
 8235 
 8236 // Convert oop pointer into compressed form
 8237 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8238   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8239   match(Set dst (EncodeP src));
 8240   effect(KILL cr);
 8241   ins_cost(INSN_COST * 3);
 8242   format %{ "encode_heap_oop $dst, $src" %}
 8243   ins_encode %{
 8244     Register s = $src$$Register;
 8245     Register d = $dst$$Register;
 8246     __ encode_heap_oop(d, s);
 8247   %}
 8248   ins_pipe(ialu_reg);
 8249 %}
 8250 
 8251 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8252   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8253   match(Set dst (EncodeP src));
 8254   ins_cost(INSN_COST * 3);
 8255   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8256   ins_encode %{
 8257     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8258   %}
 8259   ins_pipe(ialu_reg);
 8260 %}
 8261 
 8262 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8263   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8264             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8265   match(Set dst (DecodeN src));
 8266   ins_cost(INSN_COST * 3);
 8267   format %{ "decode_heap_oop $dst, $src" %}
 8268   ins_encode %{
 8269     Register s = $src$$Register;
 8270     Register d = $dst$$Register;
 8271     __ decode_heap_oop(d, s);
 8272   %}
 8273   ins_pipe(ialu_reg);
 8274 %}
 8275 
 8276 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8277   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8278             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8279   match(Set dst (DecodeN src));
 8280   ins_cost(INSN_COST * 3);
 8281   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8282   ins_encode %{
 8283     Register s = $src$$Register;
 8284     Register d = $dst$$Register;
 8285     __ decode_heap_oop_not_null(d, s);
 8286   %}
 8287   ins_pipe(ialu_reg);
 8288 %}
 8289 
 8290 // n.b. AArch64 implementations of encode_klass_not_null and
 8291 // decode_klass_not_null do not modify the flags register so, unlike
 8292 // Intel, we don't kill CR as a side effect here
 8293 
 8294 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8295   match(Set dst (EncodePKlass src));
 8296 
 8297   ins_cost(INSN_COST * 3);
 8298   format %{ "encode_klass_not_null $dst,$src" %}
 8299 
 8300   ins_encode %{
 8301     Register src_reg = as_Register($src$$reg);
 8302     Register dst_reg = as_Register($dst$$reg);
 8303     __ encode_klass_not_null(dst_reg, src_reg);
 8304   %}
 8305 
 8306    ins_pipe(ialu_reg);
 8307 %}
 8308 
 8309 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8310   match(Set dst (DecodeNKlass src));
 8311 
 8312   ins_cost(INSN_COST * 3);
 8313   format %{ "decode_klass_not_null $dst,$src" %}
 8314 
 8315   ins_encode %{
 8316     Register src_reg = as_Register($src$$reg);
 8317     Register dst_reg = as_Register($dst$$reg);
 8318     if (dst_reg != src_reg) {
 8319       __ decode_klass_not_null(dst_reg, src_reg);
 8320     } else {
 8321       __ decode_klass_not_null(dst_reg);
 8322     }
 8323   %}
 8324 
 8325    ins_pipe(ialu_reg);
 8326 %}
 8327 
 8328 instruct checkCastPP(iRegPNoSp dst)
 8329 %{
 8330   match(Set dst (CheckCastPP dst));
 8331 
 8332   size(0);
 8333   format %{ "# checkcastPP of $dst" %}
 8334   ins_encode(/* empty encoding */);
 8335   ins_pipe(pipe_class_empty);
 8336 %}
 8337 
 8338 instruct castPP(iRegPNoSp dst)
 8339 %{
 8340   match(Set dst (CastPP dst));
 8341 
 8342   size(0);
 8343   format %{ "# castPP of $dst" %}
 8344   ins_encode(/* empty encoding */);
 8345   ins_pipe(pipe_class_empty);
 8346 %}
 8347 
 8348 instruct castII(iRegI dst)
 8349 %{
 8350   predicate(VerifyConstraintCasts == 0);
 8351   match(Set dst (CastII dst));
 8352 
 8353   size(0);
 8354   format %{ "# castII of $dst" %}
 8355   ins_encode(/* empty encoding */);
 8356   ins_cost(0);
 8357   ins_pipe(pipe_class_empty);
 8358 %}
 8359 
 8360 instruct castII_checked(iRegI dst, rFlagsReg cr)
 8361 %{
 8362   predicate(VerifyConstraintCasts > 0);
 8363   match(Set dst (CastII dst));
 8364   effect(KILL cr);
 8365 
 8366   format %{ "# castII_checked of $dst" %}
 8367   ins_encode %{
 8368     __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
 8369   %}
 8370   ins_pipe(pipe_slow);
 8371 %}
 8372 
 8373 instruct castLL(iRegL dst)
 8374 %{
 8375   predicate(VerifyConstraintCasts == 0);
 8376   match(Set dst (CastLL dst));
 8377 
 8378   size(0);
 8379   format %{ "# castLL of $dst" %}
 8380   ins_encode(/* empty encoding */);
 8381   ins_cost(0);
 8382   ins_pipe(pipe_class_empty);
 8383 %}
 8384 
 8385 instruct castLL_checked(iRegL dst, rFlagsReg cr)
 8386 %{
 8387   predicate(VerifyConstraintCasts > 0);
 8388   match(Set dst (CastLL dst));
 8389   effect(KILL cr);
 8390 
 8391   format %{ "# castLL_checked of $dst" %}
 8392   ins_encode %{
 8393     __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
 8394   %}
 8395   ins_pipe(pipe_slow);
 8396 %}
 8397 
 8398 instruct castHH(vRegF dst)
 8399 %{
 8400   match(Set dst (CastHH dst));
 8401   size(0);
 8402   format %{ "# castHH of $dst" %}
 8403   ins_encode(/* empty encoding */);
 8404   ins_cost(0);
 8405   ins_pipe(pipe_class_empty);
 8406 %}
 8407 
 8408 instruct castFF(vRegF dst)
 8409 %{
 8410   match(Set dst (CastFF dst));
 8411 
 8412   size(0);
 8413   format %{ "# castFF of $dst" %}
 8414   ins_encode(/* empty encoding */);
 8415   ins_cost(0);
 8416   ins_pipe(pipe_class_empty);
 8417 %}
 8418 
 8419 instruct castDD(vRegD dst)
 8420 %{
 8421   match(Set dst (CastDD dst));
 8422 
 8423   size(0);
 8424   format %{ "# castDD of $dst" %}
 8425   ins_encode(/* empty encoding */);
 8426   ins_cost(0);
 8427   ins_pipe(pipe_class_empty);
 8428 %}
 8429 
 8430 instruct castVV(vReg dst)
 8431 %{
 8432   match(Set dst (CastVV dst));
 8433 
 8434   size(0);
 8435   format %{ "# castVV of $dst" %}
 8436   ins_encode(/* empty encoding */);
 8437   ins_cost(0);
 8438   ins_pipe(pipe_class_empty);
 8439 %}
 8440 
 8441 instruct castVVMask(pRegGov dst)
 8442 %{
 8443   match(Set dst (CastVV dst));
 8444 
 8445   size(0);
 8446   format %{ "# castVV of $dst" %}
 8447   ins_encode(/* empty encoding */);
 8448   ins_cost(0);
 8449   ins_pipe(pipe_class_empty);
 8450 %}
 8451 
 8452 // ============================================================================
 8453 // Atomic operation instructions
 8454 //
 8455 
 8456 // standard CompareAndSwapX when we are using barriers
 8457 // these have higher priority than the rules selected by a predicate
 8458 
 8459 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 8460 // can't match them
 8461 
 8462 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8463 
 8464   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8465   ins_cost(2 * VOLATILE_REF_COST);
 8466 
 8467   effect(KILL cr);
 8468 
 8469   format %{
 8470     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8471     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8472   %}
 8473 
 8474   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 8475             aarch64_enc_cset_eq(res));
 8476 
 8477   ins_pipe(pipe_slow);
 8478 %}
 8479 
 8480 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8481 
 8482   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8483   ins_cost(2 * VOLATILE_REF_COST);
 8484 
 8485   effect(KILL cr);
 8486 
 8487   format %{
 8488     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8489     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8490   %}
 8491 
 8492   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 8493             aarch64_enc_cset_eq(res));
 8494 
 8495   ins_pipe(pipe_slow);
 8496 %}
 8497 
 8498 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8499 
 8500   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8501   ins_cost(2 * VOLATILE_REF_COST);
 8502 
 8503   effect(KILL cr);
 8504 
 8505  format %{
 8506     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8507     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8508  %}
 8509 
 8510  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8511             aarch64_enc_cset_eq(res));
 8512 
 8513   ins_pipe(pipe_slow);
 8514 %}
 8515 
 8516 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8517 
 8518   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8519   ins_cost(2 * VOLATILE_REF_COST);
 8520 
 8521   effect(KILL cr);
 8522 
 8523  format %{
 8524     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8525     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8526  %}
 8527 
 8528  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8529             aarch64_enc_cset_eq(res));
 8530 
 8531   ins_pipe(pipe_slow);
 8532 %}
 8533 
 8534 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8535 
 8536   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8537   predicate(n->as_LoadStore()->barrier_data() == 0);
 8538   ins_cost(2 * VOLATILE_REF_COST);
 8539 
 8540   effect(KILL cr);
 8541 
 8542  format %{
 8543     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8544     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8545  %}
 8546 
 8547  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8548             aarch64_enc_cset_eq(res));
 8549 
 8550   ins_pipe(pipe_slow);
 8551 %}
 8552 
 8553 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8554 
 8555   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8556   predicate(n->as_LoadStore()->barrier_data() == 0);
 8557   ins_cost(2 * VOLATILE_REF_COST);
 8558 
 8559   effect(KILL cr);
 8560 
 8561  format %{
 8562     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8563     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8564  %}
 8565 
 8566  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8567             aarch64_enc_cset_eq(res));
 8568 
 8569   ins_pipe(pipe_slow);
 8570 %}
 8571 
 8572 // alternative CompareAndSwapX when we are eliding barriers
 8573 
 8574 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8575 
 8576   predicate(needs_acquiring_load_exclusive(n));
 8577   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8578   ins_cost(VOLATILE_REF_COST);
 8579 
 8580   effect(KILL cr);
 8581 
 8582   format %{
 8583     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8584     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8585   %}
 8586 
 8587   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 8588             aarch64_enc_cset_eq(res));
 8589 
 8590   ins_pipe(pipe_slow);
 8591 %}
 8592 
 8593 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8594 
 8595   predicate(needs_acquiring_load_exclusive(n));
 8596   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8597   ins_cost(VOLATILE_REF_COST);
 8598 
 8599   effect(KILL cr);
 8600 
 8601   format %{
 8602     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8603     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8604   %}
 8605 
 8606   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 8607             aarch64_enc_cset_eq(res));
 8608 
 8609   ins_pipe(pipe_slow);
 8610 %}
 8611 
 8612 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8613 
 8614   predicate(needs_acquiring_load_exclusive(n));
 8615   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8616   ins_cost(VOLATILE_REF_COST);
 8617 
 8618   effect(KILL cr);
 8619 
 8620  format %{
 8621     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8622     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8623  %}
 8624 
 8625  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8626             aarch64_enc_cset_eq(res));
 8627 
 8628   ins_pipe(pipe_slow);
 8629 %}
 8630 
 8631 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8632 
 8633   predicate(needs_acquiring_load_exclusive(n));
 8634   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8635   ins_cost(VOLATILE_REF_COST);
 8636 
 8637   effect(KILL cr);
 8638 
 8639  format %{
 8640     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8641     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8642  %}
 8643 
 8644  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8645             aarch64_enc_cset_eq(res));
 8646 
 8647   ins_pipe(pipe_slow);
 8648 %}
 8649 
 8650 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8651 
 8652   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8653   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8654   ins_cost(VOLATILE_REF_COST);
 8655 
 8656   effect(KILL cr);
 8657 
 8658  format %{
 8659     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8660     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8661  %}
 8662 
 8663  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8664             aarch64_enc_cset_eq(res));
 8665 
 8666   ins_pipe(pipe_slow);
 8667 %}
 8668 
 8669 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8670 
 8671   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 8672   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8673   ins_cost(VOLATILE_REF_COST);
 8674 
 8675   effect(KILL cr);
 8676 
 8677  format %{
 8678     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8679     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8680  %}
 8681 
 8682  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8683             aarch64_enc_cset_eq(res));
 8684 
 8685   ins_pipe(pipe_slow);
 8686 %}
 8687 
 8688 
 8689 // ---------------------------------------------------------------------
 8690 
 8691 // BEGIN This section of the file is automatically generated. Do not edit --------------
 8692 
 8693 // Sundry CAS operations.  Note that release is always true,
 8694 // regardless of the memory ordering of the CAS.  This is because we
 8695 // need the volatile case to be sequentially consistent but there is
 8696 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 8697 // can't check the type of memory ordering here, so we always emit a
 8698 // STLXR.
 8699 
 8700 // This section is generated from cas.m4
 8701 
 8702 
 8703 // This pattern is generated automatically from cas.m4.
 8704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8705 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8706   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8707   ins_cost(2 * VOLATILE_REF_COST);
 8708   effect(TEMP_DEF res, KILL cr);
 8709   format %{
 8710     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8711   %}
 8712   ins_encode %{
 8713     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8714                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8715                /*weak*/ false, $res$$Register);
 8716     __ sxtbw($res$$Register, $res$$Register);
 8717   %}
 8718   ins_pipe(pipe_slow);
 8719 %}
 8720 
 8721 // This pattern is generated automatically from cas.m4.
 8722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8723 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8724   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 8725   ins_cost(2 * VOLATILE_REF_COST);
 8726   effect(TEMP_DEF res, KILL cr);
 8727   format %{
 8728     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8729   %}
 8730   ins_encode %{
 8731     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8732                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 8733                /*weak*/ false, $res$$Register);
 8734     __ sxthw($res$$Register, $res$$Register);
 8735   %}
 8736   ins_pipe(pipe_slow);
 8737 %}
 8738 
 8739 // This pattern is generated automatically from cas.m4.
 8740 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8741 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8742   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 8743   ins_cost(2 * VOLATILE_REF_COST);
 8744   effect(TEMP_DEF res, KILL cr);
 8745   format %{
 8746     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8747   %}
 8748   ins_encode %{
 8749     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8750                Assembler::word, /*acquire*/ false, /*release*/ true,
 8751                /*weak*/ false, $res$$Register);
 8752   %}
 8753   ins_pipe(pipe_slow);
 8754 %}
 8755 
 8756 // This pattern is generated automatically from cas.m4.
 8757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8758 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8759   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 8760   ins_cost(2 * VOLATILE_REF_COST);
 8761   effect(TEMP_DEF res, KILL cr);
 8762   format %{
 8763     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8764   %}
 8765   ins_encode %{
 8766     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8767                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8768                /*weak*/ false, $res$$Register);
 8769   %}
 8770   ins_pipe(pipe_slow);
 8771 %}
 8772 
 8773 // This pattern is generated automatically from cas.m4.
 8774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8775 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8776   predicate(n->as_LoadStore()->barrier_data() == 0);
 8777   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 8778   ins_cost(2 * VOLATILE_REF_COST);
 8779   effect(TEMP_DEF res, KILL cr);
 8780   format %{
 8781     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8782   %}
 8783   ins_encode %{
 8784     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8785                Assembler::word, /*acquire*/ false, /*release*/ true,
 8786                /*weak*/ false, $res$$Register);
 8787   %}
 8788   ins_pipe(pipe_slow);
 8789 %}
 8790 
 8791 // This pattern is generated automatically from cas.m4.
 8792 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8793 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8794   predicate(n->as_LoadStore()->barrier_data() == 0);
 8795   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 8796   ins_cost(2 * VOLATILE_REF_COST);
 8797   effect(TEMP_DEF res, KILL cr);
 8798   format %{
 8799     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8800   %}
 8801   ins_encode %{
 8802     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8803                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8804                /*weak*/ false, $res$$Register);
 8805   %}
 8806   ins_pipe(pipe_slow);
 8807 %}
 8808 
 8809 // This pattern is generated automatically from cas.m4.
 8810 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8811 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8812   predicate(needs_acquiring_load_exclusive(n));
 8813   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8814   ins_cost(VOLATILE_REF_COST);
 8815   effect(TEMP_DEF res, KILL cr);
 8816   format %{
 8817     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8818   %}
 8819   ins_encode %{
 8820     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8821                Assembler::byte, /*acquire*/ true, /*release*/ true,
 8822                /*weak*/ false, $res$$Register);
 8823     __ sxtbw($res$$Register, $res$$Register);
 8824   %}
 8825   ins_pipe(pipe_slow);
 8826 %}
 8827 
 8828 // This pattern is generated automatically from cas.m4.
 8829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8830 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8831   predicate(needs_acquiring_load_exclusive(n));
 8832   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 8833   ins_cost(VOLATILE_REF_COST);
 8834   effect(TEMP_DEF res, KILL cr);
 8835   format %{
 8836     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8837   %}
 8838   ins_encode %{
 8839     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8840                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 8841                /*weak*/ false, $res$$Register);
 8842     __ sxthw($res$$Register, $res$$Register);
 8843   %}
 8844   ins_pipe(pipe_slow);
 8845 %}
 8846 
 8847 // This pattern is generated automatically from cas.m4.
 8848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8849 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8850   predicate(needs_acquiring_load_exclusive(n));
 8851   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 8852   ins_cost(VOLATILE_REF_COST);
 8853   effect(TEMP_DEF res, KILL cr);
 8854   format %{
 8855     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8856   %}
 8857   ins_encode %{
 8858     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8859                Assembler::word, /*acquire*/ true, /*release*/ true,
 8860                /*weak*/ false, $res$$Register);
 8861   %}
 8862   ins_pipe(pipe_slow);
 8863 %}
 8864 
 8865 // This pattern is generated automatically from cas.m4.
 8866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8867 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8868   predicate(needs_acquiring_load_exclusive(n));
 8869   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 8870   ins_cost(VOLATILE_REF_COST);
 8871   effect(TEMP_DEF res, KILL cr);
 8872   format %{
 8873     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8874   %}
 8875   ins_encode %{
 8876     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8877                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8878                /*weak*/ false, $res$$Register);
 8879   %}
 8880   ins_pipe(pipe_slow);
 8881 %}
 8882 
 8883 // This pattern is generated automatically from cas.m4.
 8884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8885 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8886   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 8887   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 8888   ins_cost(VOLATILE_REF_COST);
 8889   effect(TEMP_DEF res, KILL cr);
 8890   format %{
 8891     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8892   %}
 8893   ins_encode %{
 8894     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8895                Assembler::word, /*acquire*/ true, /*release*/ true,
 8896                /*weak*/ false, $res$$Register);
 8897   %}
 8898   ins_pipe(pipe_slow);
 8899 %}
 8900 
 8901 // This pattern is generated automatically from cas.m4.
 8902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8903 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8904   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8905   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 8906   ins_cost(VOLATILE_REF_COST);
 8907   effect(TEMP_DEF res, KILL cr);
 8908   format %{
 8909     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8910   %}
 8911   ins_encode %{
 8912     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8913                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8914                /*weak*/ false, $res$$Register);
 8915   %}
 8916   ins_pipe(pipe_slow);
 8917 %}
 8918 
 8919 // This pattern is generated automatically from cas.m4.
 8920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8921 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8922   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 8923   ins_cost(2 * VOLATILE_REF_COST);
 8924   effect(KILL cr);
 8925   format %{
 8926     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8927     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8928   %}
 8929   ins_encode %{
 8930     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8931                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8932                /*weak*/ true, noreg);
 8933     __ csetw($res$$Register, Assembler::EQ);
 8934   %}
 8935   ins_pipe(pipe_slow);
 8936 %}
 8937 
 8938 // This pattern is generated automatically from cas.m4.
 8939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8940 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8941   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 8942   ins_cost(2 * VOLATILE_REF_COST);
 8943   effect(KILL cr);
 8944   format %{
 8945     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8946     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8947   %}
 8948   ins_encode %{
 8949     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8950                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 8951                /*weak*/ true, noreg);
 8952     __ csetw($res$$Register, Assembler::EQ);
 8953   %}
 8954   ins_pipe(pipe_slow);
 8955 %}
 8956 
 8957 // This pattern is generated automatically from cas.m4.
 8958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8959 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8960   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 8961   ins_cost(2 * VOLATILE_REF_COST);
 8962   effect(KILL cr);
 8963   format %{
 8964     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8965     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8966   %}
 8967   ins_encode %{
 8968     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8969                Assembler::word, /*acquire*/ false, /*release*/ true,
 8970                /*weak*/ true, noreg);
 8971     __ csetw($res$$Register, Assembler::EQ);
 8972   %}
 8973   ins_pipe(pipe_slow);
 8974 %}
 8975 
 8976 // This pattern is generated automatically from cas.m4.
 8977 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8978 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8979   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 8980   ins_cost(2 * VOLATILE_REF_COST);
 8981   effect(KILL cr);
 8982   format %{
 8983     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8984     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8985   %}
 8986   ins_encode %{
 8987     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8988                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8989                /*weak*/ true, noreg);
 8990     __ csetw($res$$Register, Assembler::EQ);
 8991   %}
 8992   ins_pipe(pipe_slow);
 8993 %}
 8994 
 8995 // This pattern is generated automatically from cas.m4.
 8996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8997 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8998   predicate(n->as_LoadStore()->barrier_data() == 0);
 8999   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9000   ins_cost(2 * VOLATILE_REF_COST);
 9001   effect(KILL cr);
 9002   format %{
 9003     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9004     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9005   %}
 9006   ins_encode %{
 9007     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9008                Assembler::word, /*acquire*/ false, /*release*/ true,
 9009                /*weak*/ true, noreg);
 9010     __ csetw($res$$Register, Assembler::EQ);
 9011   %}
 9012   ins_pipe(pipe_slow);
 9013 %}
 9014 
 9015 // This pattern is generated automatically from cas.m4.
 9016 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9017 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9018   predicate(n->as_LoadStore()->barrier_data() == 0);
 9019   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9020   ins_cost(2 * VOLATILE_REF_COST);
 9021   effect(KILL cr);
 9022   format %{
 9023     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9024     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9025   %}
 9026   ins_encode %{
 9027     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9028                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9029                /*weak*/ true, noreg);
 9030     __ csetw($res$$Register, Assembler::EQ);
 9031   %}
 9032   ins_pipe(pipe_slow);
 9033 %}
 9034 
 9035 // This pattern is generated automatically from cas.m4.
 9036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9037 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9038   predicate(needs_acquiring_load_exclusive(n));
 9039   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9040   ins_cost(VOLATILE_REF_COST);
 9041   effect(KILL cr);
 9042   format %{
 9043     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9044     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9045   %}
 9046   ins_encode %{
 9047     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9048                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9049                /*weak*/ true, noreg);
 9050     __ csetw($res$$Register, Assembler::EQ);
 9051   %}
 9052   ins_pipe(pipe_slow);
 9053 %}
 9054 
 9055 // This pattern is generated automatically from cas.m4.
 9056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9057 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9058   predicate(needs_acquiring_load_exclusive(n));
 9059   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9060   ins_cost(VOLATILE_REF_COST);
 9061   effect(KILL cr);
 9062   format %{
 9063     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9064     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9065   %}
 9066   ins_encode %{
 9067     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9068                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9069                /*weak*/ true, noreg);
 9070     __ csetw($res$$Register, Assembler::EQ);
 9071   %}
 9072   ins_pipe(pipe_slow);
 9073 %}
 9074 
 9075 // This pattern is generated automatically from cas.m4.
 9076 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9077 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9078   predicate(needs_acquiring_load_exclusive(n));
 9079   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9080   ins_cost(VOLATILE_REF_COST);
 9081   effect(KILL cr);
 9082   format %{
 9083     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9084     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9085   %}
 9086   ins_encode %{
 9087     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9088                Assembler::word, /*acquire*/ true, /*release*/ true,
 9089                /*weak*/ true, noreg);
 9090     __ csetw($res$$Register, Assembler::EQ);
 9091   %}
 9092   ins_pipe(pipe_slow);
 9093 %}
 9094 
 9095 // This pattern is generated automatically from cas.m4.
 9096 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9097 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9098   predicate(needs_acquiring_load_exclusive(n));
 9099   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9100   ins_cost(VOLATILE_REF_COST);
 9101   effect(KILL cr);
 9102   format %{
 9103     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9104     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9105   %}
 9106   ins_encode %{
 9107     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9108                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9109                /*weak*/ true, noreg);
 9110     __ csetw($res$$Register, Assembler::EQ);
 9111   %}
 9112   ins_pipe(pipe_slow);
 9113 %}
 9114 
 9115 // This pattern is generated automatically from cas.m4.
 9116 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9117 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9118   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 9119   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9120   ins_cost(VOLATILE_REF_COST);
 9121   effect(KILL cr);
 9122   format %{
 9123     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9124     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9125   %}
 9126   ins_encode %{
 9127     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9128                Assembler::word, /*acquire*/ true, /*release*/ true,
 9129                /*weak*/ true, noreg);
 9130     __ csetw($res$$Register, Assembler::EQ);
 9131   %}
 9132   ins_pipe(pipe_slow);
 9133 %}
 9134 
 9135 // This pattern is generated automatically from cas.m4.
 9136 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9137 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9138   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9139   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9140   ins_cost(VOLATILE_REF_COST);
 9141   effect(KILL cr);
 9142   format %{
 9143     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9144     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9145   %}
 9146   ins_encode %{
 9147     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9148                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9149                /*weak*/ true, noreg);
 9150     __ csetw($res$$Register, Assembler::EQ);
 9151   %}
 9152   ins_pipe(pipe_slow);
 9153 %}
 9154 
 9155 // END This section of the file is automatically generated. Do not edit --------------
 9156 // ---------------------------------------------------------------------
 9157 
 9158 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9159   match(Set prev (GetAndSetI mem newv));
 9160   ins_cost(2 * VOLATILE_REF_COST);
 9161   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9162   ins_encode %{
 9163     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9164   %}
 9165   ins_pipe(pipe_serial);
 9166 %}
 9167 
 9168 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9169   match(Set prev (GetAndSetL mem newv));
 9170   ins_cost(2 * VOLATILE_REF_COST);
 9171   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9172   ins_encode %{
 9173     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9174   %}
 9175   ins_pipe(pipe_serial);
 9176 %}
 9177 
 9178 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9179   predicate(n->as_LoadStore()->barrier_data() == 0);
 9180   match(Set prev (GetAndSetN mem newv));
 9181   ins_cost(2 * VOLATILE_REF_COST);
 9182   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9183   ins_encode %{
 9184     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9185   %}
 9186   ins_pipe(pipe_serial);
 9187 %}
 9188 
 9189 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9190   predicate(n->as_LoadStore()->barrier_data() == 0);
 9191   match(Set prev (GetAndSetP mem newv));
 9192   ins_cost(2 * VOLATILE_REF_COST);
 9193   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9194   ins_encode %{
 9195     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9196   %}
 9197   ins_pipe(pipe_serial);
 9198 %}
 9199 
 9200 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9201   predicate(needs_acquiring_load_exclusive(n));
 9202   match(Set prev (GetAndSetI mem newv));
 9203   ins_cost(VOLATILE_REF_COST);
 9204   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9205   ins_encode %{
 9206     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9207   %}
 9208   ins_pipe(pipe_serial);
 9209 %}
 9210 
 9211 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9212   predicate(needs_acquiring_load_exclusive(n));
 9213   match(Set prev (GetAndSetL mem newv));
 9214   ins_cost(VOLATILE_REF_COST);
 9215   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9216   ins_encode %{
 9217     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9218   %}
 9219   ins_pipe(pipe_serial);
 9220 %}
 9221 
 9222 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9223   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 9224   match(Set prev (GetAndSetN mem newv));
 9225   ins_cost(VOLATILE_REF_COST);
 9226   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9227   ins_encode %{
 9228     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9229   %}
 9230   ins_pipe(pipe_serial);
 9231 %}
 9232 
 9233 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9234   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9235   match(Set prev (GetAndSetP mem newv));
 9236   ins_cost(VOLATILE_REF_COST);
 9237   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9238   ins_encode %{
 9239     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9240   %}
 9241   ins_pipe(pipe_serial);
 9242 %}
 9243 
 9244 
 9245 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9246   match(Set newval (GetAndAddL mem incr));
 9247   ins_cost(2 * VOLATILE_REF_COST + 1);
 9248   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9249   ins_encode %{
 9250     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9251   %}
 9252   ins_pipe(pipe_serial);
 9253 %}
 9254 
 9255 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9256   predicate(n->as_LoadStore()->result_not_used());
 9257   match(Set dummy (GetAndAddL mem incr));
 9258   ins_cost(2 * VOLATILE_REF_COST);
 9259   format %{ "get_and_addL [$mem], $incr" %}
 9260   ins_encode %{
 9261     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9262   %}
 9263   ins_pipe(pipe_serial);
 9264 %}
 9265 
 9266 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9267   match(Set newval (GetAndAddL mem incr));
 9268   ins_cost(2 * VOLATILE_REF_COST + 1);
 9269   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9270   ins_encode %{
 9271     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9272   %}
 9273   ins_pipe(pipe_serial);
 9274 %}
 9275 
 9276 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9277   predicate(n->as_LoadStore()->result_not_used());
 9278   match(Set dummy (GetAndAddL mem incr));
 9279   ins_cost(2 * VOLATILE_REF_COST);
 9280   format %{ "get_and_addL [$mem], $incr" %}
 9281   ins_encode %{
 9282     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9283   %}
 9284   ins_pipe(pipe_serial);
 9285 %}
 9286 
 9287 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9288   match(Set newval (GetAndAddI mem incr));
 9289   ins_cost(2 * VOLATILE_REF_COST + 1);
 9290   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9291   ins_encode %{
 9292     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9293   %}
 9294   ins_pipe(pipe_serial);
 9295 %}
 9296 
 9297 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9298   predicate(n->as_LoadStore()->result_not_used());
 9299   match(Set dummy (GetAndAddI mem incr));
 9300   ins_cost(2 * VOLATILE_REF_COST);
 9301   format %{ "get_and_addI [$mem], $incr" %}
 9302   ins_encode %{
 9303     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9304   %}
 9305   ins_pipe(pipe_serial);
 9306 %}
 9307 
 9308 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9309   match(Set newval (GetAndAddI mem incr));
 9310   ins_cost(2 * VOLATILE_REF_COST + 1);
 9311   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9312   ins_encode %{
 9313     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9314   %}
 9315   ins_pipe(pipe_serial);
 9316 %}
 9317 
 9318 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9319   predicate(n->as_LoadStore()->result_not_used());
 9320   match(Set dummy (GetAndAddI mem incr));
 9321   ins_cost(2 * VOLATILE_REF_COST);
 9322   format %{ "get_and_addI [$mem], $incr" %}
 9323   ins_encode %{
 9324     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9325   %}
 9326   ins_pipe(pipe_serial);
 9327 %}
 9328 
 9329 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9330   predicate(needs_acquiring_load_exclusive(n));
 9331   match(Set newval (GetAndAddL mem incr));
 9332   ins_cost(VOLATILE_REF_COST + 1);
 9333   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9334   ins_encode %{
 9335     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9336   %}
 9337   ins_pipe(pipe_serial);
 9338 %}
 9339 
 9340 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9341   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9342   match(Set dummy (GetAndAddL mem incr));
 9343   ins_cost(VOLATILE_REF_COST);
 9344   format %{ "get_and_addL_acq [$mem], $incr" %}
 9345   ins_encode %{
 9346     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9347   %}
 9348   ins_pipe(pipe_serial);
 9349 %}
 9350 
 9351 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9352   predicate(needs_acquiring_load_exclusive(n));
 9353   match(Set newval (GetAndAddL mem incr));
 9354   ins_cost(VOLATILE_REF_COST + 1);
 9355   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9356   ins_encode %{
 9357     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9358   %}
 9359   ins_pipe(pipe_serial);
 9360 %}
 9361 
 9362 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9363   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9364   match(Set dummy (GetAndAddL mem incr));
 9365   ins_cost(VOLATILE_REF_COST);
 9366   format %{ "get_and_addL_acq [$mem], $incr" %}
 9367   ins_encode %{
 9368     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9369   %}
 9370   ins_pipe(pipe_serial);
 9371 %}
 9372 
 9373 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9374   predicate(needs_acquiring_load_exclusive(n));
 9375   match(Set newval (GetAndAddI mem incr));
 9376   ins_cost(VOLATILE_REF_COST + 1);
 9377   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9378   ins_encode %{
 9379     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9380   %}
 9381   ins_pipe(pipe_serial);
 9382 %}
 9383 
 9384 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9385   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9386   match(Set dummy (GetAndAddI mem incr));
 9387   ins_cost(VOLATILE_REF_COST);
 9388   format %{ "get_and_addI_acq [$mem], $incr" %}
 9389   ins_encode %{
 9390     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9391   %}
 9392   ins_pipe(pipe_serial);
 9393 %}
 9394 
 9395 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9396   predicate(needs_acquiring_load_exclusive(n));
 9397   match(Set newval (GetAndAddI mem incr));
 9398   ins_cost(VOLATILE_REF_COST + 1);
 9399   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9400   ins_encode %{
 9401     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9402   %}
 9403   ins_pipe(pipe_serial);
 9404 %}
 9405 
 9406 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9407   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9408   match(Set dummy (GetAndAddI mem incr));
 9409   ins_cost(VOLATILE_REF_COST);
 9410   format %{ "get_and_addI_acq [$mem], $incr" %}
 9411   ins_encode %{
 9412     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9413   %}
 9414   ins_pipe(pipe_serial);
 9415 %}
 9416 
 9417 // Manifest a CmpU result in an integer register.
 9418 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9419 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
 9420 %{
 9421   match(Set dst (CmpU3 src1 src2));
 9422   effect(KILL flags);
 9423 
 9424   ins_cost(INSN_COST * 3);
 9425   format %{
 9426       "cmpw $src1, $src2\n\t"
 9427       "csetw $dst, ne\n\t"
 9428       "cnegw $dst, lo\t# CmpU3(reg)"
 9429   %}
 9430   ins_encode %{
 9431     __ cmpw($src1$$Register, $src2$$Register);
 9432     __ csetw($dst$$Register, Assembler::NE);
 9433     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9434   %}
 9435 
 9436   ins_pipe(pipe_class_default);
 9437 %}
 9438 
 9439 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
 9440 %{
 9441   match(Set dst (CmpU3 src1 src2));
 9442   effect(KILL flags);
 9443 
 9444   ins_cost(INSN_COST * 3);
 9445   format %{
 9446       "subsw zr, $src1, $src2\n\t"
 9447       "csetw $dst, ne\n\t"
 9448       "cnegw $dst, lo\t# CmpU3(imm)"
 9449   %}
 9450   ins_encode %{
 9451     __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
 9452     __ csetw($dst$$Register, Assembler::NE);
 9453     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9454   %}
 9455 
 9456   ins_pipe(pipe_class_default);
 9457 %}
 9458 
 9459 // Manifest a CmpUL result in an integer register.
 9460 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9461 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9462 %{
 9463   match(Set dst (CmpUL3 src1 src2));
 9464   effect(KILL flags);
 9465 
 9466   ins_cost(INSN_COST * 3);
 9467   format %{
 9468       "cmp $src1, $src2\n\t"
 9469       "csetw $dst, ne\n\t"
 9470       "cnegw $dst, lo\t# CmpUL3(reg)"
 9471   %}
 9472   ins_encode %{
 9473     __ cmp($src1$$Register, $src2$$Register);
 9474     __ csetw($dst$$Register, Assembler::NE);
 9475     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9476   %}
 9477 
 9478   ins_pipe(pipe_class_default);
 9479 %}
 9480 
 9481 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9482 %{
 9483   match(Set dst (CmpUL3 src1 src2));
 9484   effect(KILL flags);
 9485 
 9486   ins_cost(INSN_COST * 3);
 9487   format %{
 9488       "subs zr, $src1, $src2\n\t"
 9489       "csetw $dst, ne\n\t"
 9490       "cnegw $dst, lo\t# CmpUL3(imm)"
 9491   %}
 9492   ins_encode %{
 9493     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9494     __ csetw($dst$$Register, Assembler::NE);
 9495     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9496   %}
 9497 
 9498   ins_pipe(pipe_class_default);
 9499 %}
 9500 
 9501 // Manifest a CmpL result in an integer register.
 9502 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9503 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9504 %{
 9505   match(Set dst (CmpL3 src1 src2));
 9506   effect(KILL flags);
 9507 
 9508   ins_cost(INSN_COST * 3);
 9509   format %{
 9510       "cmp $src1, $src2\n\t"
 9511       "csetw $dst, ne\n\t"
 9512       "cnegw $dst, lt\t# CmpL3(reg)"
 9513   %}
 9514   ins_encode %{
 9515     __ cmp($src1$$Register, $src2$$Register);
 9516     __ csetw($dst$$Register, Assembler::NE);
 9517     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9518   %}
 9519 
 9520   ins_pipe(pipe_class_default);
 9521 %}
 9522 
 9523 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9524 %{
 9525   match(Set dst (CmpL3 src1 src2));
 9526   effect(KILL flags);
 9527 
 9528   ins_cost(INSN_COST * 3);
 9529   format %{
 9530       "subs zr, $src1, $src2\n\t"
 9531       "csetw $dst, ne\n\t"
 9532       "cnegw $dst, lt\t# CmpL3(imm)"
 9533   %}
 9534   ins_encode %{
 9535     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9536     __ csetw($dst$$Register, Assembler::NE);
 9537     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9538   %}
 9539 
 9540   ins_pipe(pipe_class_default);
 9541 %}
 9542 
 9543 // ============================================================================
 9544 // Conditional Move Instructions
 9545 
 9546 // n.b. we have identical rules for both a signed compare op (cmpOp)
 9547 // and an unsigned compare op (cmpOpU). it would be nice if we could
 9548 // define an op class which merged both inputs and use it to type the
 9549 // argument to a single rule. unfortunatelyt his fails because the
 9550 // opclass does not live up to the COND_INTER interface of its
 9551 // component operands. When the generic code tries to negate the
 9552 // operand it ends up running the generci Machoper::negate method
 9553 // which throws a ShouldNotHappen. So, we have to provide two flavours
 9554 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
 9555 
 9556 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9557   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9558 
 9559   ins_cost(INSN_COST * 2);
 9560   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
 9561 
 9562   ins_encode %{
 9563     __ cselw(as_Register($dst$$reg),
 9564              as_Register($src2$$reg),
 9565              as_Register($src1$$reg),
 9566              (Assembler::Condition)$cmp$$cmpcode);
 9567   %}
 9568 
 9569   ins_pipe(icond_reg_reg);
 9570 %}
 9571 
 9572 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9573   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9574 
 9575   ins_cost(INSN_COST * 2);
 9576   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
 9577 
 9578   ins_encode %{
 9579     __ cselw(as_Register($dst$$reg),
 9580              as_Register($src2$$reg),
 9581              as_Register($src1$$reg),
 9582              (Assembler::Condition)$cmp$$cmpcode);
 9583   %}
 9584 
 9585   ins_pipe(icond_reg_reg);
 9586 %}
 9587 
 9588 // special cases where one arg is zero
 9589 
 9590 // n.b. this is selected in preference to the rule above because it
 9591 // avoids loading constant 0 into a source register
 9592 
 9593 // TODO
 9594 // we ought only to be able to cull one of these variants as the ideal
 9595 // transforms ought always to order the zero consistently (to left/right?)
 9596 
 9597 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9598   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9599 
 9600   ins_cost(INSN_COST * 2);
 9601   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
 9602 
 9603   ins_encode %{
 9604     __ cselw(as_Register($dst$$reg),
 9605              as_Register($src$$reg),
 9606              zr,
 9607              (Assembler::Condition)$cmp$$cmpcode);
 9608   %}
 9609 
 9610   ins_pipe(icond_reg);
 9611 %}
 9612 
 9613 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9614   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9615 
 9616   ins_cost(INSN_COST * 2);
 9617   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
 9618 
 9619   ins_encode %{
 9620     __ cselw(as_Register($dst$$reg),
 9621              as_Register($src$$reg),
 9622              zr,
 9623              (Assembler::Condition)$cmp$$cmpcode);
 9624   %}
 9625 
 9626   ins_pipe(icond_reg);
 9627 %}
 9628 
 9629 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9630   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9631 
 9632   ins_cost(INSN_COST * 2);
 9633   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
 9634 
 9635   ins_encode %{
 9636     __ cselw(as_Register($dst$$reg),
 9637              zr,
 9638              as_Register($src$$reg),
 9639              (Assembler::Condition)$cmp$$cmpcode);
 9640   %}
 9641 
 9642   ins_pipe(icond_reg);
 9643 %}
 9644 
 9645 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9646   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9647 
 9648   ins_cost(INSN_COST * 2);
 9649   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
 9650 
 9651   ins_encode %{
 9652     __ cselw(as_Register($dst$$reg),
 9653              zr,
 9654              as_Register($src$$reg),
 9655              (Assembler::Condition)$cmp$$cmpcode);
 9656   %}
 9657 
 9658   ins_pipe(icond_reg);
 9659 %}
 9660 
 9661 // special case for creating a boolean 0 or 1
 9662 
 9663 // n.b. this is selected in preference to the rule above because it
 9664 // avoids loading constants 0 and 1 into a source register
 9665 
 9666 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9667   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9668 
 9669   ins_cost(INSN_COST * 2);
 9670   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
 9671 
 9672   ins_encode %{
 9673     // equivalently
 9674     // cset(as_Register($dst$$reg),
 9675     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9676     __ csincw(as_Register($dst$$reg),
 9677              zr,
 9678              zr,
 9679              (Assembler::Condition)$cmp$$cmpcode);
 9680   %}
 9681 
 9682   ins_pipe(icond_none);
 9683 %}
 9684 
 9685 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9686   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9687 
 9688   ins_cost(INSN_COST * 2);
 9689   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
 9690 
 9691   ins_encode %{
 9692     // equivalently
 9693     // cset(as_Register($dst$$reg),
 9694     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9695     __ csincw(as_Register($dst$$reg),
 9696              zr,
 9697              zr,
 9698              (Assembler::Condition)$cmp$$cmpcode);
 9699   %}
 9700 
 9701   ins_pipe(icond_none);
 9702 %}
 9703 
 9704 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9705   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9706 
 9707   ins_cost(INSN_COST * 2);
 9708   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
 9709 
 9710   ins_encode %{
 9711     __ csel(as_Register($dst$$reg),
 9712             as_Register($src2$$reg),
 9713             as_Register($src1$$reg),
 9714             (Assembler::Condition)$cmp$$cmpcode);
 9715   %}
 9716 
 9717   ins_pipe(icond_reg_reg);
 9718 %}
 9719 
 9720 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9721   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9722 
 9723   ins_cost(INSN_COST * 2);
 9724   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
 9725 
 9726   ins_encode %{
 9727     __ csel(as_Register($dst$$reg),
 9728             as_Register($src2$$reg),
 9729             as_Register($src1$$reg),
 9730             (Assembler::Condition)$cmp$$cmpcode);
 9731   %}
 9732 
 9733   ins_pipe(icond_reg_reg);
 9734 %}
 9735 
 9736 // special cases where one arg is zero
 9737 
 9738 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9739   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9740 
 9741   ins_cost(INSN_COST * 2);
 9742   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
 9743 
 9744   ins_encode %{
 9745     __ csel(as_Register($dst$$reg),
 9746             zr,
 9747             as_Register($src$$reg),
 9748             (Assembler::Condition)$cmp$$cmpcode);
 9749   %}
 9750 
 9751   ins_pipe(icond_reg);
 9752 %}
 9753 
 9754 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9755   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9756 
 9757   ins_cost(INSN_COST * 2);
 9758   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
 9759 
 9760   ins_encode %{
 9761     __ csel(as_Register($dst$$reg),
 9762             zr,
 9763             as_Register($src$$reg),
 9764             (Assembler::Condition)$cmp$$cmpcode);
 9765   %}
 9766 
 9767   ins_pipe(icond_reg);
 9768 %}
 9769 
 9770 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9771   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9772 
 9773   ins_cost(INSN_COST * 2);
 9774   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
 9775 
 9776   ins_encode %{
 9777     __ csel(as_Register($dst$$reg),
 9778             as_Register($src$$reg),
 9779             zr,
 9780             (Assembler::Condition)$cmp$$cmpcode);
 9781   %}
 9782 
 9783   ins_pipe(icond_reg);
 9784 %}
 9785 
 9786 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9787   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9788 
 9789   ins_cost(INSN_COST * 2);
 9790   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
 9791 
 9792   ins_encode %{
 9793     __ csel(as_Register($dst$$reg),
 9794             as_Register($src$$reg),
 9795             zr,
 9796             (Assembler::Condition)$cmp$$cmpcode);
 9797   %}
 9798 
 9799   ins_pipe(icond_reg);
 9800 %}
 9801 
 9802 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
 9803   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
 9804 
 9805   ins_cost(INSN_COST * 2);
 9806   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
 9807 
 9808   ins_encode %{
 9809     __ csel(as_Register($dst$$reg),
 9810             as_Register($src2$$reg),
 9811             as_Register($src1$$reg),
 9812             (Assembler::Condition)$cmp$$cmpcode);
 9813   %}
 9814 
 9815   ins_pipe(icond_reg_reg);
 9816 %}
 9817 
 9818 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
 9819   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
 9820 
 9821   ins_cost(INSN_COST * 2);
 9822   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
 9823 
 9824   ins_encode %{
 9825     __ csel(as_Register($dst$$reg),
 9826             as_Register($src2$$reg),
 9827             as_Register($src1$$reg),
 9828             (Assembler::Condition)$cmp$$cmpcode);
 9829   %}
 9830 
 9831   ins_pipe(icond_reg_reg);
 9832 %}
 9833 
 9834 // special cases where one arg is zero
 9835 
 9836 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
 9837   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
 9838 
 9839   ins_cost(INSN_COST * 2);
 9840   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
 9841 
 9842   ins_encode %{
 9843     __ csel(as_Register($dst$$reg),
 9844             zr,
 9845             as_Register($src$$reg),
 9846             (Assembler::Condition)$cmp$$cmpcode);
 9847   %}
 9848 
 9849   ins_pipe(icond_reg);
 9850 %}
 9851 
 9852 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
 9853   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
 9854 
 9855   ins_cost(INSN_COST * 2);
 9856   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
 9857 
 9858   ins_encode %{
 9859     __ csel(as_Register($dst$$reg),
 9860             zr,
 9861             as_Register($src$$reg),
 9862             (Assembler::Condition)$cmp$$cmpcode);
 9863   %}
 9864 
 9865   ins_pipe(icond_reg);
 9866 %}
 9867 
 9868 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
 9869   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
 9870 
 9871   ins_cost(INSN_COST * 2);
 9872   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
 9873 
 9874   ins_encode %{
 9875     __ csel(as_Register($dst$$reg),
 9876             as_Register($src$$reg),
 9877             zr,
 9878             (Assembler::Condition)$cmp$$cmpcode);
 9879   %}
 9880 
 9881   ins_pipe(icond_reg);
 9882 %}
 9883 
 9884 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
 9885   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
 9886 
 9887   ins_cost(INSN_COST * 2);
 9888   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
 9889 
 9890   ins_encode %{
 9891     __ csel(as_Register($dst$$reg),
 9892             as_Register($src$$reg),
 9893             zr,
 9894             (Assembler::Condition)$cmp$$cmpcode);
 9895   %}
 9896 
 9897   ins_pipe(icond_reg);
 9898 %}
 9899 
 9900 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
 9901   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
 9902 
 9903   ins_cost(INSN_COST * 2);
 9904   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
 9905 
 9906   ins_encode %{
 9907     __ cselw(as_Register($dst$$reg),
 9908              as_Register($src2$$reg),
 9909              as_Register($src1$$reg),
 9910              (Assembler::Condition)$cmp$$cmpcode);
 9911   %}
 9912 
 9913   ins_pipe(icond_reg_reg);
 9914 %}
 9915 
 9916 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
 9917   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
 9918 
 9919   ins_cost(INSN_COST * 2);
 9920   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
 9921 
 9922   ins_encode %{
 9923     __ cselw(as_Register($dst$$reg),
 9924              as_Register($src2$$reg),
 9925              as_Register($src1$$reg),
 9926              (Assembler::Condition)$cmp$$cmpcode);
 9927   %}
 9928 
 9929   ins_pipe(icond_reg_reg);
 9930 %}
 9931 
 9932 // special cases where one arg is zero
 9933 
 9934 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
 9935   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
 9936 
 9937   ins_cost(INSN_COST * 2);
 9938   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
 9939 
 9940   ins_encode %{
 9941     __ cselw(as_Register($dst$$reg),
 9942              zr,
 9943              as_Register($src$$reg),
 9944              (Assembler::Condition)$cmp$$cmpcode);
 9945   %}
 9946 
 9947   ins_pipe(icond_reg);
 9948 %}
 9949 
 9950 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
 9951   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
 9952 
 9953   ins_cost(INSN_COST * 2);
 9954   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
 9955 
 9956   ins_encode %{
 9957     __ cselw(as_Register($dst$$reg),
 9958              zr,
 9959              as_Register($src$$reg),
 9960              (Assembler::Condition)$cmp$$cmpcode);
 9961   %}
 9962 
 9963   ins_pipe(icond_reg);
 9964 %}
 9965 
 9966 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
 9967   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
 9968 
 9969   ins_cost(INSN_COST * 2);
 9970   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
 9971 
 9972   ins_encode %{
 9973     __ cselw(as_Register($dst$$reg),
 9974              as_Register($src$$reg),
 9975              zr,
 9976              (Assembler::Condition)$cmp$$cmpcode);
 9977   %}
 9978 
 9979   ins_pipe(icond_reg);
 9980 %}
 9981 
 9982 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
 9983   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
 9984 
 9985   ins_cost(INSN_COST * 2);
 9986   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
 9987 
 9988   ins_encode %{
 9989     __ cselw(as_Register($dst$$reg),
 9990              as_Register($src$$reg),
 9991              zr,
 9992              (Assembler::Condition)$cmp$$cmpcode);
 9993   %}
 9994 
 9995   ins_pipe(icond_reg);
 9996 %}
 9997 
 9998 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
 9999 %{
10000   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10001 
10002   ins_cost(INSN_COST * 3);
10003 
10004   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10005   ins_encode %{
10006     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10007     __ fcsels(as_FloatRegister($dst$$reg),
10008               as_FloatRegister($src2$$reg),
10009               as_FloatRegister($src1$$reg),
10010               cond);
10011   %}
10012 
10013   ins_pipe(fp_cond_reg_reg_s);
10014 %}
10015 
10016 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10017 %{
10018   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10019 
10020   ins_cost(INSN_COST * 3);
10021 
10022   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10023   ins_encode %{
10024     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10025     __ fcsels(as_FloatRegister($dst$$reg),
10026               as_FloatRegister($src2$$reg),
10027               as_FloatRegister($src1$$reg),
10028               cond);
10029   %}
10030 
10031   ins_pipe(fp_cond_reg_reg_s);
10032 %}
10033 
10034 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10035 %{
10036   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10037 
10038   ins_cost(INSN_COST * 3);
10039 
10040   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10041   ins_encode %{
10042     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10043     __ fcseld(as_FloatRegister($dst$$reg),
10044               as_FloatRegister($src2$$reg),
10045               as_FloatRegister($src1$$reg),
10046               cond);
10047   %}
10048 
10049   ins_pipe(fp_cond_reg_reg_d);
10050 %}
10051 
10052 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10053 %{
10054   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10055 
10056   ins_cost(INSN_COST * 3);
10057 
10058   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10059   ins_encode %{
10060     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10061     __ fcseld(as_FloatRegister($dst$$reg),
10062               as_FloatRegister($src2$$reg),
10063               as_FloatRegister($src1$$reg),
10064               cond);
10065   %}
10066 
10067   ins_pipe(fp_cond_reg_reg_d);
10068 %}
10069 
10070 // ============================================================================
10071 // Arithmetic Instructions
10072 //
10073 
10074 // Integer Addition
10075 
10076 // TODO
10077 // these currently employ operations which do not set CR and hence are
10078 // not flagged as killing CR but we would like to isolate the cases
10079 // where we want to set flags from those where we don't. need to work
10080 // out how to do that.
10081 
10082 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10083   match(Set dst (AddI src1 src2));
10084 
10085   ins_cost(INSN_COST);
10086   format %{ "addw  $dst, $src1, $src2" %}
10087 
10088   ins_encode %{
10089     __ addw(as_Register($dst$$reg),
10090             as_Register($src1$$reg),
10091             as_Register($src2$$reg));
10092   %}
10093 
10094   ins_pipe(ialu_reg_reg);
10095 %}
10096 
10097 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10098   match(Set dst (AddI src1 src2));
10099 
10100   ins_cost(INSN_COST);
10101   format %{ "addw $dst, $src1, $src2" %}
10102 
10103   // use opcode to indicate that this is an add not a sub
10104   opcode(0x0);
10105 
10106   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10107 
10108   ins_pipe(ialu_reg_imm);
10109 %}
10110 
10111 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10112   match(Set dst (AddI (ConvL2I src1) src2));
10113 
10114   ins_cost(INSN_COST);
10115   format %{ "addw $dst, $src1, $src2" %}
10116 
10117   // use opcode to indicate that this is an add not a sub
10118   opcode(0x0);
10119 
10120   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10121 
10122   ins_pipe(ialu_reg_imm);
10123 %}
10124 
10125 // Pointer Addition
10126 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
10127   match(Set dst (AddP src1 src2));
10128 
10129   ins_cost(INSN_COST);
10130   format %{ "add $dst, $src1, $src2\t# ptr" %}
10131 
10132   ins_encode %{
10133     __ add(as_Register($dst$$reg),
10134            as_Register($src1$$reg),
10135            as_Register($src2$$reg));
10136   %}
10137 
10138   ins_pipe(ialu_reg_reg);
10139 %}
10140 
10141 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
10142   match(Set dst (AddP src1 (ConvI2L src2)));
10143 
10144   ins_cost(1.9 * INSN_COST);
10145   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10146 
10147   ins_encode %{
10148     __ add(as_Register($dst$$reg),
10149            as_Register($src1$$reg),
10150            as_Register($src2$$reg), ext::sxtw);
10151   %}
10152 
10153   ins_pipe(ialu_reg_reg);
10154 %}
10155 
10156 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
10157   match(Set dst (AddP src1 (LShiftL src2 scale)));
10158 
10159   ins_cost(1.9 * INSN_COST);
10160   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10161 
10162   ins_encode %{
10163     __ lea(as_Register($dst$$reg),
10164            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10165                    Address::lsl($scale$$constant)));
10166   %}
10167 
10168   ins_pipe(ialu_reg_reg_shift);
10169 %}
10170 
10171 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
10172   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10173 
10174   ins_cost(1.9 * INSN_COST);
10175   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10176 
10177   ins_encode %{
10178     __ lea(as_Register($dst$$reg),
10179            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10180                    Address::sxtw($scale$$constant)));
10181   %}
10182 
10183   ins_pipe(ialu_reg_reg_shift);
10184 %}
10185 
10186 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10187   match(Set dst (LShiftL (ConvI2L src) scale));
10188 
10189   ins_cost(INSN_COST);
10190   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10191 
10192   ins_encode %{
10193     __ sbfiz(as_Register($dst$$reg),
10194           as_Register($src$$reg),
10195           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10196   %}
10197 
10198   ins_pipe(ialu_reg_shift);
10199 %}
10200 
10201 // Pointer Immediate Addition
10202 // n.b. this needs to be more expensive than using an indirect memory
10203 // operand
10204 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10205   match(Set dst (AddP src1 src2));
10206 
10207   ins_cost(INSN_COST);
10208   format %{ "add $dst, $src1, $src2\t# ptr" %}
10209 
10210   // use opcode to indicate that this is an add not a sub
10211   opcode(0x0);
10212 
10213   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10214 
10215   ins_pipe(ialu_reg_imm);
10216 %}
10217 
10218 // Long Addition
10219 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10220 
10221   match(Set dst (AddL src1 src2));
10222 
10223   ins_cost(INSN_COST);
10224   format %{ "add  $dst, $src1, $src2" %}
10225 
10226   ins_encode %{
10227     __ add(as_Register($dst$$reg),
10228            as_Register($src1$$reg),
10229            as_Register($src2$$reg));
10230   %}
10231 
10232   ins_pipe(ialu_reg_reg);
10233 %}
10234 
10235 // No constant pool entries requiredLong Immediate Addition.
10236 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10237   match(Set dst (AddL src1 src2));
10238 
10239   ins_cost(INSN_COST);
10240   format %{ "add $dst, $src1, $src2" %}
10241 
10242   // use opcode to indicate that this is an add not a sub
10243   opcode(0x0);
10244 
10245   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10246 
10247   ins_pipe(ialu_reg_imm);
10248 %}
10249 
10250 // Integer Subtraction
10251 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10252   match(Set dst (SubI src1 src2));
10253 
10254   ins_cost(INSN_COST);
10255   format %{ "subw  $dst, $src1, $src2" %}
10256 
10257   ins_encode %{
10258     __ subw(as_Register($dst$$reg),
10259             as_Register($src1$$reg),
10260             as_Register($src2$$reg));
10261   %}
10262 
10263   ins_pipe(ialu_reg_reg);
10264 %}
10265 
10266 // Immediate Subtraction
10267 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10268   match(Set dst (SubI src1 src2));
10269 
10270   ins_cost(INSN_COST);
10271   format %{ "subw $dst, $src1, $src2" %}
10272 
10273   // use opcode to indicate that this is a sub not an add
10274   opcode(0x1);
10275 
10276   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10277 
10278   ins_pipe(ialu_reg_imm);
10279 %}
10280 
10281 // Long Subtraction
10282 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10283 
10284   match(Set dst (SubL src1 src2));
10285 
10286   ins_cost(INSN_COST);
10287   format %{ "sub  $dst, $src1, $src2" %}
10288 
10289   ins_encode %{
10290     __ sub(as_Register($dst$$reg),
10291            as_Register($src1$$reg),
10292            as_Register($src2$$reg));
10293   %}
10294 
10295   ins_pipe(ialu_reg_reg);
10296 %}
10297 
10298 // No constant pool entries requiredLong Immediate Subtraction.
10299 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10300   match(Set dst (SubL src1 src2));
10301 
10302   ins_cost(INSN_COST);
10303   format %{ "sub$dst, $src1, $src2" %}
10304 
10305   // use opcode to indicate that this is a sub not an add
10306   opcode(0x1);
10307 
10308   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10309 
10310   ins_pipe(ialu_reg_imm);
10311 %}
10312 
10313 // Integer Negation (special case for sub)
10314 
10315 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10316   match(Set dst (SubI zero src));
10317 
10318   ins_cost(INSN_COST);
10319   format %{ "negw $dst, $src\t# int" %}
10320 
10321   ins_encode %{
10322     __ negw(as_Register($dst$$reg),
10323             as_Register($src$$reg));
10324   %}
10325 
10326   ins_pipe(ialu_reg);
10327 %}
10328 
10329 // Long Negation
10330 
10331 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10332   match(Set dst (SubL zero src));
10333 
10334   ins_cost(INSN_COST);
10335   format %{ "neg $dst, $src\t# long" %}
10336 
10337   ins_encode %{
10338     __ neg(as_Register($dst$$reg),
10339            as_Register($src$$reg));
10340   %}
10341 
10342   ins_pipe(ialu_reg);
10343 %}
10344 
10345 // Integer Multiply
10346 
10347 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10348   match(Set dst (MulI src1 src2));
10349 
10350   ins_cost(INSN_COST * 3);
10351   format %{ "mulw  $dst, $src1, $src2" %}
10352 
10353   ins_encode %{
10354     __ mulw(as_Register($dst$$reg),
10355             as_Register($src1$$reg),
10356             as_Register($src2$$reg));
10357   %}
10358 
10359   ins_pipe(imul_reg_reg);
10360 %}
10361 
10362 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10363   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10364 
10365   ins_cost(INSN_COST * 3);
10366   format %{ "smull  $dst, $src1, $src2" %}
10367 
10368   ins_encode %{
10369     __ smull(as_Register($dst$$reg),
10370              as_Register($src1$$reg),
10371              as_Register($src2$$reg));
10372   %}
10373 
10374   ins_pipe(imul_reg_reg);
10375 %}
10376 
10377 // Long Multiply
10378 
10379 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10380   match(Set dst (MulL src1 src2));
10381 
10382   ins_cost(INSN_COST * 5);
10383   format %{ "mul  $dst, $src1, $src2" %}
10384 
10385   ins_encode %{
10386     __ mul(as_Register($dst$$reg),
10387            as_Register($src1$$reg),
10388            as_Register($src2$$reg));
10389   %}
10390 
10391   ins_pipe(lmul_reg_reg);
10392 %}
10393 
10394 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10395 %{
10396   match(Set dst (MulHiL src1 src2));
10397 
10398   ins_cost(INSN_COST * 7);
10399   format %{ "smulh   $dst, $src1, $src2\t# mulhi" %}
10400 
10401   ins_encode %{
10402     __ smulh(as_Register($dst$$reg),
10403              as_Register($src1$$reg),
10404              as_Register($src2$$reg));
10405   %}
10406 
10407   ins_pipe(lmul_reg_reg);
10408 %}
10409 
10410 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10411 %{
10412   match(Set dst (UMulHiL src1 src2));
10413 
10414   ins_cost(INSN_COST * 7);
10415   format %{ "umulh   $dst, $src1, $src2\t# umulhi" %}
10416 
10417   ins_encode %{
10418     __ umulh(as_Register($dst$$reg),
10419              as_Register($src1$$reg),
10420              as_Register($src2$$reg));
10421   %}
10422 
10423   ins_pipe(lmul_reg_reg);
10424 %}
10425 
10426 // Combined Integer Multiply & Add/Sub
10427 
10428 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10429   match(Set dst (AddI src3 (MulI src1 src2)));
10430 
10431   ins_cost(INSN_COST * 3);
10432   format %{ "madd  $dst, $src1, $src2, $src3" %}
10433 
10434   ins_encode %{
10435     __ maddw(as_Register($dst$$reg),
10436              as_Register($src1$$reg),
10437              as_Register($src2$$reg),
10438              as_Register($src3$$reg));
10439   %}
10440 
10441   ins_pipe(imac_reg_reg);
10442 %}
10443 
10444 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10445   match(Set dst (SubI src3 (MulI src1 src2)));
10446 
10447   ins_cost(INSN_COST * 3);
10448   format %{ "msub  $dst, $src1, $src2, $src3" %}
10449 
10450   ins_encode %{
10451     __ msubw(as_Register($dst$$reg),
10452              as_Register($src1$$reg),
10453              as_Register($src2$$reg),
10454              as_Register($src3$$reg));
10455   %}
10456 
10457   ins_pipe(imac_reg_reg);
10458 %}
10459 
10460 // Combined Integer Multiply & Neg
10461 
10462 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10463   match(Set dst (MulI (SubI zero src1) src2));
10464 
10465   ins_cost(INSN_COST * 3);
10466   format %{ "mneg  $dst, $src1, $src2" %}
10467 
10468   ins_encode %{
10469     __ mnegw(as_Register($dst$$reg),
10470              as_Register($src1$$reg),
10471              as_Register($src2$$reg));
10472   %}
10473 
10474   ins_pipe(imac_reg_reg);
10475 %}
10476 
10477 // Combined Long Multiply & Add/Sub
10478 
10479 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10480   match(Set dst (AddL src3 (MulL src1 src2)));
10481 
10482   ins_cost(INSN_COST * 5);
10483   format %{ "madd  $dst, $src1, $src2, $src3" %}
10484 
10485   ins_encode %{
10486     __ madd(as_Register($dst$$reg),
10487             as_Register($src1$$reg),
10488             as_Register($src2$$reg),
10489             as_Register($src3$$reg));
10490   %}
10491 
10492   ins_pipe(lmac_reg_reg);
10493 %}
10494 
10495 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10496   match(Set dst (SubL src3 (MulL src1 src2)));
10497 
10498   ins_cost(INSN_COST * 5);
10499   format %{ "msub  $dst, $src1, $src2, $src3" %}
10500 
10501   ins_encode %{
10502     __ msub(as_Register($dst$$reg),
10503             as_Register($src1$$reg),
10504             as_Register($src2$$reg),
10505             as_Register($src3$$reg));
10506   %}
10507 
10508   ins_pipe(lmac_reg_reg);
10509 %}
10510 
10511 // Combined Long Multiply & Neg
10512 
10513 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10514   match(Set dst (MulL (SubL zero src1) src2));
10515 
10516   ins_cost(INSN_COST * 5);
10517   format %{ "mneg  $dst, $src1, $src2" %}
10518 
10519   ins_encode %{
10520     __ mneg(as_Register($dst$$reg),
10521             as_Register($src1$$reg),
10522             as_Register($src2$$reg));
10523   %}
10524 
10525   ins_pipe(lmac_reg_reg);
10526 %}
10527 
10528 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10529 
10530 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10531   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10532 
10533   ins_cost(INSN_COST * 3);
10534   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10535 
10536   ins_encode %{
10537     __ smaddl(as_Register($dst$$reg),
10538               as_Register($src1$$reg),
10539               as_Register($src2$$reg),
10540               as_Register($src3$$reg));
10541   %}
10542 
10543   ins_pipe(imac_reg_reg);
10544 %}
10545 
10546 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10547   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10548 
10549   ins_cost(INSN_COST * 3);
10550   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10551 
10552   ins_encode %{
10553     __ smsubl(as_Register($dst$$reg),
10554               as_Register($src1$$reg),
10555               as_Register($src2$$reg),
10556               as_Register($src3$$reg));
10557   %}
10558 
10559   ins_pipe(imac_reg_reg);
10560 %}
10561 
10562 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10563   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10564 
10565   ins_cost(INSN_COST * 3);
10566   format %{ "smnegl  $dst, $src1, $src2" %}
10567 
10568   ins_encode %{
10569     __ smnegl(as_Register($dst$$reg),
10570               as_Register($src1$$reg),
10571               as_Register($src2$$reg));
10572   %}
10573 
10574   ins_pipe(imac_reg_reg);
10575 %}
10576 
10577 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10578 
10579 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10580   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10581 
10582   ins_cost(INSN_COST * 5);
10583   format %{ "mulw  rscratch1, $src1, $src2\n\t"
10584             "maddw $dst, $src3, $src4, rscratch1" %}
10585 
10586   ins_encode %{
10587     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10588     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10589 
10590   ins_pipe(imac_reg_reg);
10591 %}
10592 
10593 // Integer Divide
10594 
10595 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10596   match(Set dst (DivI src1 src2));
10597 
10598   ins_cost(INSN_COST * 19);
10599   format %{ "sdivw  $dst, $src1, $src2" %}
10600 
10601   ins_encode(aarch64_enc_divw(dst, src1, src2));
10602   ins_pipe(idiv_reg_reg);
10603 %}
10604 
10605 // Long Divide
10606 
10607 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10608   match(Set dst (DivL src1 src2));
10609 
10610   ins_cost(INSN_COST * 35);
10611   format %{ "sdiv   $dst, $src1, $src2" %}
10612 
10613   ins_encode(aarch64_enc_div(dst, src1, src2));
10614   ins_pipe(ldiv_reg_reg);
10615 %}
10616 
10617 // Integer Remainder
10618 
10619 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10620   match(Set dst (ModI src1 src2));
10621 
10622   ins_cost(INSN_COST * 22);
10623   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
10624             "msubw  $dst, rscratch1, $src2, $src1" %}
10625 
10626   ins_encode(aarch64_enc_modw(dst, src1, src2));
10627   ins_pipe(idiv_reg_reg);
10628 %}
10629 
10630 // Long Remainder
10631 
10632 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10633   match(Set dst (ModL src1 src2));
10634 
10635   ins_cost(INSN_COST * 38);
10636   format %{ "sdiv   rscratch1, $src1, $src2\n"
10637             "msub   $dst, rscratch1, $src2, $src1" %}
10638 
10639   ins_encode(aarch64_enc_mod(dst, src1, src2));
10640   ins_pipe(ldiv_reg_reg);
10641 %}
10642 
10643 // Unsigned Integer Divide
10644 
10645 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10646   match(Set dst (UDivI src1 src2));
10647 
10648   ins_cost(INSN_COST * 19);
10649   format %{ "udivw  $dst, $src1, $src2" %}
10650 
10651   ins_encode %{
10652     __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10653   %}
10654 
10655   ins_pipe(idiv_reg_reg);
10656 %}
10657 
10658 //  Unsigned Long Divide
10659 
10660 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10661   match(Set dst (UDivL src1 src2));
10662 
10663   ins_cost(INSN_COST * 35);
10664   format %{ "udiv   $dst, $src1, $src2" %}
10665 
10666   ins_encode %{
10667     __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10668   %}
10669 
10670   ins_pipe(ldiv_reg_reg);
10671 %}
10672 
10673 // Unsigned Integer Remainder
10674 
10675 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10676   match(Set dst (UModI src1 src2));
10677 
10678   ins_cost(INSN_COST * 22);
10679   format %{ "udivw  rscratch1, $src1, $src2\n\t"
10680             "msubw  $dst, rscratch1, $src2, $src1" %}
10681 
10682   ins_encode %{
10683     __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10684     __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10685   %}
10686 
10687   ins_pipe(idiv_reg_reg);
10688 %}
10689 
10690 // Unsigned Long Remainder
10691 
10692 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10693   match(Set dst (UModL src1 src2));
10694 
10695   ins_cost(INSN_COST * 38);
10696   format %{ "udiv   rscratch1, $src1, $src2\n"
10697             "msub   $dst, rscratch1, $src2, $src1" %}
10698 
10699   ins_encode %{
10700     __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10701     __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10702   %}
10703 
10704   ins_pipe(ldiv_reg_reg);
10705 %}
10706 
10707 // Integer Shifts
10708 
10709 // Shift Left Register
10710 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10711   match(Set dst (LShiftI src1 src2));
10712 
10713   ins_cost(INSN_COST * 2);
10714   format %{ "lslvw  $dst, $src1, $src2" %}
10715 
10716   ins_encode %{
10717     __ lslvw(as_Register($dst$$reg),
10718              as_Register($src1$$reg),
10719              as_Register($src2$$reg));
10720   %}
10721 
10722   ins_pipe(ialu_reg_reg_vshift);
10723 %}
10724 
10725 // Shift Left Immediate
10726 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10727   match(Set dst (LShiftI src1 src2));
10728 
10729   ins_cost(INSN_COST);
10730   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10731 
10732   ins_encode %{
10733     __ lslw(as_Register($dst$$reg),
10734             as_Register($src1$$reg),
10735             $src2$$constant & 0x1f);
10736   %}
10737 
10738   ins_pipe(ialu_reg_shift);
10739 %}
10740 
10741 // Shift Right Logical Register
10742 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10743   match(Set dst (URShiftI src1 src2));
10744 
10745   ins_cost(INSN_COST * 2);
10746   format %{ "lsrvw  $dst, $src1, $src2" %}
10747 
10748   ins_encode %{
10749     __ lsrvw(as_Register($dst$$reg),
10750              as_Register($src1$$reg),
10751              as_Register($src2$$reg));
10752   %}
10753 
10754   ins_pipe(ialu_reg_reg_vshift);
10755 %}
10756 
10757 // Shift Right Logical Immediate
10758 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10759   match(Set dst (URShiftI src1 src2));
10760 
10761   ins_cost(INSN_COST);
10762   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10763 
10764   ins_encode %{
10765     __ lsrw(as_Register($dst$$reg),
10766             as_Register($src1$$reg),
10767             $src2$$constant & 0x1f);
10768   %}
10769 
10770   ins_pipe(ialu_reg_shift);
10771 %}
10772 
10773 // Shift Right Arithmetic Register
10774 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10775   match(Set dst (RShiftI src1 src2));
10776 
10777   ins_cost(INSN_COST * 2);
10778   format %{ "asrvw  $dst, $src1, $src2" %}
10779 
10780   ins_encode %{
10781     __ asrvw(as_Register($dst$$reg),
10782              as_Register($src1$$reg),
10783              as_Register($src2$$reg));
10784   %}
10785 
10786   ins_pipe(ialu_reg_reg_vshift);
10787 %}
10788 
10789 // Shift Right Arithmetic Immediate
10790 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10791   match(Set dst (RShiftI src1 src2));
10792 
10793   ins_cost(INSN_COST);
10794   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10795 
10796   ins_encode %{
10797     __ asrw(as_Register($dst$$reg),
10798             as_Register($src1$$reg),
10799             $src2$$constant & 0x1f);
10800   %}
10801 
10802   ins_pipe(ialu_reg_shift);
10803 %}
10804 
10805 // Combined Int Mask and Right Shift (using UBFM)
10806 // TODO
10807 
10808 // Long Shifts
10809 
10810 // Shift Left Register
10811 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10812   match(Set dst (LShiftL src1 src2));
10813 
10814   ins_cost(INSN_COST * 2);
10815   format %{ "lslv  $dst, $src1, $src2" %}
10816 
10817   ins_encode %{
10818     __ lslv(as_Register($dst$$reg),
10819             as_Register($src1$$reg),
10820             as_Register($src2$$reg));
10821   %}
10822 
10823   ins_pipe(ialu_reg_reg_vshift);
10824 %}
10825 
10826 // Shift Left Immediate
10827 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10828   match(Set dst (LShiftL src1 src2));
10829 
10830   ins_cost(INSN_COST);
10831   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10832 
10833   ins_encode %{
10834     __ lsl(as_Register($dst$$reg),
10835             as_Register($src1$$reg),
10836             $src2$$constant & 0x3f);
10837   %}
10838 
10839   ins_pipe(ialu_reg_shift);
10840 %}
10841 
10842 // Shift Right Logical Register
10843 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10844   match(Set dst (URShiftL src1 src2));
10845 
10846   ins_cost(INSN_COST * 2);
10847   format %{ "lsrv  $dst, $src1, $src2" %}
10848 
10849   ins_encode %{
10850     __ lsrv(as_Register($dst$$reg),
10851             as_Register($src1$$reg),
10852             as_Register($src2$$reg));
10853   %}
10854 
10855   ins_pipe(ialu_reg_reg_vshift);
10856 %}
10857 
10858 // Shift Right Logical Immediate
10859 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10860   match(Set dst (URShiftL src1 src2));
10861 
10862   ins_cost(INSN_COST);
10863   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10864 
10865   ins_encode %{
10866     __ lsr(as_Register($dst$$reg),
10867            as_Register($src1$$reg),
10868            $src2$$constant & 0x3f);
10869   %}
10870 
10871   ins_pipe(ialu_reg_shift);
10872 %}
10873 
10874 // A special-case pattern for card table stores.
10875 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10876   match(Set dst (URShiftL (CastP2X src1) src2));
10877 
10878   ins_cost(INSN_COST);
10879   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10880 
10881   ins_encode %{
10882     __ lsr(as_Register($dst$$reg),
10883            as_Register($src1$$reg),
10884            $src2$$constant & 0x3f);
10885   %}
10886 
10887   ins_pipe(ialu_reg_shift);
10888 %}
10889 
10890 // Shift Right Arithmetic Register
10891 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10892   match(Set dst (RShiftL src1 src2));
10893 
10894   ins_cost(INSN_COST * 2);
10895   format %{ "asrv  $dst, $src1, $src2" %}
10896 
10897   ins_encode %{
10898     __ asrv(as_Register($dst$$reg),
10899             as_Register($src1$$reg),
10900             as_Register($src2$$reg));
10901   %}
10902 
10903   ins_pipe(ialu_reg_reg_vshift);
10904 %}
10905 
10906 // Shift Right Arithmetic Immediate
10907 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10908   match(Set dst (RShiftL src1 src2));
10909 
10910   ins_cost(INSN_COST);
10911   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10912 
10913   ins_encode %{
10914     __ asr(as_Register($dst$$reg),
10915            as_Register($src1$$reg),
10916            $src2$$constant & 0x3f);
10917   %}
10918 
10919   ins_pipe(ialu_reg_shift);
10920 %}
10921 
10922 // BEGIN This section of the file is automatically generated. Do not edit --------------
10923 // This section is generated from aarch64_ad.m4
10924 
10925 // This pattern is automatically generated from aarch64_ad.m4
10926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10927 instruct regL_not_reg(iRegLNoSp dst,
10928                          iRegL src1, immL_M1 m1,
10929                          rFlagsReg cr) %{
10930   match(Set dst (XorL src1 m1));
10931   ins_cost(INSN_COST);
10932   format %{ "eon  $dst, $src1, zr" %}
10933 
10934   ins_encode %{
10935     __ eon(as_Register($dst$$reg),
10936               as_Register($src1$$reg),
10937               zr,
10938               Assembler::LSL, 0);
10939   %}
10940 
10941   ins_pipe(ialu_reg);
10942 %}
10943 
10944 // This pattern is automatically generated from aarch64_ad.m4
10945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10946 instruct regI_not_reg(iRegINoSp dst,
10947                          iRegIorL2I src1, immI_M1 m1,
10948                          rFlagsReg cr) %{
10949   match(Set dst (XorI src1 m1));
10950   ins_cost(INSN_COST);
10951   format %{ "eonw  $dst, $src1, zr" %}
10952 
10953   ins_encode %{
10954     __ eonw(as_Register($dst$$reg),
10955               as_Register($src1$$reg),
10956               zr,
10957               Assembler::LSL, 0);
10958   %}
10959 
10960   ins_pipe(ialu_reg);
10961 %}
10962 
10963 // This pattern is automatically generated from aarch64_ad.m4
10964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10965 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10966                               immI0 zero, iRegIorL2I src1, immI src2) %{
10967   match(Set dst (SubI zero (URShiftI src1 src2)));
10968 
10969   ins_cost(1.9 * INSN_COST);
10970   format %{ "negw  $dst, $src1, LSR $src2" %}
10971 
10972   ins_encode %{
10973     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10974             Assembler::LSR, $src2$$constant & 0x1f);
10975   %}
10976 
10977   ins_pipe(ialu_reg_shift);
10978 %}
10979 
10980 // This pattern is automatically generated from aarch64_ad.m4
10981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10982 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10983                               immI0 zero, iRegIorL2I src1, immI src2) %{
10984   match(Set dst (SubI zero (RShiftI src1 src2)));
10985 
10986   ins_cost(1.9 * INSN_COST);
10987   format %{ "negw  $dst, $src1, ASR $src2" %}
10988 
10989   ins_encode %{
10990     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10991             Assembler::ASR, $src2$$constant & 0x1f);
10992   %}
10993 
10994   ins_pipe(ialu_reg_shift);
10995 %}
10996 
10997 // This pattern is automatically generated from aarch64_ad.m4
10998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10999 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11000                               immI0 zero, iRegIorL2I src1, immI src2) %{
11001   match(Set dst (SubI zero (LShiftI src1 src2)));
11002 
11003   ins_cost(1.9 * INSN_COST);
11004   format %{ "negw  $dst, $src1, LSL $src2" %}
11005 
11006   ins_encode %{
11007     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11008             Assembler::LSL, $src2$$constant & 0x1f);
11009   %}
11010 
11011   ins_pipe(ialu_reg_shift);
11012 %}
11013 
11014 // This pattern is automatically generated from aarch64_ad.m4
11015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11016 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11017                               immL0 zero, iRegL src1, immI src2) %{
11018   match(Set dst (SubL zero (URShiftL src1 src2)));
11019 
11020   ins_cost(1.9 * INSN_COST);
11021   format %{ "neg  $dst, $src1, LSR $src2" %}
11022 
11023   ins_encode %{
11024     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11025             Assembler::LSR, $src2$$constant & 0x3f);
11026   %}
11027 
11028   ins_pipe(ialu_reg_shift);
11029 %}
11030 
11031 // This pattern is automatically generated from aarch64_ad.m4
11032 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11033 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11034                               immL0 zero, iRegL src1, immI src2) %{
11035   match(Set dst (SubL zero (RShiftL src1 src2)));
11036 
11037   ins_cost(1.9 * INSN_COST);
11038   format %{ "neg  $dst, $src1, ASR $src2" %}
11039 
11040   ins_encode %{
11041     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11042             Assembler::ASR, $src2$$constant & 0x3f);
11043   %}
11044 
11045   ins_pipe(ialu_reg_shift);
11046 %}
11047 
11048 // This pattern is automatically generated from aarch64_ad.m4
11049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11050 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11051                               immL0 zero, iRegL src1, immI src2) %{
11052   match(Set dst (SubL zero (LShiftL src1 src2)));
11053 
11054   ins_cost(1.9 * INSN_COST);
11055   format %{ "neg  $dst, $src1, LSL $src2" %}
11056 
11057   ins_encode %{
11058     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11059             Assembler::LSL, $src2$$constant & 0x3f);
11060   %}
11061 
11062   ins_pipe(ialu_reg_shift);
11063 %}
11064 
11065 // This pattern is automatically generated from aarch64_ad.m4
11066 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11067 instruct AndI_reg_not_reg(iRegINoSp dst,
11068                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11069   match(Set dst (AndI src1 (XorI src2 m1)));
11070   ins_cost(INSN_COST);
11071   format %{ "bicw  $dst, $src1, $src2" %}
11072 
11073   ins_encode %{
11074     __ bicw(as_Register($dst$$reg),
11075               as_Register($src1$$reg),
11076               as_Register($src2$$reg),
11077               Assembler::LSL, 0);
11078   %}
11079 
11080   ins_pipe(ialu_reg_reg);
11081 %}
11082 
11083 // This pattern is automatically generated from aarch64_ad.m4
11084 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11085 instruct AndL_reg_not_reg(iRegLNoSp dst,
11086                          iRegL src1, iRegL src2, immL_M1 m1) %{
11087   match(Set dst (AndL src1 (XorL src2 m1)));
11088   ins_cost(INSN_COST);
11089   format %{ "bic  $dst, $src1, $src2" %}
11090 
11091   ins_encode %{
11092     __ bic(as_Register($dst$$reg),
11093               as_Register($src1$$reg),
11094               as_Register($src2$$reg),
11095               Assembler::LSL, 0);
11096   %}
11097 
11098   ins_pipe(ialu_reg_reg);
11099 %}
11100 
11101 // This pattern is automatically generated from aarch64_ad.m4
11102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11103 instruct OrI_reg_not_reg(iRegINoSp dst,
11104                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11105   match(Set dst (OrI src1 (XorI src2 m1)));
11106   ins_cost(INSN_COST);
11107   format %{ "ornw  $dst, $src1, $src2" %}
11108 
11109   ins_encode %{
11110     __ ornw(as_Register($dst$$reg),
11111               as_Register($src1$$reg),
11112               as_Register($src2$$reg),
11113               Assembler::LSL, 0);
11114   %}
11115 
11116   ins_pipe(ialu_reg_reg);
11117 %}
11118 
11119 // This pattern is automatically generated from aarch64_ad.m4
11120 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11121 instruct OrL_reg_not_reg(iRegLNoSp dst,
11122                          iRegL src1, iRegL src2, immL_M1 m1) %{
11123   match(Set dst (OrL src1 (XorL src2 m1)));
11124   ins_cost(INSN_COST);
11125   format %{ "orn  $dst, $src1, $src2" %}
11126 
11127   ins_encode %{
11128     __ orn(as_Register($dst$$reg),
11129               as_Register($src1$$reg),
11130               as_Register($src2$$reg),
11131               Assembler::LSL, 0);
11132   %}
11133 
11134   ins_pipe(ialu_reg_reg);
11135 %}
11136 
11137 // This pattern is automatically generated from aarch64_ad.m4
11138 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11139 instruct XorI_reg_not_reg(iRegINoSp dst,
11140                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11141   match(Set dst (XorI m1 (XorI src2 src1)));
11142   ins_cost(INSN_COST);
11143   format %{ "eonw  $dst, $src1, $src2" %}
11144 
11145   ins_encode %{
11146     __ eonw(as_Register($dst$$reg),
11147               as_Register($src1$$reg),
11148               as_Register($src2$$reg),
11149               Assembler::LSL, 0);
11150   %}
11151 
11152   ins_pipe(ialu_reg_reg);
11153 %}
11154 
11155 // This pattern is automatically generated from aarch64_ad.m4
11156 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11157 instruct XorL_reg_not_reg(iRegLNoSp dst,
11158                          iRegL src1, iRegL src2, immL_M1 m1) %{
11159   match(Set dst (XorL m1 (XorL src2 src1)));
11160   ins_cost(INSN_COST);
11161   format %{ "eon  $dst, $src1, $src2" %}
11162 
11163   ins_encode %{
11164     __ eon(as_Register($dst$$reg),
11165               as_Register($src1$$reg),
11166               as_Register($src2$$reg),
11167               Assembler::LSL, 0);
11168   %}
11169 
11170   ins_pipe(ialu_reg_reg);
11171 %}
11172 
11173 // This pattern is automatically generated from aarch64_ad.m4
11174 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11175 // val & (-1 ^ (val >>> shift)) ==> bicw
11176 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11177                          iRegIorL2I src1, iRegIorL2I src2,
11178                          immI src3, immI_M1 src4) %{
11179   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11180   ins_cost(1.9 * INSN_COST);
11181   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11182 
11183   ins_encode %{
11184     __ bicw(as_Register($dst$$reg),
11185               as_Register($src1$$reg),
11186               as_Register($src2$$reg),
11187               Assembler::LSR,
11188               $src3$$constant & 0x1f);
11189   %}
11190 
11191   ins_pipe(ialu_reg_reg_shift);
11192 %}
11193 
11194 // This pattern is automatically generated from aarch64_ad.m4
11195 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11196 // val & (-1 ^ (val >>> shift)) ==> bic
11197 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11198                          iRegL src1, iRegL src2,
11199                          immI src3, immL_M1 src4) %{
11200   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11201   ins_cost(1.9 * INSN_COST);
11202   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11203 
11204   ins_encode %{
11205     __ bic(as_Register($dst$$reg),
11206               as_Register($src1$$reg),
11207               as_Register($src2$$reg),
11208               Assembler::LSR,
11209               $src3$$constant & 0x3f);
11210   %}
11211 
11212   ins_pipe(ialu_reg_reg_shift);
11213 %}
11214 
11215 // This pattern is automatically generated from aarch64_ad.m4
11216 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11217 // val & (-1 ^ (val >> shift)) ==> bicw
11218 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11219                          iRegIorL2I src1, iRegIorL2I src2,
11220                          immI src3, immI_M1 src4) %{
11221   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11222   ins_cost(1.9 * INSN_COST);
11223   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11224 
11225   ins_encode %{
11226     __ bicw(as_Register($dst$$reg),
11227               as_Register($src1$$reg),
11228               as_Register($src2$$reg),
11229               Assembler::ASR,
11230               $src3$$constant & 0x1f);
11231   %}
11232 
11233   ins_pipe(ialu_reg_reg_shift);
11234 %}
11235 
11236 // This pattern is automatically generated from aarch64_ad.m4
11237 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11238 // val & (-1 ^ (val >> shift)) ==> bic
11239 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11240                          iRegL src1, iRegL src2,
11241                          immI src3, immL_M1 src4) %{
11242   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11243   ins_cost(1.9 * INSN_COST);
11244   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11245 
11246   ins_encode %{
11247     __ bic(as_Register($dst$$reg),
11248               as_Register($src1$$reg),
11249               as_Register($src2$$reg),
11250               Assembler::ASR,
11251               $src3$$constant & 0x3f);
11252   %}
11253 
11254   ins_pipe(ialu_reg_reg_shift);
11255 %}
11256 
11257 // This pattern is automatically generated from aarch64_ad.m4
11258 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11259 // val & (-1 ^ (val ror shift)) ==> bicw
11260 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11261                          iRegIorL2I src1, iRegIorL2I src2,
11262                          immI src3, immI_M1 src4) %{
11263   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11264   ins_cost(1.9 * INSN_COST);
11265   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11266 
11267   ins_encode %{
11268     __ bicw(as_Register($dst$$reg),
11269               as_Register($src1$$reg),
11270               as_Register($src2$$reg),
11271               Assembler::ROR,
11272               $src3$$constant & 0x1f);
11273   %}
11274 
11275   ins_pipe(ialu_reg_reg_shift);
11276 %}
11277 
11278 // This pattern is automatically generated from aarch64_ad.m4
11279 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11280 // val & (-1 ^ (val ror shift)) ==> bic
11281 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11282                          iRegL src1, iRegL src2,
11283                          immI src3, immL_M1 src4) %{
11284   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11285   ins_cost(1.9 * INSN_COST);
11286   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11287 
11288   ins_encode %{
11289     __ bic(as_Register($dst$$reg),
11290               as_Register($src1$$reg),
11291               as_Register($src2$$reg),
11292               Assembler::ROR,
11293               $src3$$constant & 0x3f);
11294   %}
11295 
11296   ins_pipe(ialu_reg_reg_shift);
11297 %}
11298 
11299 // This pattern is automatically generated from aarch64_ad.m4
11300 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11301 // val & (-1 ^ (val << shift)) ==> bicw
11302 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11303                          iRegIorL2I src1, iRegIorL2I src2,
11304                          immI src3, immI_M1 src4) %{
11305   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11306   ins_cost(1.9 * INSN_COST);
11307   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11308 
11309   ins_encode %{
11310     __ bicw(as_Register($dst$$reg),
11311               as_Register($src1$$reg),
11312               as_Register($src2$$reg),
11313               Assembler::LSL,
11314               $src3$$constant & 0x1f);
11315   %}
11316 
11317   ins_pipe(ialu_reg_reg_shift);
11318 %}
11319 
11320 // This pattern is automatically generated from aarch64_ad.m4
11321 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11322 // val & (-1 ^ (val << shift)) ==> bic
11323 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11324                          iRegL src1, iRegL src2,
11325                          immI src3, immL_M1 src4) %{
11326   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11327   ins_cost(1.9 * INSN_COST);
11328   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11329 
11330   ins_encode %{
11331     __ bic(as_Register($dst$$reg),
11332               as_Register($src1$$reg),
11333               as_Register($src2$$reg),
11334               Assembler::LSL,
11335               $src3$$constant & 0x3f);
11336   %}
11337 
11338   ins_pipe(ialu_reg_reg_shift);
11339 %}
11340 
11341 // This pattern is automatically generated from aarch64_ad.m4
11342 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11343 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11344 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11345                          iRegIorL2I src1, iRegIorL2I src2,
11346                          immI src3, immI_M1 src4) %{
11347   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11348   ins_cost(1.9 * INSN_COST);
11349   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11350 
11351   ins_encode %{
11352     __ eonw(as_Register($dst$$reg),
11353               as_Register($src1$$reg),
11354               as_Register($src2$$reg),
11355               Assembler::LSR,
11356               $src3$$constant & 0x1f);
11357   %}
11358 
11359   ins_pipe(ialu_reg_reg_shift);
11360 %}
11361 
11362 // This pattern is automatically generated from aarch64_ad.m4
11363 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11364 // val ^ (-1 ^ (val >>> shift)) ==> eon
11365 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11366                          iRegL src1, iRegL src2,
11367                          immI src3, immL_M1 src4) %{
11368   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11369   ins_cost(1.9 * INSN_COST);
11370   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11371 
11372   ins_encode %{
11373     __ eon(as_Register($dst$$reg),
11374               as_Register($src1$$reg),
11375               as_Register($src2$$reg),
11376               Assembler::LSR,
11377               $src3$$constant & 0x3f);
11378   %}
11379 
11380   ins_pipe(ialu_reg_reg_shift);
11381 %}
11382 
11383 // This pattern is automatically generated from aarch64_ad.m4
11384 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11385 // val ^ (-1 ^ (val >> shift)) ==> eonw
11386 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11387                          iRegIorL2I src1, iRegIorL2I src2,
11388                          immI src3, immI_M1 src4) %{
11389   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11390   ins_cost(1.9 * INSN_COST);
11391   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11392 
11393   ins_encode %{
11394     __ eonw(as_Register($dst$$reg),
11395               as_Register($src1$$reg),
11396               as_Register($src2$$reg),
11397               Assembler::ASR,
11398               $src3$$constant & 0x1f);
11399   %}
11400 
11401   ins_pipe(ialu_reg_reg_shift);
11402 %}
11403 
11404 // This pattern is automatically generated from aarch64_ad.m4
11405 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11406 // val ^ (-1 ^ (val >> shift)) ==> eon
11407 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11408                          iRegL src1, iRegL src2,
11409                          immI src3, immL_M1 src4) %{
11410   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11411   ins_cost(1.9 * INSN_COST);
11412   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11413 
11414   ins_encode %{
11415     __ eon(as_Register($dst$$reg),
11416               as_Register($src1$$reg),
11417               as_Register($src2$$reg),
11418               Assembler::ASR,
11419               $src3$$constant & 0x3f);
11420   %}
11421 
11422   ins_pipe(ialu_reg_reg_shift);
11423 %}
11424 
11425 // This pattern is automatically generated from aarch64_ad.m4
11426 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11427 // val ^ (-1 ^ (val ror shift)) ==> eonw
11428 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11429                          iRegIorL2I src1, iRegIorL2I src2,
11430                          immI src3, immI_M1 src4) %{
11431   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11432   ins_cost(1.9 * INSN_COST);
11433   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11434 
11435   ins_encode %{
11436     __ eonw(as_Register($dst$$reg),
11437               as_Register($src1$$reg),
11438               as_Register($src2$$reg),
11439               Assembler::ROR,
11440               $src3$$constant & 0x1f);
11441   %}
11442 
11443   ins_pipe(ialu_reg_reg_shift);
11444 %}
11445 
11446 // This pattern is automatically generated from aarch64_ad.m4
11447 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11448 // val ^ (-1 ^ (val ror shift)) ==> eon
11449 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11450                          iRegL src1, iRegL src2,
11451                          immI src3, immL_M1 src4) %{
11452   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11453   ins_cost(1.9 * INSN_COST);
11454   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11455 
11456   ins_encode %{
11457     __ eon(as_Register($dst$$reg),
11458               as_Register($src1$$reg),
11459               as_Register($src2$$reg),
11460               Assembler::ROR,
11461               $src3$$constant & 0x3f);
11462   %}
11463 
11464   ins_pipe(ialu_reg_reg_shift);
11465 %}
11466 
11467 // This pattern is automatically generated from aarch64_ad.m4
11468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11469 // val ^ (-1 ^ (val << shift)) ==> eonw
11470 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11471                          iRegIorL2I src1, iRegIorL2I src2,
11472                          immI src3, immI_M1 src4) %{
11473   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11474   ins_cost(1.9 * INSN_COST);
11475   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11476 
11477   ins_encode %{
11478     __ eonw(as_Register($dst$$reg),
11479               as_Register($src1$$reg),
11480               as_Register($src2$$reg),
11481               Assembler::LSL,
11482               $src3$$constant & 0x1f);
11483   %}
11484 
11485   ins_pipe(ialu_reg_reg_shift);
11486 %}
11487 
11488 // This pattern is automatically generated from aarch64_ad.m4
11489 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11490 // val ^ (-1 ^ (val << shift)) ==> eon
11491 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11492                          iRegL src1, iRegL src2,
11493                          immI src3, immL_M1 src4) %{
11494   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11495   ins_cost(1.9 * INSN_COST);
11496   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11497 
11498   ins_encode %{
11499     __ eon(as_Register($dst$$reg),
11500               as_Register($src1$$reg),
11501               as_Register($src2$$reg),
11502               Assembler::LSL,
11503               $src3$$constant & 0x3f);
11504   %}
11505 
11506   ins_pipe(ialu_reg_reg_shift);
11507 %}
11508 
11509 // This pattern is automatically generated from aarch64_ad.m4
11510 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11511 // val | (-1 ^ (val >>> shift)) ==> ornw
11512 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11513                          iRegIorL2I src1, iRegIorL2I src2,
11514                          immI src3, immI_M1 src4) %{
11515   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11516   ins_cost(1.9 * INSN_COST);
11517   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11518 
11519   ins_encode %{
11520     __ ornw(as_Register($dst$$reg),
11521               as_Register($src1$$reg),
11522               as_Register($src2$$reg),
11523               Assembler::LSR,
11524               $src3$$constant & 0x1f);
11525   %}
11526 
11527   ins_pipe(ialu_reg_reg_shift);
11528 %}
11529 
11530 // This pattern is automatically generated from aarch64_ad.m4
11531 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11532 // val | (-1 ^ (val >>> shift)) ==> orn
11533 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11534                          iRegL src1, iRegL src2,
11535                          immI src3, immL_M1 src4) %{
11536   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11537   ins_cost(1.9 * INSN_COST);
11538   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11539 
11540   ins_encode %{
11541     __ orn(as_Register($dst$$reg),
11542               as_Register($src1$$reg),
11543               as_Register($src2$$reg),
11544               Assembler::LSR,
11545               $src3$$constant & 0x3f);
11546   %}
11547 
11548   ins_pipe(ialu_reg_reg_shift);
11549 %}
11550 
11551 // This pattern is automatically generated from aarch64_ad.m4
11552 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11553 // val | (-1 ^ (val >> shift)) ==> ornw
11554 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11555                          iRegIorL2I src1, iRegIorL2I src2,
11556                          immI src3, immI_M1 src4) %{
11557   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11558   ins_cost(1.9 * INSN_COST);
11559   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11560 
11561   ins_encode %{
11562     __ ornw(as_Register($dst$$reg),
11563               as_Register($src1$$reg),
11564               as_Register($src2$$reg),
11565               Assembler::ASR,
11566               $src3$$constant & 0x1f);
11567   %}
11568 
11569   ins_pipe(ialu_reg_reg_shift);
11570 %}
11571 
11572 // This pattern is automatically generated from aarch64_ad.m4
11573 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11574 // val | (-1 ^ (val >> shift)) ==> orn
11575 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11576                          iRegL src1, iRegL src2,
11577                          immI src3, immL_M1 src4) %{
11578   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11579   ins_cost(1.9 * INSN_COST);
11580   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11581 
11582   ins_encode %{
11583     __ orn(as_Register($dst$$reg),
11584               as_Register($src1$$reg),
11585               as_Register($src2$$reg),
11586               Assembler::ASR,
11587               $src3$$constant & 0x3f);
11588   %}
11589 
11590   ins_pipe(ialu_reg_reg_shift);
11591 %}
11592 
11593 // This pattern is automatically generated from aarch64_ad.m4
11594 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11595 // val | (-1 ^ (val ror shift)) ==> ornw
11596 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11597                          iRegIorL2I src1, iRegIorL2I src2,
11598                          immI src3, immI_M1 src4) %{
11599   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11600   ins_cost(1.9 * INSN_COST);
11601   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11602 
11603   ins_encode %{
11604     __ ornw(as_Register($dst$$reg),
11605               as_Register($src1$$reg),
11606               as_Register($src2$$reg),
11607               Assembler::ROR,
11608               $src3$$constant & 0x1f);
11609   %}
11610 
11611   ins_pipe(ialu_reg_reg_shift);
11612 %}
11613 
11614 // This pattern is automatically generated from aarch64_ad.m4
11615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11616 // val | (-1 ^ (val ror shift)) ==> orn
11617 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11618                          iRegL src1, iRegL src2,
11619                          immI src3, immL_M1 src4) %{
11620   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11621   ins_cost(1.9 * INSN_COST);
11622   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11623 
11624   ins_encode %{
11625     __ orn(as_Register($dst$$reg),
11626               as_Register($src1$$reg),
11627               as_Register($src2$$reg),
11628               Assembler::ROR,
11629               $src3$$constant & 0x3f);
11630   %}
11631 
11632   ins_pipe(ialu_reg_reg_shift);
11633 %}
11634 
11635 // This pattern is automatically generated from aarch64_ad.m4
11636 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11637 // val | (-1 ^ (val << shift)) ==> ornw
11638 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11639                          iRegIorL2I src1, iRegIorL2I src2,
11640                          immI src3, immI_M1 src4) %{
11641   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11642   ins_cost(1.9 * INSN_COST);
11643   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11644 
11645   ins_encode %{
11646     __ ornw(as_Register($dst$$reg),
11647               as_Register($src1$$reg),
11648               as_Register($src2$$reg),
11649               Assembler::LSL,
11650               $src3$$constant & 0x1f);
11651   %}
11652 
11653   ins_pipe(ialu_reg_reg_shift);
11654 %}
11655 
11656 // This pattern is automatically generated from aarch64_ad.m4
11657 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11658 // val | (-1 ^ (val << shift)) ==> orn
11659 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11660                          iRegL src1, iRegL src2,
11661                          immI src3, immL_M1 src4) %{
11662   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11663   ins_cost(1.9 * INSN_COST);
11664   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11665 
11666   ins_encode %{
11667     __ orn(as_Register($dst$$reg),
11668               as_Register($src1$$reg),
11669               as_Register($src2$$reg),
11670               Assembler::LSL,
11671               $src3$$constant & 0x3f);
11672   %}
11673 
11674   ins_pipe(ialu_reg_reg_shift);
11675 %}
11676 
11677 // This pattern is automatically generated from aarch64_ad.m4
11678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11679 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11680                          iRegIorL2I src1, iRegIorL2I src2,
11681                          immI src3) %{
11682   match(Set dst (AndI src1 (URShiftI src2 src3)));
11683 
11684   ins_cost(1.9 * INSN_COST);
11685   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
11686 
11687   ins_encode %{
11688     __ andw(as_Register($dst$$reg),
11689               as_Register($src1$$reg),
11690               as_Register($src2$$reg),
11691               Assembler::LSR,
11692               $src3$$constant & 0x1f);
11693   %}
11694 
11695   ins_pipe(ialu_reg_reg_shift);
11696 %}
11697 
11698 // This pattern is automatically generated from aarch64_ad.m4
11699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11700 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11701                          iRegL src1, iRegL src2,
11702                          immI src3) %{
11703   match(Set dst (AndL src1 (URShiftL src2 src3)));
11704 
11705   ins_cost(1.9 * INSN_COST);
11706   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
11707 
11708   ins_encode %{
11709     __ andr(as_Register($dst$$reg),
11710               as_Register($src1$$reg),
11711               as_Register($src2$$reg),
11712               Assembler::LSR,
11713               $src3$$constant & 0x3f);
11714   %}
11715 
11716   ins_pipe(ialu_reg_reg_shift);
11717 %}
11718 
11719 // This pattern is automatically generated from aarch64_ad.m4
11720 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11721 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11722                          iRegIorL2I src1, iRegIorL2I src2,
11723                          immI src3) %{
11724   match(Set dst (AndI src1 (RShiftI src2 src3)));
11725 
11726   ins_cost(1.9 * INSN_COST);
11727   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
11728 
11729   ins_encode %{
11730     __ andw(as_Register($dst$$reg),
11731               as_Register($src1$$reg),
11732               as_Register($src2$$reg),
11733               Assembler::ASR,
11734               $src3$$constant & 0x1f);
11735   %}
11736 
11737   ins_pipe(ialu_reg_reg_shift);
11738 %}
11739 
11740 // This pattern is automatically generated from aarch64_ad.m4
11741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11742 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11743                          iRegL src1, iRegL src2,
11744                          immI src3) %{
11745   match(Set dst (AndL src1 (RShiftL src2 src3)));
11746 
11747   ins_cost(1.9 * INSN_COST);
11748   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
11749 
11750   ins_encode %{
11751     __ andr(as_Register($dst$$reg),
11752               as_Register($src1$$reg),
11753               as_Register($src2$$reg),
11754               Assembler::ASR,
11755               $src3$$constant & 0x3f);
11756   %}
11757 
11758   ins_pipe(ialu_reg_reg_shift);
11759 %}
11760 
11761 // This pattern is automatically generated from aarch64_ad.m4
11762 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11763 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11764                          iRegIorL2I src1, iRegIorL2I src2,
11765                          immI src3) %{
11766   match(Set dst (AndI src1 (LShiftI src2 src3)));
11767 
11768   ins_cost(1.9 * INSN_COST);
11769   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
11770 
11771   ins_encode %{
11772     __ andw(as_Register($dst$$reg),
11773               as_Register($src1$$reg),
11774               as_Register($src2$$reg),
11775               Assembler::LSL,
11776               $src3$$constant & 0x1f);
11777   %}
11778 
11779   ins_pipe(ialu_reg_reg_shift);
11780 %}
11781 
11782 // This pattern is automatically generated from aarch64_ad.m4
11783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11784 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11785                          iRegL src1, iRegL src2,
11786                          immI src3) %{
11787   match(Set dst (AndL src1 (LShiftL src2 src3)));
11788 
11789   ins_cost(1.9 * INSN_COST);
11790   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
11791 
11792   ins_encode %{
11793     __ andr(as_Register($dst$$reg),
11794               as_Register($src1$$reg),
11795               as_Register($src2$$reg),
11796               Assembler::LSL,
11797               $src3$$constant & 0x3f);
11798   %}
11799 
11800   ins_pipe(ialu_reg_reg_shift);
11801 %}
11802 
11803 // This pattern is automatically generated from aarch64_ad.m4
11804 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11805 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11806                          iRegIorL2I src1, iRegIorL2I src2,
11807                          immI src3) %{
11808   match(Set dst (AndI src1 (RotateRight src2 src3)));
11809 
11810   ins_cost(1.9 * INSN_COST);
11811   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
11812 
11813   ins_encode %{
11814     __ andw(as_Register($dst$$reg),
11815               as_Register($src1$$reg),
11816               as_Register($src2$$reg),
11817               Assembler::ROR,
11818               $src3$$constant & 0x1f);
11819   %}
11820 
11821   ins_pipe(ialu_reg_reg_shift);
11822 %}
11823 
11824 // This pattern is automatically generated from aarch64_ad.m4
11825 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11826 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11827                          iRegL src1, iRegL src2,
11828                          immI src3) %{
11829   match(Set dst (AndL src1 (RotateRight src2 src3)));
11830 
11831   ins_cost(1.9 * INSN_COST);
11832   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
11833 
11834   ins_encode %{
11835     __ andr(as_Register($dst$$reg),
11836               as_Register($src1$$reg),
11837               as_Register($src2$$reg),
11838               Assembler::ROR,
11839               $src3$$constant & 0x3f);
11840   %}
11841 
11842   ins_pipe(ialu_reg_reg_shift);
11843 %}
11844 
11845 // This pattern is automatically generated from aarch64_ad.m4
11846 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11847 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11848                          iRegIorL2I src1, iRegIorL2I src2,
11849                          immI src3) %{
11850   match(Set dst (XorI src1 (URShiftI src2 src3)));
11851 
11852   ins_cost(1.9 * INSN_COST);
11853   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
11854 
11855   ins_encode %{
11856     __ eorw(as_Register($dst$$reg),
11857               as_Register($src1$$reg),
11858               as_Register($src2$$reg),
11859               Assembler::LSR,
11860               $src3$$constant & 0x1f);
11861   %}
11862 
11863   ins_pipe(ialu_reg_reg_shift);
11864 %}
11865 
11866 // This pattern is automatically generated from aarch64_ad.m4
11867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11868 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11869                          iRegL src1, iRegL src2,
11870                          immI src3) %{
11871   match(Set dst (XorL src1 (URShiftL src2 src3)));
11872 
11873   ins_cost(1.9 * INSN_COST);
11874   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
11875 
11876   ins_encode %{
11877     __ eor(as_Register($dst$$reg),
11878               as_Register($src1$$reg),
11879               as_Register($src2$$reg),
11880               Assembler::LSR,
11881               $src3$$constant & 0x3f);
11882   %}
11883 
11884   ins_pipe(ialu_reg_reg_shift);
11885 %}
11886 
11887 // This pattern is automatically generated from aarch64_ad.m4
11888 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11889 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11890                          iRegIorL2I src1, iRegIorL2I src2,
11891                          immI src3) %{
11892   match(Set dst (XorI src1 (RShiftI src2 src3)));
11893 
11894   ins_cost(1.9 * INSN_COST);
11895   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
11896 
11897   ins_encode %{
11898     __ eorw(as_Register($dst$$reg),
11899               as_Register($src1$$reg),
11900               as_Register($src2$$reg),
11901               Assembler::ASR,
11902               $src3$$constant & 0x1f);
11903   %}
11904 
11905   ins_pipe(ialu_reg_reg_shift);
11906 %}
11907 
11908 // This pattern is automatically generated from aarch64_ad.m4
11909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11910 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11911                          iRegL src1, iRegL src2,
11912                          immI src3) %{
11913   match(Set dst (XorL src1 (RShiftL src2 src3)));
11914 
11915   ins_cost(1.9 * INSN_COST);
11916   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
11917 
11918   ins_encode %{
11919     __ eor(as_Register($dst$$reg),
11920               as_Register($src1$$reg),
11921               as_Register($src2$$reg),
11922               Assembler::ASR,
11923               $src3$$constant & 0x3f);
11924   %}
11925 
11926   ins_pipe(ialu_reg_reg_shift);
11927 %}
11928 
11929 // This pattern is automatically generated from aarch64_ad.m4
11930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11931 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11932                          iRegIorL2I src1, iRegIorL2I src2,
11933                          immI src3) %{
11934   match(Set dst (XorI src1 (LShiftI src2 src3)));
11935 
11936   ins_cost(1.9 * INSN_COST);
11937   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
11938 
11939   ins_encode %{
11940     __ eorw(as_Register($dst$$reg),
11941               as_Register($src1$$reg),
11942               as_Register($src2$$reg),
11943               Assembler::LSL,
11944               $src3$$constant & 0x1f);
11945   %}
11946 
11947   ins_pipe(ialu_reg_reg_shift);
11948 %}
11949 
11950 // This pattern is automatically generated from aarch64_ad.m4
11951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11952 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11953                          iRegL src1, iRegL src2,
11954                          immI src3) %{
11955   match(Set dst (XorL src1 (LShiftL src2 src3)));
11956 
11957   ins_cost(1.9 * INSN_COST);
11958   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
11959 
11960   ins_encode %{
11961     __ eor(as_Register($dst$$reg),
11962               as_Register($src1$$reg),
11963               as_Register($src2$$reg),
11964               Assembler::LSL,
11965               $src3$$constant & 0x3f);
11966   %}
11967 
11968   ins_pipe(ialu_reg_reg_shift);
11969 %}
11970 
11971 // This pattern is automatically generated from aarch64_ad.m4
11972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11973 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11974                          iRegIorL2I src1, iRegIorL2I src2,
11975                          immI src3) %{
11976   match(Set dst (XorI src1 (RotateRight src2 src3)));
11977 
11978   ins_cost(1.9 * INSN_COST);
11979   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
11980 
11981   ins_encode %{
11982     __ eorw(as_Register($dst$$reg),
11983               as_Register($src1$$reg),
11984               as_Register($src2$$reg),
11985               Assembler::ROR,
11986               $src3$$constant & 0x1f);
11987   %}
11988 
11989   ins_pipe(ialu_reg_reg_shift);
11990 %}
11991 
11992 // This pattern is automatically generated from aarch64_ad.m4
11993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11994 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11995                          iRegL src1, iRegL src2,
11996                          immI src3) %{
11997   match(Set dst (XorL src1 (RotateRight src2 src3)));
11998 
11999   ins_cost(1.9 * INSN_COST);
12000   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12001 
12002   ins_encode %{
12003     __ eor(as_Register($dst$$reg),
12004               as_Register($src1$$reg),
12005               as_Register($src2$$reg),
12006               Assembler::ROR,
12007               $src3$$constant & 0x3f);
12008   %}
12009 
12010   ins_pipe(ialu_reg_reg_shift);
12011 %}
12012 
12013 // This pattern is automatically generated from aarch64_ad.m4
12014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12015 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12016                          iRegIorL2I src1, iRegIorL2I src2,
12017                          immI src3) %{
12018   match(Set dst (OrI src1 (URShiftI src2 src3)));
12019 
12020   ins_cost(1.9 * INSN_COST);
12021   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12022 
12023   ins_encode %{
12024     __ orrw(as_Register($dst$$reg),
12025               as_Register($src1$$reg),
12026               as_Register($src2$$reg),
12027               Assembler::LSR,
12028               $src3$$constant & 0x1f);
12029   %}
12030 
12031   ins_pipe(ialu_reg_reg_shift);
12032 %}
12033 
12034 // This pattern is automatically generated from aarch64_ad.m4
12035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12036 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12037                          iRegL src1, iRegL src2,
12038                          immI src3) %{
12039   match(Set dst (OrL src1 (URShiftL src2 src3)));
12040 
12041   ins_cost(1.9 * INSN_COST);
12042   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12043 
12044   ins_encode %{
12045     __ orr(as_Register($dst$$reg),
12046               as_Register($src1$$reg),
12047               as_Register($src2$$reg),
12048               Assembler::LSR,
12049               $src3$$constant & 0x3f);
12050   %}
12051 
12052   ins_pipe(ialu_reg_reg_shift);
12053 %}
12054 
12055 // This pattern is automatically generated from aarch64_ad.m4
12056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12057 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12058                          iRegIorL2I src1, iRegIorL2I src2,
12059                          immI src3) %{
12060   match(Set dst (OrI src1 (RShiftI src2 src3)));
12061 
12062   ins_cost(1.9 * INSN_COST);
12063   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12064 
12065   ins_encode %{
12066     __ orrw(as_Register($dst$$reg),
12067               as_Register($src1$$reg),
12068               as_Register($src2$$reg),
12069               Assembler::ASR,
12070               $src3$$constant & 0x1f);
12071   %}
12072 
12073   ins_pipe(ialu_reg_reg_shift);
12074 %}
12075 
12076 // This pattern is automatically generated from aarch64_ad.m4
12077 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12078 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12079                          iRegL src1, iRegL src2,
12080                          immI src3) %{
12081   match(Set dst (OrL src1 (RShiftL src2 src3)));
12082 
12083   ins_cost(1.9 * INSN_COST);
12084   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12085 
12086   ins_encode %{
12087     __ orr(as_Register($dst$$reg),
12088               as_Register($src1$$reg),
12089               as_Register($src2$$reg),
12090               Assembler::ASR,
12091               $src3$$constant & 0x3f);
12092   %}
12093 
12094   ins_pipe(ialu_reg_reg_shift);
12095 %}
12096 
12097 // This pattern is automatically generated from aarch64_ad.m4
12098 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12099 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12100                          iRegIorL2I src1, iRegIorL2I src2,
12101                          immI src3) %{
12102   match(Set dst (OrI src1 (LShiftI src2 src3)));
12103 
12104   ins_cost(1.9 * INSN_COST);
12105   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12106 
12107   ins_encode %{
12108     __ orrw(as_Register($dst$$reg),
12109               as_Register($src1$$reg),
12110               as_Register($src2$$reg),
12111               Assembler::LSL,
12112               $src3$$constant & 0x1f);
12113   %}
12114 
12115   ins_pipe(ialu_reg_reg_shift);
12116 %}
12117 
12118 // This pattern is automatically generated from aarch64_ad.m4
12119 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12120 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12121                          iRegL src1, iRegL src2,
12122                          immI src3) %{
12123   match(Set dst (OrL src1 (LShiftL src2 src3)));
12124 
12125   ins_cost(1.9 * INSN_COST);
12126   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12127 
12128   ins_encode %{
12129     __ orr(as_Register($dst$$reg),
12130               as_Register($src1$$reg),
12131               as_Register($src2$$reg),
12132               Assembler::LSL,
12133               $src3$$constant & 0x3f);
12134   %}
12135 
12136   ins_pipe(ialu_reg_reg_shift);
12137 %}
12138 
12139 // This pattern is automatically generated from aarch64_ad.m4
12140 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12141 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12142                          iRegIorL2I src1, iRegIorL2I src2,
12143                          immI src3) %{
12144   match(Set dst (OrI src1 (RotateRight src2 src3)));
12145 
12146   ins_cost(1.9 * INSN_COST);
12147   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12148 
12149   ins_encode %{
12150     __ orrw(as_Register($dst$$reg),
12151               as_Register($src1$$reg),
12152               as_Register($src2$$reg),
12153               Assembler::ROR,
12154               $src3$$constant & 0x1f);
12155   %}
12156 
12157   ins_pipe(ialu_reg_reg_shift);
12158 %}
12159 
12160 // This pattern is automatically generated from aarch64_ad.m4
12161 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12162 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12163                          iRegL src1, iRegL src2,
12164                          immI src3) %{
12165   match(Set dst (OrL src1 (RotateRight src2 src3)));
12166 
12167   ins_cost(1.9 * INSN_COST);
12168   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12169 
12170   ins_encode %{
12171     __ orr(as_Register($dst$$reg),
12172               as_Register($src1$$reg),
12173               as_Register($src2$$reg),
12174               Assembler::ROR,
12175               $src3$$constant & 0x3f);
12176   %}
12177 
12178   ins_pipe(ialu_reg_reg_shift);
12179 %}
12180 
12181 // This pattern is automatically generated from aarch64_ad.m4
12182 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12183 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12184                          iRegIorL2I src1, iRegIorL2I src2,
12185                          immI src3) %{
12186   match(Set dst (AddI src1 (URShiftI src2 src3)));
12187 
12188   ins_cost(1.9 * INSN_COST);
12189   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12190 
12191   ins_encode %{
12192     __ addw(as_Register($dst$$reg),
12193               as_Register($src1$$reg),
12194               as_Register($src2$$reg),
12195               Assembler::LSR,
12196               $src3$$constant & 0x1f);
12197   %}
12198 
12199   ins_pipe(ialu_reg_reg_shift);
12200 %}
12201 
12202 // This pattern is automatically generated from aarch64_ad.m4
12203 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12204 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12205                          iRegL src1, iRegL src2,
12206                          immI src3) %{
12207   match(Set dst (AddL src1 (URShiftL src2 src3)));
12208 
12209   ins_cost(1.9 * INSN_COST);
12210   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12211 
12212   ins_encode %{
12213     __ add(as_Register($dst$$reg),
12214               as_Register($src1$$reg),
12215               as_Register($src2$$reg),
12216               Assembler::LSR,
12217               $src3$$constant & 0x3f);
12218   %}
12219 
12220   ins_pipe(ialu_reg_reg_shift);
12221 %}
12222 
12223 // This pattern is automatically generated from aarch64_ad.m4
12224 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12225 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12226                          iRegIorL2I src1, iRegIorL2I src2,
12227                          immI src3) %{
12228   match(Set dst (AddI src1 (RShiftI src2 src3)));
12229 
12230   ins_cost(1.9 * INSN_COST);
12231   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12232 
12233   ins_encode %{
12234     __ addw(as_Register($dst$$reg),
12235               as_Register($src1$$reg),
12236               as_Register($src2$$reg),
12237               Assembler::ASR,
12238               $src3$$constant & 0x1f);
12239   %}
12240 
12241   ins_pipe(ialu_reg_reg_shift);
12242 %}
12243 
12244 // This pattern is automatically generated from aarch64_ad.m4
12245 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12246 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12247                          iRegL src1, iRegL src2,
12248                          immI src3) %{
12249   match(Set dst (AddL src1 (RShiftL src2 src3)));
12250 
12251   ins_cost(1.9 * INSN_COST);
12252   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12253 
12254   ins_encode %{
12255     __ add(as_Register($dst$$reg),
12256               as_Register($src1$$reg),
12257               as_Register($src2$$reg),
12258               Assembler::ASR,
12259               $src3$$constant & 0x3f);
12260   %}
12261 
12262   ins_pipe(ialu_reg_reg_shift);
12263 %}
12264 
12265 // This pattern is automatically generated from aarch64_ad.m4
12266 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12267 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12268                          iRegIorL2I src1, iRegIorL2I src2,
12269                          immI src3) %{
12270   match(Set dst (AddI src1 (LShiftI src2 src3)));
12271 
12272   ins_cost(1.9 * INSN_COST);
12273   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12274 
12275   ins_encode %{
12276     __ addw(as_Register($dst$$reg),
12277               as_Register($src1$$reg),
12278               as_Register($src2$$reg),
12279               Assembler::LSL,
12280               $src3$$constant & 0x1f);
12281   %}
12282 
12283   ins_pipe(ialu_reg_reg_shift);
12284 %}
12285 
12286 // This pattern is automatically generated from aarch64_ad.m4
12287 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12288 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12289                          iRegL src1, iRegL src2,
12290                          immI src3) %{
12291   match(Set dst (AddL src1 (LShiftL src2 src3)));
12292 
12293   ins_cost(1.9 * INSN_COST);
12294   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12295 
12296   ins_encode %{
12297     __ add(as_Register($dst$$reg),
12298               as_Register($src1$$reg),
12299               as_Register($src2$$reg),
12300               Assembler::LSL,
12301               $src3$$constant & 0x3f);
12302   %}
12303 
12304   ins_pipe(ialu_reg_reg_shift);
12305 %}
12306 
12307 // This pattern is automatically generated from aarch64_ad.m4
12308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12309 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12310                          iRegIorL2I src1, iRegIorL2I src2,
12311                          immI src3) %{
12312   match(Set dst (SubI src1 (URShiftI src2 src3)));
12313 
12314   ins_cost(1.9 * INSN_COST);
12315   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12316 
12317   ins_encode %{
12318     __ subw(as_Register($dst$$reg),
12319               as_Register($src1$$reg),
12320               as_Register($src2$$reg),
12321               Assembler::LSR,
12322               $src3$$constant & 0x1f);
12323   %}
12324 
12325   ins_pipe(ialu_reg_reg_shift);
12326 %}
12327 
12328 // This pattern is automatically generated from aarch64_ad.m4
12329 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12330 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12331                          iRegL src1, iRegL src2,
12332                          immI src3) %{
12333   match(Set dst (SubL src1 (URShiftL src2 src3)));
12334 
12335   ins_cost(1.9 * INSN_COST);
12336   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12337 
12338   ins_encode %{
12339     __ sub(as_Register($dst$$reg),
12340               as_Register($src1$$reg),
12341               as_Register($src2$$reg),
12342               Assembler::LSR,
12343               $src3$$constant & 0x3f);
12344   %}
12345 
12346   ins_pipe(ialu_reg_reg_shift);
12347 %}
12348 
12349 // This pattern is automatically generated from aarch64_ad.m4
12350 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12351 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12352                          iRegIorL2I src1, iRegIorL2I src2,
12353                          immI src3) %{
12354   match(Set dst (SubI src1 (RShiftI src2 src3)));
12355 
12356   ins_cost(1.9 * INSN_COST);
12357   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12358 
12359   ins_encode %{
12360     __ subw(as_Register($dst$$reg),
12361               as_Register($src1$$reg),
12362               as_Register($src2$$reg),
12363               Assembler::ASR,
12364               $src3$$constant & 0x1f);
12365   %}
12366 
12367   ins_pipe(ialu_reg_reg_shift);
12368 %}
12369 
12370 // This pattern is automatically generated from aarch64_ad.m4
12371 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12372 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12373                          iRegL src1, iRegL src2,
12374                          immI src3) %{
12375   match(Set dst (SubL src1 (RShiftL src2 src3)));
12376 
12377   ins_cost(1.9 * INSN_COST);
12378   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12379 
12380   ins_encode %{
12381     __ sub(as_Register($dst$$reg),
12382               as_Register($src1$$reg),
12383               as_Register($src2$$reg),
12384               Assembler::ASR,
12385               $src3$$constant & 0x3f);
12386   %}
12387 
12388   ins_pipe(ialu_reg_reg_shift);
12389 %}
12390 
12391 // This pattern is automatically generated from aarch64_ad.m4
12392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12393 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12394                          iRegIorL2I src1, iRegIorL2I src2,
12395                          immI src3) %{
12396   match(Set dst (SubI src1 (LShiftI src2 src3)));
12397 
12398   ins_cost(1.9 * INSN_COST);
12399   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12400 
12401   ins_encode %{
12402     __ subw(as_Register($dst$$reg),
12403               as_Register($src1$$reg),
12404               as_Register($src2$$reg),
12405               Assembler::LSL,
12406               $src3$$constant & 0x1f);
12407   %}
12408 
12409   ins_pipe(ialu_reg_reg_shift);
12410 %}
12411 
12412 // This pattern is automatically generated from aarch64_ad.m4
12413 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12414 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12415                          iRegL src1, iRegL src2,
12416                          immI src3) %{
12417   match(Set dst (SubL src1 (LShiftL src2 src3)));
12418 
12419   ins_cost(1.9 * INSN_COST);
12420   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12421 
12422   ins_encode %{
12423     __ sub(as_Register($dst$$reg),
12424               as_Register($src1$$reg),
12425               as_Register($src2$$reg),
12426               Assembler::LSL,
12427               $src3$$constant & 0x3f);
12428   %}
12429 
12430   ins_pipe(ialu_reg_reg_shift);
12431 %}
12432 
12433 // This pattern is automatically generated from aarch64_ad.m4
12434 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12435 
12436 // Shift Left followed by Shift Right.
12437 // This idiom is used by the compiler for the i2b bytecode etc.
12438 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12439 %{
12440   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12441   ins_cost(INSN_COST * 2);
12442   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12443   ins_encode %{
12444     int lshift = $lshift_count$$constant & 63;
12445     int rshift = $rshift_count$$constant & 63;
12446     int s = 63 - lshift;
12447     int r = (rshift - lshift) & 63;
12448     __ sbfm(as_Register($dst$$reg),
12449             as_Register($src$$reg),
12450             r, s);
12451   %}
12452 
12453   ins_pipe(ialu_reg_shift);
12454 %}
12455 
12456 // This pattern is automatically generated from aarch64_ad.m4
12457 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12458 
12459 // Shift Left followed by Shift Right.
12460 // This idiom is used by the compiler for the i2b bytecode etc.
12461 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12462 %{
12463   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12464   ins_cost(INSN_COST * 2);
12465   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12466   ins_encode %{
12467     int lshift = $lshift_count$$constant & 31;
12468     int rshift = $rshift_count$$constant & 31;
12469     int s = 31 - lshift;
12470     int r = (rshift - lshift) & 31;
12471     __ sbfmw(as_Register($dst$$reg),
12472             as_Register($src$$reg),
12473             r, s);
12474   %}
12475 
12476   ins_pipe(ialu_reg_shift);
12477 %}
12478 
12479 // This pattern is automatically generated from aarch64_ad.m4
12480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12481 
12482 // Shift Left followed by Shift Right.
12483 // This idiom is used by the compiler for the i2b bytecode etc.
12484 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12485 %{
12486   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12487   ins_cost(INSN_COST * 2);
12488   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12489   ins_encode %{
12490     int lshift = $lshift_count$$constant & 63;
12491     int rshift = $rshift_count$$constant & 63;
12492     int s = 63 - lshift;
12493     int r = (rshift - lshift) & 63;
12494     __ ubfm(as_Register($dst$$reg),
12495             as_Register($src$$reg),
12496             r, s);
12497   %}
12498 
12499   ins_pipe(ialu_reg_shift);
12500 %}
12501 
12502 // This pattern is automatically generated from aarch64_ad.m4
12503 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12504 
12505 // Shift Left followed by Shift Right.
12506 // This idiom is used by the compiler for the i2b bytecode etc.
12507 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12508 %{
12509   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12510   ins_cost(INSN_COST * 2);
12511   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12512   ins_encode %{
12513     int lshift = $lshift_count$$constant & 31;
12514     int rshift = $rshift_count$$constant & 31;
12515     int s = 31 - lshift;
12516     int r = (rshift - lshift) & 31;
12517     __ ubfmw(as_Register($dst$$reg),
12518             as_Register($src$$reg),
12519             r, s);
12520   %}
12521 
12522   ins_pipe(ialu_reg_shift);
12523 %}
12524 
12525 // Bitfield extract with shift & mask
12526 
12527 // This pattern is automatically generated from aarch64_ad.m4
12528 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12529 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12530 %{
12531   match(Set dst (AndI (URShiftI src rshift) mask));
12532   // Make sure we are not going to exceed what ubfxw can do.
12533   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12534 
12535   ins_cost(INSN_COST);
12536   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12537   ins_encode %{
12538     int rshift = $rshift$$constant & 31;
12539     intptr_t mask = $mask$$constant;
12540     int width = exact_log2(mask+1);
12541     __ ubfxw(as_Register($dst$$reg),
12542             as_Register($src$$reg), rshift, width);
12543   %}
12544   ins_pipe(ialu_reg_shift);
12545 %}
12546 
12547 // This pattern is automatically generated from aarch64_ad.m4
12548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12549 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12550 %{
12551   match(Set dst (AndL (URShiftL src rshift) mask));
12552   // Make sure we are not going to exceed what ubfx can do.
12553   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12554 
12555   ins_cost(INSN_COST);
12556   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12557   ins_encode %{
12558     int rshift = $rshift$$constant & 63;
12559     intptr_t mask = $mask$$constant;
12560     int width = exact_log2_long(mask+1);
12561     __ ubfx(as_Register($dst$$reg),
12562             as_Register($src$$reg), rshift, width);
12563   %}
12564   ins_pipe(ialu_reg_shift);
12565 %}
12566 
12567 
12568 // This pattern is automatically generated from aarch64_ad.m4
12569 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12570 
12571 // We can use ubfx when extending an And with a mask when we know mask
12572 // is positive.  We know that because immI_bitmask guarantees it.
12573 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12574 %{
12575   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12576   // Make sure we are not going to exceed what ubfxw can do.
12577   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12578 
12579   ins_cost(INSN_COST * 2);
12580   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12581   ins_encode %{
12582     int rshift = $rshift$$constant & 31;
12583     intptr_t mask = $mask$$constant;
12584     int width = exact_log2(mask+1);
12585     __ ubfx(as_Register($dst$$reg),
12586             as_Register($src$$reg), rshift, width);
12587   %}
12588   ins_pipe(ialu_reg_shift);
12589 %}
12590 
12591 
12592 // This pattern is automatically generated from aarch64_ad.m4
12593 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12594 
12595 // We can use ubfiz when masking by a positive number and then left shifting the result.
12596 // We know that the mask is positive because immI_bitmask guarantees it.
12597 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12598 %{
12599   match(Set dst (LShiftI (AndI src mask) lshift));
12600   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12601 
12602   ins_cost(INSN_COST);
12603   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12604   ins_encode %{
12605     int lshift = $lshift$$constant & 31;
12606     intptr_t mask = $mask$$constant;
12607     int width = exact_log2(mask+1);
12608     __ ubfizw(as_Register($dst$$reg),
12609           as_Register($src$$reg), lshift, width);
12610   %}
12611   ins_pipe(ialu_reg_shift);
12612 %}
12613 
12614 // This pattern is automatically generated from aarch64_ad.m4
12615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12616 
12617 // We can use ubfiz when masking by a positive number and then left shifting the result.
12618 // We know that the mask is positive because immL_bitmask guarantees it.
12619 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12620 %{
12621   match(Set dst (LShiftL (AndL src mask) lshift));
12622   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12623 
12624   ins_cost(INSN_COST);
12625   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12626   ins_encode %{
12627     int lshift = $lshift$$constant & 63;
12628     intptr_t mask = $mask$$constant;
12629     int width = exact_log2_long(mask+1);
12630     __ ubfiz(as_Register($dst$$reg),
12631           as_Register($src$$reg), lshift, width);
12632   %}
12633   ins_pipe(ialu_reg_shift);
12634 %}
12635 
12636 // This pattern is automatically generated from aarch64_ad.m4
12637 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12638 
12639 // We can use ubfiz when masking by a positive number and then left shifting the result.
12640 // We know that the mask is positive because immI_bitmask guarantees it.
12641 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12642 %{
12643   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12644   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12645 
12646   ins_cost(INSN_COST);
12647   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12648   ins_encode %{
12649     int lshift = $lshift$$constant & 31;
12650     intptr_t mask = $mask$$constant;
12651     int width = exact_log2(mask+1);
12652     __ ubfizw(as_Register($dst$$reg),
12653           as_Register($src$$reg), lshift, width);
12654   %}
12655   ins_pipe(ialu_reg_shift);
12656 %}
12657 
12658 // This pattern is automatically generated from aarch64_ad.m4
12659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12660 
12661 // We can use ubfiz when masking by a positive number and then left shifting the result.
12662 // We know that the mask is positive because immL_bitmask guarantees it.
12663 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12664 %{
12665   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12666   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12667 
12668   ins_cost(INSN_COST);
12669   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12670   ins_encode %{
12671     int lshift = $lshift$$constant & 63;
12672     intptr_t mask = $mask$$constant;
12673     int width = exact_log2_long(mask+1);
12674     __ ubfiz(as_Register($dst$$reg),
12675           as_Register($src$$reg), lshift, width);
12676   %}
12677   ins_pipe(ialu_reg_shift);
12678 %}
12679 
12680 
12681 // This pattern is automatically generated from aarch64_ad.m4
12682 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12683 
12684 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12685 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12686 %{
12687   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12688   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12689 
12690   ins_cost(INSN_COST);
12691   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12692   ins_encode %{
12693     int lshift = $lshift$$constant & 63;
12694     intptr_t mask = $mask$$constant;
12695     int width = exact_log2(mask+1);
12696     __ ubfiz(as_Register($dst$$reg),
12697              as_Register($src$$reg), lshift, width);
12698   %}
12699   ins_pipe(ialu_reg_shift);
12700 %}
12701 
12702 // This pattern is automatically generated from aarch64_ad.m4
12703 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12704 
12705 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12706 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12707 %{
12708   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12709   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12710 
12711   ins_cost(INSN_COST);
12712   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12713   ins_encode %{
12714     int lshift = $lshift$$constant & 31;
12715     intptr_t mask = $mask$$constant;
12716     int width = exact_log2(mask+1);
12717     __ ubfiz(as_Register($dst$$reg),
12718              as_Register($src$$reg), lshift, width);
12719   %}
12720   ins_pipe(ialu_reg_shift);
12721 %}
12722 
12723 // This pattern is automatically generated from aarch64_ad.m4
12724 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12725 
12726 // Can skip int2long conversions after AND with small bitmask
12727 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12728 %{
12729   match(Set dst (ConvI2L (AndI src msk)));
12730   ins_cost(INSN_COST);
12731   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12732   ins_encode %{
12733     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12734   %}
12735   ins_pipe(ialu_reg_shift);
12736 %}
12737 
12738 
12739 // Rotations
12740 
12741 // This pattern is automatically generated from aarch64_ad.m4
12742 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12743 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12744 %{
12745   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12746   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12747 
12748   ins_cost(INSN_COST);
12749   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12750 
12751   ins_encode %{
12752     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12753             $rshift$$constant & 63);
12754   %}
12755   ins_pipe(ialu_reg_reg_extr);
12756 %}
12757 
12758 
12759 // This pattern is automatically generated from aarch64_ad.m4
12760 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12761 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12762 %{
12763   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12764   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12765 
12766   ins_cost(INSN_COST);
12767   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12768 
12769   ins_encode %{
12770     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12771             $rshift$$constant & 31);
12772   %}
12773   ins_pipe(ialu_reg_reg_extr);
12774 %}
12775 
12776 
12777 // This pattern is automatically generated from aarch64_ad.m4
12778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12779 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12780 %{
12781   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12782   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12783 
12784   ins_cost(INSN_COST);
12785   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12786 
12787   ins_encode %{
12788     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12789             $rshift$$constant & 63);
12790   %}
12791   ins_pipe(ialu_reg_reg_extr);
12792 %}
12793 
12794 
12795 // This pattern is automatically generated from aarch64_ad.m4
12796 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12797 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12798 %{
12799   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12800   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12801 
12802   ins_cost(INSN_COST);
12803   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12804 
12805   ins_encode %{
12806     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12807             $rshift$$constant & 31);
12808   %}
12809   ins_pipe(ialu_reg_reg_extr);
12810 %}
12811 
12812 // This pattern is automatically generated from aarch64_ad.m4
12813 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12814 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12815 %{
12816   match(Set dst (RotateRight src shift));
12817 
12818   ins_cost(INSN_COST);
12819   format %{ "ror    $dst, $src, $shift" %}
12820 
12821   ins_encode %{
12822      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12823                $shift$$constant & 0x1f);
12824   %}
12825   ins_pipe(ialu_reg_reg_vshift);
12826 %}
12827 
12828 // This pattern is automatically generated from aarch64_ad.m4
12829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12830 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12831 %{
12832   match(Set dst (RotateRight src shift));
12833 
12834   ins_cost(INSN_COST);
12835   format %{ "ror    $dst, $src, $shift" %}
12836 
12837   ins_encode %{
12838      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12839                $shift$$constant & 0x3f);
12840   %}
12841   ins_pipe(ialu_reg_reg_vshift);
12842 %}
12843 
12844 // This pattern is automatically generated from aarch64_ad.m4
12845 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12846 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12847 %{
12848   match(Set dst (RotateRight src shift));
12849 
12850   ins_cost(INSN_COST);
12851   format %{ "ror    $dst, $src, $shift" %}
12852 
12853   ins_encode %{
12854      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12855   %}
12856   ins_pipe(ialu_reg_reg_vshift);
12857 %}
12858 
12859 // This pattern is automatically generated from aarch64_ad.m4
12860 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12861 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12862 %{
12863   match(Set dst (RotateRight src shift));
12864 
12865   ins_cost(INSN_COST);
12866   format %{ "ror    $dst, $src, $shift" %}
12867 
12868   ins_encode %{
12869      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12870   %}
12871   ins_pipe(ialu_reg_reg_vshift);
12872 %}
12873 
12874 // This pattern is automatically generated from aarch64_ad.m4
12875 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12876 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12877 %{
12878   match(Set dst (RotateLeft src shift));
12879 
12880   ins_cost(INSN_COST);
12881   format %{ "rol    $dst, $src, $shift" %}
12882 
12883   ins_encode %{
12884      __ subw(rscratch1, zr, as_Register($shift$$reg));
12885      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12886   %}
12887   ins_pipe(ialu_reg_reg_vshift);
12888 %}
12889 
12890 // This pattern is automatically generated from aarch64_ad.m4
12891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12892 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12893 %{
12894   match(Set dst (RotateLeft src shift));
12895 
12896   ins_cost(INSN_COST);
12897   format %{ "rol    $dst, $src, $shift" %}
12898 
12899   ins_encode %{
12900      __ subw(rscratch1, zr, as_Register($shift$$reg));
12901      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12902   %}
12903   ins_pipe(ialu_reg_reg_vshift);
12904 %}
12905 
12906 
12907 // Add/subtract (extended)
12908 
12909 // This pattern is automatically generated from aarch64_ad.m4
12910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12911 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12912 %{
12913   match(Set dst (AddL src1 (ConvI2L src2)));
12914   ins_cost(INSN_COST);
12915   format %{ "add  $dst, $src1, $src2, sxtw" %}
12916 
12917    ins_encode %{
12918      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12919             as_Register($src2$$reg), ext::sxtw);
12920    %}
12921   ins_pipe(ialu_reg_reg);
12922 %}
12923 
12924 // This pattern is automatically generated from aarch64_ad.m4
12925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12926 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12927 %{
12928   match(Set dst (SubL src1 (ConvI2L src2)));
12929   ins_cost(INSN_COST);
12930   format %{ "sub  $dst, $src1, $src2, sxtw" %}
12931 
12932    ins_encode %{
12933      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12934             as_Register($src2$$reg), ext::sxtw);
12935    %}
12936   ins_pipe(ialu_reg_reg);
12937 %}
12938 
12939 // This pattern is automatically generated from aarch64_ad.m4
12940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12941 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12942 %{
12943   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12944   ins_cost(INSN_COST);
12945   format %{ "add  $dst, $src1, $src2, sxth" %}
12946 
12947    ins_encode %{
12948      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12949             as_Register($src2$$reg), ext::sxth);
12950    %}
12951   ins_pipe(ialu_reg_reg);
12952 %}
12953 
12954 // This pattern is automatically generated from aarch64_ad.m4
12955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12956 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12957 %{
12958   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12959   ins_cost(INSN_COST);
12960   format %{ "add  $dst, $src1, $src2, sxtb" %}
12961 
12962    ins_encode %{
12963      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12964             as_Register($src2$$reg), ext::sxtb);
12965    %}
12966   ins_pipe(ialu_reg_reg);
12967 %}
12968 
12969 // This pattern is automatically generated from aarch64_ad.m4
12970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12971 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12972 %{
12973   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12974   ins_cost(INSN_COST);
12975   format %{ "add  $dst, $src1, $src2, uxtb" %}
12976 
12977    ins_encode %{
12978      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12979             as_Register($src2$$reg), ext::uxtb);
12980    %}
12981   ins_pipe(ialu_reg_reg);
12982 %}
12983 
12984 // This pattern is automatically generated from aarch64_ad.m4
12985 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12986 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12987 %{
12988   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12989   ins_cost(INSN_COST);
12990   format %{ "add  $dst, $src1, $src2, sxth" %}
12991 
12992    ins_encode %{
12993      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12994             as_Register($src2$$reg), ext::sxth);
12995    %}
12996   ins_pipe(ialu_reg_reg);
12997 %}
12998 
12999 // This pattern is automatically generated from aarch64_ad.m4
13000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13001 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13002 %{
13003   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13004   ins_cost(INSN_COST);
13005   format %{ "add  $dst, $src1, $src2, sxtw" %}
13006 
13007    ins_encode %{
13008      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13009             as_Register($src2$$reg), ext::sxtw);
13010    %}
13011   ins_pipe(ialu_reg_reg);
13012 %}
13013 
13014 // This pattern is automatically generated from aarch64_ad.m4
13015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13016 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13017 %{
13018   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13019   ins_cost(INSN_COST);
13020   format %{ "add  $dst, $src1, $src2, sxtb" %}
13021 
13022    ins_encode %{
13023      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13024             as_Register($src2$$reg), ext::sxtb);
13025    %}
13026   ins_pipe(ialu_reg_reg);
13027 %}
13028 
13029 // This pattern is automatically generated from aarch64_ad.m4
13030 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13031 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13032 %{
13033   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13034   ins_cost(INSN_COST);
13035   format %{ "add  $dst, $src1, $src2, uxtb" %}
13036 
13037    ins_encode %{
13038      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13039             as_Register($src2$$reg), ext::uxtb);
13040    %}
13041   ins_pipe(ialu_reg_reg);
13042 %}
13043 
13044 // This pattern is automatically generated from aarch64_ad.m4
13045 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13046 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13047 %{
13048   match(Set dst (AddI src1 (AndI src2 mask)));
13049   ins_cost(INSN_COST);
13050   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13051 
13052    ins_encode %{
13053      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13054             as_Register($src2$$reg), ext::uxtb);
13055    %}
13056   ins_pipe(ialu_reg_reg);
13057 %}
13058 
13059 // This pattern is automatically generated from aarch64_ad.m4
13060 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13061 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13062 %{
13063   match(Set dst (AddI src1 (AndI src2 mask)));
13064   ins_cost(INSN_COST);
13065   format %{ "addw  $dst, $src1, $src2, uxth" %}
13066 
13067    ins_encode %{
13068      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13069             as_Register($src2$$reg), ext::uxth);
13070    %}
13071   ins_pipe(ialu_reg_reg);
13072 %}
13073 
13074 // This pattern is automatically generated from aarch64_ad.m4
13075 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13076 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13077 %{
13078   match(Set dst (AddL src1 (AndL src2 mask)));
13079   ins_cost(INSN_COST);
13080   format %{ "add  $dst, $src1, $src2, uxtb" %}
13081 
13082    ins_encode %{
13083      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13084             as_Register($src2$$reg), ext::uxtb);
13085    %}
13086   ins_pipe(ialu_reg_reg);
13087 %}
13088 
13089 // This pattern is automatically generated from aarch64_ad.m4
13090 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13091 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13092 %{
13093   match(Set dst (AddL src1 (AndL src2 mask)));
13094   ins_cost(INSN_COST);
13095   format %{ "add  $dst, $src1, $src2, uxth" %}
13096 
13097    ins_encode %{
13098      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13099             as_Register($src2$$reg), ext::uxth);
13100    %}
13101   ins_pipe(ialu_reg_reg);
13102 %}
13103 
13104 // This pattern is automatically generated from aarch64_ad.m4
13105 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13106 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13107 %{
13108   match(Set dst (AddL src1 (AndL src2 mask)));
13109   ins_cost(INSN_COST);
13110   format %{ "add  $dst, $src1, $src2, uxtw" %}
13111 
13112    ins_encode %{
13113      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13114             as_Register($src2$$reg), ext::uxtw);
13115    %}
13116   ins_pipe(ialu_reg_reg);
13117 %}
13118 
13119 // This pattern is automatically generated from aarch64_ad.m4
13120 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13121 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13122 %{
13123   match(Set dst (SubI src1 (AndI src2 mask)));
13124   ins_cost(INSN_COST);
13125   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13126 
13127    ins_encode %{
13128      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13129             as_Register($src2$$reg), ext::uxtb);
13130    %}
13131   ins_pipe(ialu_reg_reg);
13132 %}
13133 
13134 // This pattern is automatically generated from aarch64_ad.m4
13135 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13136 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13137 %{
13138   match(Set dst (SubI src1 (AndI src2 mask)));
13139   ins_cost(INSN_COST);
13140   format %{ "subw  $dst, $src1, $src2, uxth" %}
13141 
13142    ins_encode %{
13143      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13144             as_Register($src2$$reg), ext::uxth);
13145    %}
13146   ins_pipe(ialu_reg_reg);
13147 %}
13148 
13149 // This pattern is automatically generated from aarch64_ad.m4
13150 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13151 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13152 %{
13153   match(Set dst (SubL src1 (AndL src2 mask)));
13154   ins_cost(INSN_COST);
13155   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13156 
13157    ins_encode %{
13158      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13159             as_Register($src2$$reg), ext::uxtb);
13160    %}
13161   ins_pipe(ialu_reg_reg);
13162 %}
13163 
13164 // This pattern is automatically generated from aarch64_ad.m4
13165 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13166 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13167 %{
13168   match(Set dst (SubL src1 (AndL src2 mask)));
13169   ins_cost(INSN_COST);
13170   format %{ "sub  $dst, $src1, $src2, uxth" %}
13171 
13172    ins_encode %{
13173      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13174             as_Register($src2$$reg), ext::uxth);
13175    %}
13176   ins_pipe(ialu_reg_reg);
13177 %}
13178 
13179 // This pattern is automatically generated from aarch64_ad.m4
13180 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13181 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13182 %{
13183   match(Set dst (SubL src1 (AndL src2 mask)));
13184   ins_cost(INSN_COST);
13185   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13186 
13187    ins_encode %{
13188      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13189             as_Register($src2$$reg), ext::uxtw);
13190    %}
13191   ins_pipe(ialu_reg_reg);
13192 %}
13193 
13194 
13195 // This pattern is automatically generated from aarch64_ad.m4
13196 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13197 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13198 %{
13199   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13200   ins_cost(1.9 * INSN_COST);
13201   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13202 
13203    ins_encode %{
13204      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13205             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13206    %}
13207   ins_pipe(ialu_reg_reg_shift);
13208 %}
13209 
13210 // This pattern is automatically generated from aarch64_ad.m4
13211 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13212 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13213 %{
13214   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13215   ins_cost(1.9 * INSN_COST);
13216   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13217 
13218    ins_encode %{
13219      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13220             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13221    %}
13222   ins_pipe(ialu_reg_reg_shift);
13223 %}
13224 
13225 // This pattern is automatically generated from aarch64_ad.m4
13226 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13227 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13228 %{
13229   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13230   ins_cost(1.9 * INSN_COST);
13231   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13232 
13233    ins_encode %{
13234      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13235             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13236    %}
13237   ins_pipe(ialu_reg_reg_shift);
13238 %}
13239 
13240 // This pattern is automatically generated from aarch64_ad.m4
13241 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13242 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13243 %{
13244   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13245   ins_cost(1.9 * INSN_COST);
13246   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13247 
13248    ins_encode %{
13249      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13250             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13251    %}
13252   ins_pipe(ialu_reg_reg_shift);
13253 %}
13254 
13255 // This pattern is automatically generated from aarch64_ad.m4
13256 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13257 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13258 %{
13259   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13260   ins_cost(1.9 * INSN_COST);
13261   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13262 
13263    ins_encode %{
13264      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13265             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13266    %}
13267   ins_pipe(ialu_reg_reg_shift);
13268 %}
13269 
13270 // This pattern is automatically generated from aarch64_ad.m4
13271 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13272 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13273 %{
13274   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13275   ins_cost(1.9 * INSN_COST);
13276   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13277 
13278    ins_encode %{
13279      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13280             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13281    %}
13282   ins_pipe(ialu_reg_reg_shift);
13283 %}
13284 
13285 // This pattern is automatically generated from aarch64_ad.m4
13286 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13287 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13288 %{
13289   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13290   ins_cost(1.9 * INSN_COST);
13291   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13292 
13293    ins_encode %{
13294      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13295             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13296    %}
13297   ins_pipe(ialu_reg_reg_shift);
13298 %}
13299 
13300 // This pattern is automatically generated from aarch64_ad.m4
13301 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13302 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13303 %{
13304   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13305   ins_cost(1.9 * INSN_COST);
13306   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13307 
13308    ins_encode %{
13309      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13310             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13311    %}
13312   ins_pipe(ialu_reg_reg_shift);
13313 %}
13314 
13315 // This pattern is automatically generated from aarch64_ad.m4
13316 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13317 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13318 %{
13319   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13320   ins_cost(1.9 * INSN_COST);
13321   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13322 
13323    ins_encode %{
13324      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13325             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13326    %}
13327   ins_pipe(ialu_reg_reg_shift);
13328 %}
13329 
13330 // This pattern is automatically generated from aarch64_ad.m4
13331 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13332 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13333 %{
13334   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13335   ins_cost(1.9 * INSN_COST);
13336   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13337 
13338    ins_encode %{
13339      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13340             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13341    %}
13342   ins_pipe(ialu_reg_reg_shift);
13343 %}
13344 
13345 // This pattern is automatically generated from aarch64_ad.m4
13346 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13347 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13348 %{
13349   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13350   ins_cost(1.9 * INSN_COST);
13351   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13352 
13353    ins_encode %{
13354      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13355             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13356    %}
13357   ins_pipe(ialu_reg_reg_shift);
13358 %}
13359 
13360 // This pattern is automatically generated from aarch64_ad.m4
13361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13362 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13363 %{
13364   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13365   ins_cost(1.9 * INSN_COST);
13366   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13367 
13368    ins_encode %{
13369      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13370             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13371    %}
13372   ins_pipe(ialu_reg_reg_shift);
13373 %}
13374 
13375 // This pattern is automatically generated from aarch64_ad.m4
13376 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13377 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13378 %{
13379   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13380   ins_cost(1.9 * INSN_COST);
13381   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13382 
13383    ins_encode %{
13384      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13385             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13386    %}
13387   ins_pipe(ialu_reg_reg_shift);
13388 %}
13389 
13390 // This pattern is automatically generated from aarch64_ad.m4
13391 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13392 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13393 %{
13394   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13395   ins_cost(1.9 * INSN_COST);
13396   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13397 
13398    ins_encode %{
13399      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13400             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13401    %}
13402   ins_pipe(ialu_reg_reg_shift);
13403 %}
13404 
13405 // This pattern is automatically generated from aarch64_ad.m4
13406 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13407 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13408 %{
13409   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13410   ins_cost(1.9 * INSN_COST);
13411   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13412 
13413    ins_encode %{
13414      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13415             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13416    %}
13417   ins_pipe(ialu_reg_reg_shift);
13418 %}
13419 
13420 // This pattern is automatically generated from aarch64_ad.m4
13421 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13422 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13423 %{
13424   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13425   ins_cost(1.9 * INSN_COST);
13426   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13427 
13428    ins_encode %{
13429      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13430             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13431    %}
13432   ins_pipe(ialu_reg_reg_shift);
13433 %}
13434 
13435 // This pattern is automatically generated from aarch64_ad.m4
13436 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13437 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13438 %{
13439   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13440   ins_cost(1.9 * INSN_COST);
13441   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13442 
13443    ins_encode %{
13444      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13445             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13446    %}
13447   ins_pipe(ialu_reg_reg_shift);
13448 %}
13449 
13450 // This pattern is automatically generated from aarch64_ad.m4
13451 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13452 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13453 %{
13454   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13455   ins_cost(1.9 * INSN_COST);
13456   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13457 
13458    ins_encode %{
13459      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13460             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13461    %}
13462   ins_pipe(ialu_reg_reg_shift);
13463 %}
13464 
13465 // This pattern is automatically generated from aarch64_ad.m4
13466 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13467 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13468 %{
13469   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13470   ins_cost(1.9 * INSN_COST);
13471   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13472 
13473    ins_encode %{
13474      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13475             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13476    %}
13477   ins_pipe(ialu_reg_reg_shift);
13478 %}
13479 
13480 // This pattern is automatically generated from aarch64_ad.m4
13481 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13482 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13483 %{
13484   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13485   ins_cost(1.9 * INSN_COST);
13486   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13487 
13488    ins_encode %{
13489      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13490             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13491    %}
13492   ins_pipe(ialu_reg_reg_shift);
13493 %}
13494 
13495 // This pattern is automatically generated from aarch64_ad.m4
13496 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13497 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13498 %{
13499   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13500   ins_cost(1.9 * INSN_COST);
13501   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13502 
13503    ins_encode %{
13504      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13505             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13506    %}
13507   ins_pipe(ialu_reg_reg_shift);
13508 %}
13509 
13510 // This pattern is automatically generated from aarch64_ad.m4
13511 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13512 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13513 %{
13514   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13515   ins_cost(1.9 * INSN_COST);
13516   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13517 
13518    ins_encode %{
13519      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13520             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13521    %}
13522   ins_pipe(ialu_reg_reg_shift);
13523 %}
13524 
13525 // This pattern is automatically generated from aarch64_ad.m4
13526 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13527 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13528 %{
13529   effect(DEF dst, USE src1, USE src2, USE cr);
13530   ins_cost(INSN_COST * 2);
13531   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
13532 
13533   ins_encode %{
13534     __ cselw($dst$$Register,
13535              $src1$$Register,
13536              $src2$$Register,
13537              Assembler::LT);
13538   %}
13539   ins_pipe(icond_reg_reg);
13540 %}
13541 
13542 // This pattern is automatically generated from aarch64_ad.m4
13543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13544 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13545 %{
13546   effect(DEF dst, USE src1, USE src2, USE cr);
13547   ins_cost(INSN_COST * 2);
13548   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
13549 
13550   ins_encode %{
13551     __ cselw($dst$$Register,
13552              $src1$$Register,
13553              $src2$$Register,
13554              Assembler::GT);
13555   %}
13556   ins_pipe(icond_reg_reg);
13557 %}
13558 
13559 // This pattern is automatically generated from aarch64_ad.m4
13560 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13561 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13562 %{
13563   effect(DEF dst, USE src1, USE cr);
13564   ins_cost(INSN_COST * 2);
13565   format %{ "cselw $dst, $src1, zr lt\t"  %}
13566 
13567   ins_encode %{
13568     __ cselw($dst$$Register,
13569              $src1$$Register,
13570              zr,
13571              Assembler::LT);
13572   %}
13573   ins_pipe(icond_reg);
13574 %}
13575 
13576 // This pattern is automatically generated from aarch64_ad.m4
13577 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13578 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13579 %{
13580   effect(DEF dst, USE src1, USE cr);
13581   ins_cost(INSN_COST * 2);
13582   format %{ "cselw $dst, $src1, zr gt\t"  %}
13583 
13584   ins_encode %{
13585     __ cselw($dst$$Register,
13586              $src1$$Register,
13587              zr,
13588              Assembler::GT);
13589   %}
13590   ins_pipe(icond_reg);
13591 %}
13592 
13593 // This pattern is automatically generated from aarch64_ad.m4
13594 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13595 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13596 %{
13597   effect(DEF dst, USE src1, USE cr);
13598   ins_cost(INSN_COST * 2);
13599   format %{ "csincw $dst, $src1, zr le\t"  %}
13600 
13601   ins_encode %{
13602     __ csincw($dst$$Register,
13603              $src1$$Register,
13604              zr,
13605              Assembler::LE);
13606   %}
13607   ins_pipe(icond_reg);
13608 %}
13609 
13610 // This pattern is automatically generated from aarch64_ad.m4
13611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13612 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13613 %{
13614   effect(DEF dst, USE src1, USE cr);
13615   ins_cost(INSN_COST * 2);
13616   format %{ "csincw $dst, $src1, zr gt\t"  %}
13617 
13618   ins_encode %{
13619     __ csincw($dst$$Register,
13620              $src1$$Register,
13621              zr,
13622              Assembler::GT);
13623   %}
13624   ins_pipe(icond_reg);
13625 %}
13626 
13627 // This pattern is automatically generated from aarch64_ad.m4
13628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13629 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13630 %{
13631   effect(DEF dst, USE src1, USE cr);
13632   ins_cost(INSN_COST * 2);
13633   format %{ "csinvw $dst, $src1, zr lt\t"  %}
13634 
13635   ins_encode %{
13636     __ csinvw($dst$$Register,
13637              $src1$$Register,
13638              zr,
13639              Assembler::LT);
13640   %}
13641   ins_pipe(icond_reg);
13642 %}
13643 
13644 // This pattern is automatically generated from aarch64_ad.m4
13645 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13646 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13647 %{
13648   effect(DEF dst, USE src1, USE cr);
13649   ins_cost(INSN_COST * 2);
13650   format %{ "csinvw $dst, $src1, zr ge\t"  %}
13651 
13652   ins_encode %{
13653     __ csinvw($dst$$Register,
13654              $src1$$Register,
13655              zr,
13656              Assembler::GE);
13657   %}
13658   ins_pipe(icond_reg);
13659 %}
13660 
13661 // This pattern is automatically generated from aarch64_ad.m4
13662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13663 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13664 %{
13665   match(Set dst (MinI src imm));
13666   ins_cost(INSN_COST * 3);
13667   expand %{
13668     rFlagsReg cr;
13669     compI_reg_imm0(cr, src);
13670     cmovI_reg_imm0_lt(dst, src, cr);
13671   %}
13672 %}
13673 
13674 // This pattern is automatically generated from aarch64_ad.m4
13675 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13676 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13677 %{
13678   match(Set dst (MinI imm src));
13679   ins_cost(INSN_COST * 3);
13680   expand %{
13681     rFlagsReg cr;
13682     compI_reg_imm0(cr, src);
13683     cmovI_reg_imm0_lt(dst, src, cr);
13684   %}
13685 %}
13686 
13687 // This pattern is automatically generated from aarch64_ad.m4
13688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13689 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13690 %{
13691   match(Set dst (MinI src imm));
13692   ins_cost(INSN_COST * 3);
13693   expand %{
13694     rFlagsReg cr;
13695     compI_reg_imm0(cr, src);
13696     cmovI_reg_imm1_le(dst, src, cr);
13697   %}
13698 %}
13699 
13700 // This pattern is automatically generated from aarch64_ad.m4
13701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13702 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13703 %{
13704   match(Set dst (MinI imm src));
13705   ins_cost(INSN_COST * 3);
13706   expand %{
13707     rFlagsReg cr;
13708     compI_reg_imm0(cr, src);
13709     cmovI_reg_imm1_le(dst, src, cr);
13710   %}
13711 %}
13712 
13713 // This pattern is automatically generated from aarch64_ad.m4
13714 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13715 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13716 %{
13717   match(Set dst (MinI src imm));
13718   ins_cost(INSN_COST * 3);
13719   expand %{
13720     rFlagsReg cr;
13721     compI_reg_imm0(cr, src);
13722     cmovI_reg_immM1_lt(dst, src, cr);
13723   %}
13724 %}
13725 
13726 // This pattern is automatically generated from aarch64_ad.m4
13727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13728 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13729 %{
13730   match(Set dst (MinI imm src));
13731   ins_cost(INSN_COST * 3);
13732   expand %{
13733     rFlagsReg cr;
13734     compI_reg_imm0(cr, src);
13735     cmovI_reg_immM1_lt(dst, src, cr);
13736   %}
13737 %}
13738 
13739 // This pattern is automatically generated from aarch64_ad.m4
13740 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13741 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13742 %{
13743   match(Set dst (MaxI src imm));
13744   ins_cost(INSN_COST * 3);
13745   expand %{
13746     rFlagsReg cr;
13747     compI_reg_imm0(cr, src);
13748     cmovI_reg_imm0_gt(dst, src, cr);
13749   %}
13750 %}
13751 
13752 // This pattern is automatically generated from aarch64_ad.m4
13753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13754 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13755 %{
13756   match(Set dst (MaxI imm src));
13757   ins_cost(INSN_COST * 3);
13758   expand %{
13759     rFlagsReg cr;
13760     compI_reg_imm0(cr, src);
13761     cmovI_reg_imm0_gt(dst, src, cr);
13762   %}
13763 %}
13764 
13765 // This pattern is automatically generated from aarch64_ad.m4
13766 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13767 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13768 %{
13769   match(Set dst (MaxI src imm));
13770   ins_cost(INSN_COST * 3);
13771   expand %{
13772     rFlagsReg cr;
13773     compI_reg_imm0(cr, src);
13774     cmovI_reg_imm1_gt(dst, src, cr);
13775   %}
13776 %}
13777 
13778 // This pattern is automatically generated from aarch64_ad.m4
13779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13780 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13781 %{
13782   match(Set dst (MaxI imm src));
13783   ins_cost(INSN_COST * 3);
13784   expand %{
13785     rFlagsReg cr;
13786     compI_reg_imm0(cr, src);
13787     cmovI_reg_imm1_gt(dst, src, cr);
13788   %}
13789 %}
13790 
13791 // This pattern is automatically generated from aarch64_ad.m4
13792 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13793 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13794 %{
13795   match(Set dst (MaxI src imm));
13796   ins_cost(INSN_COST * 3);
13797   expand %{
13798     rFlagsReg cr;
13799     compI_reg_imm0(cr, src);
13800     cmovI_reg_immM1_ge(dst, src, cr);
13801   %}
13802 %}
13803 
13804 // This pattern is automatically generated from aarch64_ad.m4
13805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13806 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13807 %{
13808   match(Set dst (MaxI imm src));
13809   ins_cost(INSN_COST * 3);
13810   expand %{
13811     rFlagsReg cr;
13812     compI_reg_imm0(cr, src);
13813     cmovI_reg_immM1_ge(dst, src, cr);
13814   %}
13815 %}
13816 
13817 // This pattern is automatically generated from aarch64_ad.m4
13818 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13819 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13820 %{
13821   match(Set dst (ReverseI src));
13822   ins_cost(INSN_COST);
13823   format %{ "rbitw  $dst, $src" %}
13824   ins_encode %{
13825     __ rbitw($dst$$Register, $src$$Register);
13826   %}
13827   ins_pipe(ialu_reg);
13828 %}
13829 
13830 // This pattern is automatically generated from aarch64_ad.m4
13831 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13832 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13833 %{
13834   match(Set dst (ReverseL src));
13835   ins_cost(INSN_COST);
13836   format %{ "rbit  $dst, $src" %}
13837   ins_encode %{
13838     __ rbit($dst$$Register, $src$$Register);
13839   %}
13840   ins_pipe(ialu_reg);
13841 %}
13842 
13843 
13844 // END This section of the file is automatically generated. Do not edit --------------
13845 
13846 
13847 // ============================================================================
13848 // Floating Point Arithmetic Instructions
13849 
13850 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13851   match(Set dst (AddHF src1 src2));
13852   format %{ "faddh $dst, $src1, $src2" %}
13853   ins_encode %{
13854     __ faddh($dst$$FloatRegister,
13855              $src1$$FloatRegister,
13856              $src2$$FloatRegister);
13857   %}
13858   ins_pipe(fp_dop_reg_reg_s);
13859 %}
13860 
13861 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13862   match(Set dst (AddF src1 src2));
13863 
13864   ins_cost(INSN_COST * 5);
13865   format %{ "fadds   $dst, $src1, $src2" %}
13866 
13867   ins_encode %{
13868     __ fadds(as_FloatRegister($dst$$reg),
13869              as_FloatRegister($src1$$reg),
13870              as_FloatRegister($src2$$reg));
13871   %}
13872 
13873   ins_pipe(fp_dop_reg_reg_s);
13874 %}
13875 
13876 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13877   match(Set dst (AddD src1 src2));
13878 
13879   ins_cost(INSN_COST * 5);
13880   format %{ "faddd   $dst, $src1, $src2" %}
13881 
13882   ins_encode %{
13883     __ faddd(as_FloatRegister($dst$$reg),
13884              as_FloatRegister($src1$$reg),
13885              as_FloatRegister($src2$$reg));
13886   %}
13887 
13888   ins_pipe(fp_dop_reg_reg_d);
13889 %}
13890 
13891 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13892   match(Set dst (SubHF src1 src2));
13893   format %{ "fsubh $dst, $src1, $src2" %}
13894   ins_encode %{
13895     __ fsubh($dst$$FloatRegister,
13896              $src1$$FloatRegister,
13897              $src2$$FloatRegister);
13898   %}
13899   ins_pipe(fp_dop_reg_reg_s);
13900 %}
13901 
13902 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13903   match(Set dst (SubF src1 src2));
13904 
13905   ins_cost(INSN_COST * 5);
13906   format %{ "fsubs   $dst, $src1, $src2" %}
13907 
13908   ins_encode %{
13909     __ fsubs(as_FloatRegister($dst$$reg),
13910              as_FloatRegister($src1$$reg),
13911              as_FloatRegister($src2$$reg));
13912   %}
13913 
13914   ins_pipe(fp_dop_reg_reg_s);
13915 %}
13916 
13917 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13918   match(Set dst (SubD src1 src2));
13919 
13920   ins_cost(INSN_COST * 5);
13921   format %{ "fsubd   $dst, $src1, $src2" %}
13922 
13923   ins_encode %{
13924     __ fsubd(as_FloatRegister($dst$$reg),
13925              as_FloatRegister($src1$$reg),
13926              as_FloatRegister($src2$$reg));
13927   %}
13928 
13929   ins_pipe(fp_dop_reg_reg_d);
13930 %}
13931 
13932 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13933   match(Set dst (MulHF src1 src2));
13934   format %{ "fmulh $dst, $src1, $src2" %}
13935   ins_encode %{
13936     __ fmulh($dst$$FloatRegister,
13937              $src1$$FloatRegister,
13938              $src2$$FloatRegister);
13939   %}
13940   ins_pipe(fp_dop_reg_reg_s);
13941 %}
13942 
13943 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13944   match(Set dst (MulF src1 src2));
13945 
13946   ins_cost(INSN_COST * 6);
13947   format %{ "fmuls   $dst, $src1, $src2" %}
13948 
13949   ins_encode %{
13950     __ fmuls(as_FloatRegister($dst$$reg),
13951              as_FloatRegister($src1$$reg),
13952              as_FloatRegister($src2$$reg));
13953   %}
13954 
13955   ins_pipe(fp_dop_reg_reg_s);
13956 %}
13957 
13958 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13959   match(Set dst (MulD src1 src2));
13960 
13961   ins_cost(INSN_COST * 6);
13962   format %{ "fmuld   $dst, $src1, $src2" %}
13963 
13964   ins_encode %{
13965     __ fmuld(as_FloatRegister($dst$$reg),
13966              as_FloatRegister($src1$$reg),
13967              as_FloatRegister($src2$$reg));
13968   %}
13969 
13970   ins_pipe(fp_dop_reg_reg_d);
13971 %}
13972 
13973 // src1 * src2 + src3 (half-precision float)
13974 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13975   match(Set dst (FmaHF src3 (Binary src1 src2)));
13976   format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13977   ins_encode %{
13978     assert(UseFMA, "Needs FMA instructions support.");
13979     __ fmaddh($dst$$FloatRegister,
13980               $src1$$FloatRegister,
13981               $src2$$FloatRegister,
13982               $src3$$FloatRegister);
13983   %}
13984   ins_pipe(pipe_class_default);
13985 %}
13986 
13987 // src1 * src2 + src3
13988 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13989   match(Set dst (FmaF src3 (Binary src1 src2)));
13990 
13991   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
13992 
13993   ins_encode %{
13994     assert(UseFMA, "Needs FMA instructions support.");
13995     __ fmadds(as_FloatRegister($dst$$reg),
13996              as_FloatRegister($src1$$reg),
13997              as_FloatRegister($src2$$reg),
13998              as_FloatRegister($src3$$reg));
13999   %}
14000 
14001   ins_pipe(pipe_class_default);
14002 %}
14003 
14004 // src1 * src2 + src3
14005 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14006   match(Set dst (FmaD src3 (Binary src1 src2)));
14007 
14008   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
14009 
14010   ins_encode %{
14011     assert(UseFMA, "Needs FMA instructions support.");
14012     __ fmaddd(as_FloatRegister($dst$$reg),
14013              as_FloatRegister($src1$$reg),
14014              as_FloatRegister($src2$$reg),
14015              as_FloatRegister($src3$$reg));
14016   %}
14017 
14018   ins_pipe(pipe_class_default);
14019 %}
14020 
14021 // src1 * (-src2) + src3
14022 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
14023 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14024   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14025 
14026   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
14027 
14028   ins_encode %{
14029     assert(UseFMA, "Needs FMA instructions support.");
14030     __ fmsubs(as_FloatRegister($dst$$reg),
14031               as_FloatRegister($src1$$reg),
14032               as_FloatRegister($src2$$reg),
14033               as_FloatRegister($src3$$reg));
14034   %}
14035 
14036   ins_pipe(pipe_class_default);
14037 %}
14038 
14039 // src1 * (-src2) + src3
14040 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
14041 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14042   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14043 
14044   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
14045 
14046   ins_encode %{
14047     assert(UseFMA, "Needs FMA instructions support.");
14048     __ fmsubd(as_FloatRegister($dst$$reg),
14049               as_FloatRegister($src1$$reg),
14050               as_FloatRegister($src2$$reg),
14051               as_FloatRegister($src3$$reg));
14052   %}
14053 
14054   ins_pipe(pipe_class_default);
14055 %}
14056 
14057 // src1 * (-src2) - src3
14058 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14059 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14060   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14061 
14062   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14063 
14064   ins_encode %{
14065     assert(UseFMA, "Needs FMA instructions support.");
14066     __ fnmadds(as_FloatRegister($dst$$reg),
14067                as_FloatRegister($src1$$reg),
14068                as_FloatRegister($src2$$reg),
14069                as_FloatRegister($src3$$reg));
14070   %}
14071 
14072   ins_pipe(pipe_class_default);
14073 %}
14074 
14075 // src1 * (-src2) - src3
14076 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14077 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14078   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14079 
14080   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14081 
14082   ins_encode %{
14083     assert(UseFMA, "Needs FMA instructions support.");
14084     __ fnmaddd(as_FloatRegister($dst$$reg),
14085                as_FloatRegister($src1$$reg),
14086                as_FloatRegister($src2$$reg),
14087                as_FloatRegister($src3$$reg));
14088   %}
14089 
14090   ins_pipe(pipe_class_default);
14091 %}
14092 
14093 // src1 * src2 - src3
14094 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14095   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14096 
14097   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14098 
14099   ins_encode %{
14100     assert(UseFMA, "Needs FMA instructions support.");
14101     __ fnmsubs(as_FloatRegister($dst$$reg),
14102                as_FloatRegister($src1$$reg),
14103                as_FloatRegister($src2$$reg),
14104                as_FloatRegister($src3$$reg));
14105   %}
14106 
14107   ins_pipe(pipe_class_default);
14108 %}
14109 
14110 // src1 * src2 - src3
14111 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14112   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14113 
14114   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14115 
14116   ins_encode %{
14117     assert(UseFMA, "Needs FMA instructions support.");
14118     // n.b. insn name should be fnmsubd
14119     __ fnmsub(as_FloatRegister($dst$$reg),
14120               as_FloatRegister($src1$$reg),
14121               as_FloatRegister($src2$$reg),
14122               as_FloatRegister($src3$$reg));
14123   %}
14124 
14125   ins_pipe(pipe_class_default);
14126 %}
14127 
14128 // Math.max(HH)H (half-precision float)
14129 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14130   match(Set dst (MaxHF src1 src2));
14131   format %{ "fmaxh $dst, $src1, $src2" %}
14132   ins_encode %{
14133     __ fmaxh($dst$$FloatRegister,
14134              $src1$$FloatRegister,
14135              $src2$$FloatRegister);
14136   %}
14137   ins_pipe(fp_dop_reg_reg_s);
14138 %}
14139 
14140 // Math.min(HH)H (half-precision float)
14141 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14142   match(Set dst (MinHF src1 src2));
14143   format %{ "fminh $dst, $src1, $src2" %}
14144   ins_encode %{
14145     __ fminh($dst$$FloatRegister,
14146              $src1$$FloatRegister,
14147              $src2$$FloatRegister);
14148   %}
14149   ins_pipe(fp_dop_reg_reg_s);
14150 %}
14151 
14152 // Math.max(FF)F
14153 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14154   match(Set dst (MaxF src1 src2));
14155 
14156   format %{ "fmaxs   $dst, $src1, $src2" %}
14157   ins_encode %{
14158     __ fmaxs(as_FloatRegister($dst$$reg),
14159              as_FloatRegister($src1$$reg),
14160              as_FloatRegister($src2$$reg));
14161   %}
14162 
14163   ins_pipe(fp_dop_reg_reg_s);
14164 %}
14165 
14166 // Math.min(FF)F
14167 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14168   match(Set dst (MinF src1 src2));
14169 
14170   format %{ "fmins   $dst, $src1, $src2" %}
14171   ins_encode %{
14172     __ fmins(as_FloatRegister($dst$$reg),
14173              as_FloatRegister($src1$$reg),
14174              as_FloatRegister($src2$$reg));
14175   %}
14176 
14177   ins_pipe(fp_dop_reg_reg_s);
14178 %}
14179 
14180 // Math.max(DD)D
14181 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14182   match(Set dst (MaxD src1 src2));
14183 
14184   format %{ "fmaxd   $dst, $src1, $src2" %}
14185   ins_encode %{
14186     __ fmaxd(as_FloatRegister($dst$$reg),
14187              as_FloatRegister($src1$$reg),
14188              as_FloatRegister($src2$$reg));
14189   %}
14190 
14191   ins_pipe(fp_dop_reg_reg_d);
14192 %}
14193 
14194 // Math.min(DD)D
14195 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14196   match(Set dst (MinD src1 src2));
14197 
14198   format %{ "fmind   $dst, $src1, $src2" %}
14199   ins_encode %{
14200     __ fmind(as_FloatRegister($dst$$reg),
14201              as_FloatRegister($src1$$reg),
14202              as_FloatRegister($src2$$reg));
14203   %}
14204 
14205   ins_pipe(fp_dop_reg_reg_d);
14206 %}
14207 
14208 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14209   match(Set dst (DivHF src1  src2));
14210   format %{ "fdivh $dst, $src1, $src2" %}
14211   ins_encode %{
14212     __ fdivh($dst$$FloatRegister,
14213              $src1$$FloatRegister,
14214              $src2$$FloatRegister);
14215   %}
14216   ins_pipe(fp_div_s);
14217 %}
14218 
14219 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14220   match(Set dst (DivF src1  src2));
14221 
14222   ins_cost(INSN_COST * 18);
14223   format %{ "fdivs   $dst, $src1, $src2" %}
14224 
14225   ins_encode %{
14226     __ fdivs(as_FloatRegister($dst$$reg),
14227              as_FloatRegister($src1$$reg),
14228              as_FloatRegister($src2$$reg));
14229   %}
14230 
14231   ins_pipe(fp_div_s);
14232 %}
14233 
14234 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14235   match(Set dst (DivD src1  src2));
14236 
14237   ins_cost(INSN_COST * 32);
14238   format %{ "fdivd   $dst, $src1, $src2" %}
14239 
14240   ins_encode %{
14241     __ fdivd(as_FloatRegister($dst$$reg),
14242              as_FloatRegister($src1$$reg),
14243              as_FloatRegister($src2$$reg));
14244   %}
14245 
14246   ins_pipe(fp_div_d);
14247 %}
14248 
14249 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14250   match(Set dst (NegF src));
14251 
14252   ins_cost(INSN_COST * 3);
14253   format %{ "fneg   $dst, $src" %}
14254 
14255   ins_encode %{
14256     __ fnegs(as_FloatRegister($dst$$reg),
14257              as_FloatRegister($src$$reg));
14258   %}
14259 
14260   ins_pipe(fp_uop_s);
14261 %}
14262 
14263 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14264   match(Set dst (NegD src));
14265 
14266   ins_cost(INSN_COST * 3);
14267   format %{ "fnegd   $dst, $src" %}
14268 
14269   ins_encode %{
14270     __ fnegd(as_FloatRegister($dst$$reg),
14271              as_FloatRegister($src$$reg));
14272   %}
14273 
14274   ins_pipe(fp_uop_d);
14275 %}
14276 
14277 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14278 %{
14279   match(Set dst (AbsI src));
14280 
14281   effect(KILL cr);
14282   ins_cost(INSN_COST * 2);
14283   format %{ "cmpw  $src, zr\n\t"
14284             "cnegw $dst, $src, Assembler::LT\t# int abs"
14285   %}
14286 
14287   ins_encode %{
14288     __ cmpw(as_Register($src$$reg), zr);
14289     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14290   %}
14291   ins_pipe(pipe_class_default);
14292 %}
14293 
14294 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14295 %{
14296   match(Set dst (AbsL src));
14297 
14298   effect(KILL cr);
14299   ins_cost(INSN_COST * 2);
14300   format %{ "cmp  $src, zr\n\t"
14301             "cneg $dst, $src, Assembler::LT\t# long abs"
14302   %}
14303 
14304   ins_encode %{
14305     __ cmp(as_Register($src$$reg), zr);
14306     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14307   %}
14308   ins_pipe(pipe_class_default);
14309 %}
14310 
14311 instruct absF_reg(vRegF dst, vRegF src) %{
14312   match(Set dst (AbsF src));
14313 
14314   ins_cost(INSN_COST * 3);
14315   format %{ "fabss   $dst, $src" %}
14316   ins_encode %{
14317     __ fabss(as_FloatRegister($dst$$reg),
14318              as_FloatRegister($src$$reg));
14319   %}
14320 
14321   ins_pipe(fp_uop_s);
14322 %}
14323 
14324 instruct absD_reg(vRegD dst, vRegD src) %{
14325   match(Set dst (AbsD src));
14326 
14327   ins_cost(INSN_COST * 3);
14328   format %{ "fabsd   $dst, $src" %}
14329   ins_encode %{
14330     __ fabsd(as_FloatRegister($dst$$reg),
14331              as_FloatRegister($src$$reg));
14332   %}
14333 
14334   ins_pipe(fp_uop_d);
14335 %}
14336 
14337 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14338   match(Set dst (AbsF (SubF src1 src2)));
14339 
14340   ins_cost(INSN_COST * 3);
14341   format %{ "fabds   $dst, $src1, $src2" %}
14342   ins_encode %{
14343     __ fabds(as_FloatRegister($dst$$reg),
14344              as_FloatRegister($src1$$reg),
14345              as_FloatRegister($src2$$reg));
14346   %}
14347 
14348   ins_pipe(fp_uop_s);
14349 %}
14350 
14351 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14352   match(Set dst (AbsD (SubD src1 src2)));
14353 
14354   ins_cost(INSN_COST * 3);
14355   format %{ "fabdd   $dst, $src1, $src2" %}
14356   ins_encode %{
14357     __ fabdd(as_FloatRegister($dst$$reg),
14358              as_FloatRegister($src1$$reg),
14359              as_FloatRegister($src2$$reg));
14360   %}
14361 
14362   ins_pipe(fp_uop_d);
14363 %}
14364 
14365 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14366   match(Set dst (SqrtD src));
14367 
14368   ins_cost(INSN_COST * 50);
14369   format %{ "fsqrtd  $dst, $src" %}
14370   ins_encode %{
14371     __ fsqrtd(as_FloatRegister($dst$$reg),
14372              as_FloatRegister($src$$reg));
14373   %}
14374 
14375   ins_pipe(fp_div_s);
14376 %}
14377 
14378 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14379   match(Set dst (SqrtF src));
14380 
14381   ins_cost(INSN_COST * 50);
14382   format %{ "fsqrts  $dst, $src" %}
14383   ins_encode %{
14384     __ fsqrts(as_FloatRegister($dst$$reg),
14385              as_FloatRegister($src$$reg));
14386   %}
14387 
14388   ins_pipe(fp_div_d);
14389 %}
14390 
14391 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
14392   match(Set dst (SqrtHF src));
14393   format %{ "fsqrth $dst, $src" %}
14394   ins_encode %{
14395     __ fsqrth($dst$$FloatRegister,
14396               $src$$FloatRegister);
14397   %}
14398   ins_pipe(fp_div_s);
14399 %}
14400 
14401 // Math.rint, floor, ceil
14402 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14403   match(Set dst (RoundDoubleMode src rmode));
14404   format %{ "frint  $dst, $src, $rmode" %}
14405   ins_encode %{
14406     switch ($rmode$$constant) {
14407       case RoundDoubleModeNode::rmode_rint:
14408         __ frintnd(as_FloatRegister($dst$$reg),
14409                    as_FloatRegister($src$$reg));
14410         break;
14411       case RoundDoubleModeNode::rmode_floor:
14412         __ frintmd(as_FloatRegister($dst$$reg),
14413                    as_FloatRegister($src$$reg));
14414         break;
14415       case RoundDoubleModeNode::rmode_ceil:
14416         __ frintpd(as_FloatRegister($dst$$reg),
14417                    as_FloatRegister($src$$reg));
14418         break;
14419     }
14420   %}
14421   ins_pipe(fp_uop_d);
14422 %}
14423 
14424 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14425   match(Set dst (CopySignD src1 (Binary src2 zero)));
14426   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14427   format %{ "CopySignD  $dst $src1 $src2" %}
14428   ins_encode %{
14429     FloatRegister dst = as_FloatRegister($dst$$reg),
14430                   src1 = as_FloatRegister($src1$$reg),
14431                   src2 = as_FloatRegister($src2$$reg),
14432                   zero = as_FloatRegister($zero$$reg);
14433     __ fnegd(dst, zero);
14434     __ bsl(dst, __ T8B, src2, src1);
14435   %}
14436   ins_pipe(fp_uop_d);
14437 %}
14438 
14439 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14440   match(Set dst (CopySignF src1 src2));
14441   effect(TEMP_DEF dst, USE src1, USE src2);
14442   format %{ "CopySignF  $dst $src1 $src2" %}
14443   ins_encode %{
14444     FloatRegister dst = as_FloatRegister($dst$$reg),
14445                   src1 = as_FloatRegister($src1$$reg),
14446                   src2 = as_FloatRegister($src2$$reg);
14447     __ movi(dst, __ T2S, 0x80, 24);
14448     __ bsl(dst, __ T8B, src2, src1);
14449   %}
14450   ins_pipe(fp_uop_d);
14451 %}
14452 
14453 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14454   match(Set dst (SignumD src (Binary zero one)));
14455   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14456   format %{ "signumD  $dst, $src" %}
14457   ins_encode %{
14458     FloatRegister src = as_FloatRegister($src$$reg),
14459                   dst = as_FloatRegister($dst$$reg),
14460                   zero = as_FloatRegister($zero$$reg),
14461                   one = as_FloatRegister($one$$reg);
14462     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14463     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14464     // Bit selection instruction gets bit from "one" for each enabled bit in
14465     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14466     // NaN the whole "src" will be copied because "dst" is zero. For all other
14467     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14468     // from "src", and all other bits are copied from 1.0.
14469     __ bsl(dst, __ T8B, one, src);
14470   %}
14471   ins_pipe(fp_uop_d);
14472 %}
14473 
14474 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14475   match(Set dst (SignumF src (Binary zero one)));
14476   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14477   format %{ "signumF  $dst, $src" %}
14478   ins_encode %{
14479     FloatRegister src = as_FloatRegister($src$$reg),
14480                   dst = as_FloatRegister($dst$$reg),
14481                   zero = as_FloatRegister($zero$$reg),
14482                   one = as_FloatRegister($one$$reg);
14483     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14484     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14485     // Bit selection instruction gets bit from "one" for each enabled bit in
14486     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14487     // NaN the whole "src" will be copied because "dst" is zero. For all other
14488     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14489     // from "src", and all other bits are copied from 1.0.
14490     __ bsl(dst, __ T8B, one, src);
14491   %}
14492   ins_pipe(fp_uop_d);
14493 %}
14494 
14495 instruct onspinwait() %{
14496   match(OnSpinWait);
14497   ins_cost(INSN_COST);
14498 
14499   format %{ "onspinwait" %}
14500 
14501   ins_encode %{
14502     __ spin_wait();
14503   %}
14504   ins_pipe(pipe_class_empty);
14505 %}
14506 
14507 // ============================================================================
14508 // Logical Instructions
14509 
14510 // Integer Logical Instructions
14511 
14512 // And Instructions
14513 
14514 
14515 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14516   match(Set dst (AndI src1 src2));
14517 
14518   format %{ "andw  $dst, $src1, $src2\t# int" %}
14519 
14520   ins_cost(INSN_COST);
14521   ins_encode %{
14522     __ andw(as_Register($dst$$reg),
14523             as_Register($src1$$reg),
14524             as_Register($src2$$reg));
14525   %}
14526 
14527   ins_pipe(ialu_reg_reg);
14528 %}
14529 
14530 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14531   match(Set dst (AndI src1 src2));
14532 
14533   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14534 
14535   ins_cost(INSN_COST);
14536   ins_encode %{
14537     __ andw(as_Register($dst$$reg),
14538             as_Register($src1$$reg),
14539             (uint64_t)($src2$$constant));
14540   %}
14541 
14542   ins_pipe(ialu_reg_imm);
14543 %}
14544 
14545 // Or Instructions
14546 
14547 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14548   match(Set dst (OrI src1 src2));
14549 
14550   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14551 
14552   ins_cost(INSN_COST);
14553   ins_encode %{
14554     __ orrw(as_Register($dst$$reg),
14555             as_Register($src1$$reg),
14556             as_Register($src2$$reg));
14557   %}
14558 
14559   ins_pipe(ialu_reg_reg);
14560 %}
14561 
14562 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14563   match(Set dst (OrI src1 src2));
14564 
14565   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14566 
14567   ins_cost(INSN_COST);
14568   ins_encode %{
14569     __ orrw(as_Register($dst$$reg),
14570             as_Register($src1$$reg),
14571             (uint64_t)($src2$$constant));
14572   %}
14573 
14574   ins_pipe(ialu_reg_imm);
14575 %}
14576 
14577 // Xor Instructions
14578 
14579 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14580   match(Set dst (XorI src1 src2));
14581 
14582   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14583 
14584   ins_cost(INSN_COST);
14585   ins_encode %{
14586     __ eorw(as_Register($dst$$reg),
14587             as_Register($src1$$reg),
14588             as_Register($src2$$reg));
14589   %}
14590 
14591   ins_pipe(ialu_reg_reg);
14592 %}
14593 
14594 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14595   match(Set dst (XorI src1 src2));
14596 
14597   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14598 
14599   ins_cost(INSN_COST);
14600   ins_encode %{
14601     __ eorw(as_Register($dst$$reg),
14602             as_Register($src1$$reg),
14603             (uint64_t)($src2$$constant));
14604   %}
14605 
14606   ins_pipe(ialu_reg_imm);
14607 %}
14608 
14609 // Long Logical Instructions
14610 // TODO
14611 
14612 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14613   match(Set dst (AndL src1 src2));
14614 
14615   format %{ "and  $dst, $src1, $src2\t# int" %}
14616 
14617   ins_cost(INSN_COST);
14618   ins_encode %{
14619     __ andr(as_Register($dst$$reg),
14620             as_Register($src1$$reg),
14621             as_Register($src2$$reg));
14622   %}
14623 
14624   ins_pipe(ialu_reg_reg);
14625 %}
14626 
14627 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14628   match(Set dst (AndL src1 src2));
14629 
14630   format %{ "and  $dst, $src1, $src2\t# int" %}
14631 
14632   ins_cost(INSN_COST);
14633   ins_encode %{
14634     __ andr(as_Register($dst$$reg),
14635             as_Register($src1$$reg),
14636             (uint64_t)($src2$$constant));
14637   %}
14638 
14639   ins_pipe(ialu_reg_imm);
14640 %}
14641 
14642 // Or Instructions
14643 
14644 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14645   match(Set dst (OrL src1 src2));
14646 
14647   format %{ "orr  $dst, $src1, $src2\t# int" %}
14648 
14649   ins_cost(INSN_COST);
14650   ins_encode %{
14651     __ orr(as_Register($dst$$reg),
14652            as_Register($src1$$reg),
14653            as_Register($src2$$reg));
14654   %}
14655 
14656   ins_pipe(ialu_reg_reg);
14657 %}
14658 
14659 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14660   match(Set dst (OrL src1 src2));
14661 
14662   format %{ "orr  $dst, $src1, $src2\t# int" %}
14663 
14664   ins_cost(INSN_COST);
14665   ins_encode %{
14666     __ orr(as_Register($dst$$reg),
14667            as_Register($src1$$reg),
14668            (uint64_t)($src2$$constant));
14669   %}
14670 
14671   ins_pipe(ialu_reg_imm);
14672 %}
14673 
14674 // Xor Instructions
14675 
14676 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14677   match(Set dst (XorL src1 src2));
14678 
14679   format %{ "eor  $dst, $src1, $src2\t# int" %}
14680 
14681   ins_cost(INSN_COST);
14682   ins_encode %{
14683     __ eor(as_Register($dst$$reg),
14684            as_Register($src1$$reg),
14685            as_Register($src2$$reg));
14686   %}
14687 
14688   ins_pipe(ialu_reg_reg);
14689 %}
14690 
14691 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14692   match(Set dst (XorL src1 src2));
14693 
14694   ins_cost(INSN_COST);
14695   format %{ "eor  $dst, $src1, $src2\t# int" %}
14696 
14697   ins_encode %{
14698     __ eor(as_Register($dst$$reg),
14699            as_Register($src1$$reg),
14700            (uint64_t)($src2$$constant));
14701   %}
14702 
14703   ins_pipe(ialu_reg_imm);
14704 %}
14705 
14706 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14707 %{
14708   match(Set dst (ConvI2L src));
14709 
14710   ins_cost(INSN_COST);
14711   format %{ "sxtw  $dst, $src\t# i2l" %}
14712   ins_encode %{
14713     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14714   %}
14715   ins_pipe(ialu_reg_shift);
14716 %}
14717 
14718 // this pattern occurs in bigmath arithmetic
14719 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14720 %{
14721   match(Set dst (AndL (ConvI2L src) mask));
14722 
14723   ins_cost(INSN_COST);
14724   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14725   ins_encode %{
14726     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14727   %}
14728 
14729   ins_pipe(ialu_reg_shift);
14730 %}
14731 
14732 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14733   match(Set dst (ConvL2I src));
14734 
14735   ins_cost(INSN_COST);
14736   format %{ "movw  $dst, $src \t// l2i" %}
14737 
14738   ins_encode %{
14739     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14740   %}
14741 
14742   ins_pipe(ialu_reg);
14743 %}
14744 
14745 instruct convD2F_reg(vRegF dst, vRegD src) %{
14746   match(Set dst (ConvD2F src));
14747 
14748   ins_cost(INSN_COST * 5);
14749   format %{ "fcvtd  $dst, $src \t// d2f" %}
14750 
14751   ins_encode %{
14752     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14753   %}
14754 
14755   ins_pipe(fp_d2f);
14756 %}
14757 
14758 instruct convF2D_reg(vRegD dst, vRegF src) %{
14759   match(Set dst (ConvF2D src));
14760 
14761   ins_cost(INSN_COST * 5);
14762   format %{ "fcvts  $dst, $src \t// f2d" %}
14763 
14764   ins_encode %{
14765     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14766   %}
14767 
14768   ins_pipe(fp_f2d);
14769 %}
14770 
14771 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14772   match(Set dst (ConvF2I src));
14773 
14774   ins_cost(INSN_COST * 5);
14775   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14776 
14777   ins_encode %{
14778     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14779   %}
14780 
14781   ins_pipe(fp_f2i);
14782 %}
14783 
14784 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14785   match(Set dst (ConvF2L src));
14786 
14787   ins_cost(INSN_COST * 5);
14788   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14789 
14790   ins_encode %{
14791     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14792   %}
14793 
14794   ins_pipe(fp_f2l);
14795 %}
14796 
14797 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14798   match(Set dst (ConvF2HF src));
14799   format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14800             "smov $dst, $tmp\t# move result from $tmp to $dst"
14801   %}
14802   effect(TEMP tmp);
14803   ins_encode %{
14804       __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14805   %}
14806   ins_pipe(pipe_slow);
14807 %}
14808 
14809 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14810   match(Set dst (ConvHF2F src));
14811   format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14812             "fcvt $dst, $tmp\t# convert half to single precision"
14813   %}
14814   effect(TEMP tmp);
14815   ins_encode %{
14816       __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14817   %}
14818   ins_pipe(pipe_slow);
14819 %}
14820 
14821 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14822   match(Set dst (ConvI2F src));
14823 
14824   ins_cost(INSN_COST * 5);
14825   format %{ "scvtfws  $dst, $src \t// i2f" %}
14826 
14827   ins_encode %{
14828     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14829   %}
14830 
14831   ins_pipe(fp_i2f);
14832 %}
14833 
14834 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14835   match(Set dst (ConvL2F src));
14836 
14837   ins_cost(INSN_COST * 5);
14838   format %{ "scvtfs  $dst, $src \t// l2f" %}
14839 
14840   ins_encode %{
14841     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14842   %}
14843 
14844   ins_pipe(fp_l2f);
14845 %}
14846 
14847 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14848   match(Set dst (ConvD2I src));
14849 
14850   ins_cost(INSN_COST * 5);
14851   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14852 
14853   ins_encode %{
14854     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14855   %}
14856 
14857   ins_pipe(fp_d2i);
14858 %}
14859 
14860 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14861   match(Set dst (ConvD2L src));
14862 
14863   ins_cost(INSN_COST * 5);
14864   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14865 
14866   ins_encode %{
14867     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14868   %}
14869 
14870   ins_pipe(fp_d2l);
14871 %}
14872 
14873 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14874   match(Set dst (ConvI2D src));
14875 
14876   ins_cost(INSN_COST * 5);
14877   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14878 
14879   ins_encode %{
14880     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14881   %}
14882 
14883   ins_pipe(fp_i2d);
14884 %}
14885 
14886 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14887   match(Set dst (ConvL2D src));
14888 
14889   ins_cost(INSN_COST * 5);
14890   format %{ "scvtfd  $dst, $src \t// l2d" %}
14891 
14892   ins_encode %{
14893     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14894   %}
14895 
14896   ins_pipe(fp_l2d);
14897 %}
14898 
14899 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14900 %{
14901   match(Set dst (RoundD src));
14902   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14903   format %{ "java_round_double $dst,$src"%}
14904   ins_encode %{
14905     __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14906                          as_FloatRegister($ftmp$$reg));
14907   %}
14908   ins_pipe(pipe_slow);
14909 %}
14910 
14911 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14912 %{
14913   match(Set dst (RoundF src));
14914   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14915   format %{ "java_round_float $dst,$src"%}
14916   ins_encode %{
14917     __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14918                         as_FloatRegister($ftmp$$reg));
14919   %}
14920   ins_pipe(pipe_slow);
14921 %}
14922 
14923 // stack <-> reg and reg <-> reg shuffles with no conversion
14924 
14925 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14926 
14927   match(Set dst (MoveF2I src));
14928 
14929   effect(DEF dst, USE src);
14930 
14931   ins_cost(4 * INSN_COST);
14932 
14933   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14934 
14935   ins_encode %{
14936     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14937   %}
14938 
14939   ins_pipe(iload_reg_reg);
14940 
14941 %}
14942 
14943 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14944 
14945   match(Set dst (MoveI2F src));
14946 
14947   effect(DEF dst, USE src);
14948 
14949   ins_cost(4 * INSN_COST);
14950 
14951   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14952 
14953   ins_encode %{
14954     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14955   %}
14956 
14957   ins_pipe(pipe_class_memory);
14958 
14959 %}
14960 
14961 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14962 
14963   match(Set dst (MoveD2L src));
14964 
14965   effect(DEF dst, USE src);
14966 
14967   ins_cost(4 * INSN_COST);
14968 
14969   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14970 
14971   ins_encode %{
14972     __ ldr($dst$$Register, Address(sp, $src$$disp));
14973   %}
14974 
14975   ins_pipe(iload_reg_reg);
14976 
14977 %}
14978 
14979 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14980 
14981   match(Set dst (MoveL2D src));
14982 
14983   effect(DEF dst, USE src);
14984 
14985   ins_cost(4 * INSN_COST);
14986 
14987   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14988 
14989   ins_encode %{
14990     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14991   %}
14992 
14993   ins_pipe(pipe_class_memory);
14994 
14995 %}
14996 
14997 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14998 
14999   match(Set dst (MoveF2I src));
15000 
15001   effect(DEF dst, USE src);
15002 
15003   ins_cost(INSN_COST);
15004 
15005   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15006 
15007   ins_encode %{
15008     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15009   %}
15010 
15011   ins_pipe(pipe_class_memory);
15012 
15013 %}
15014 
15015 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15016 
15017   match(Set dst (MoveI2F src));
15018 
15019   effect(DEF dst, USE src);
15020 
15021   ins_cost(INSN_COST);
15022 
15023   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15024 
15025   ins_encode %{
15026     __ strw($src$$Register, Address(sp, $dst$$disp));
15027   %}
15028 
15029   ins_pipe(istore_reg_reg);
15030 
15031 %}
15032 
15033 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15034 
15035   match(Set dst (MoveD2L src));
15036 
15037   effect(DEF dst, USE src);
15038 
15039   ins_cost(INSN_COST);
15040 
15041   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15042 
15043   ins_encode %{
15044     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15045   %}
15046 
15047   ins_pipe(pipe_class_memory);
15048 
15049 %}
15050 
15051 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15052 
15053   match(Set dst (MoveL2D src));
15054 
15055   effect(DEF dst, USE src);
15056 
15057   ins_cost(INSN_COST);
15058 
15059   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15060 
15061   ins_encode %{
15062     __ str($src$$Register, Address(sp, $dst$$disp));
15063   %}
15064 
15065   ins_pipe(istore_reg_reg);
15066 
15067 %}
15068 
15069 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15070 
15071   match(Set dst (MoveF2I src));
15072 
15073   effect(DEF dst, USE src);
15074 
15075   ins_cost(INSN_COST);
15076 
15077   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15078 
15079   ins_encode %{
15080     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15081   %}
15082 
15083   ins_pipe(fp_f2i);
15084 
15085 %}
15086 
15087 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15088 
15089   match(Set dst (MoveI2F src));
15090 
15091   effect(DEF dst, USE src);
15092 
15093   ins_cost(INSN_COST);
15094 
15095   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15096 
15097   ins_encode %{
15098     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15099   %}
15100 
15101   ins_pipe(fp_i2f);
15102 
15103 %}
15104 
15105 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15106 
15107   match(Set dst (MoveD2L src));
15108 
15109   effect(DEF dst, USE src);
15110 
15111   ins_cost(INSN_COST);
15112 
15113   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15114 
15115   ins_encode %{
15116     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15117   %}
15118 
15119   ins_pipe(fp_d2l);
15120 
15121 %}
15122 
15123 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15124 
15125   match(Set dst (MoveL2D src));
15126 
15127   effect(DEF dst, USE src);
15128 
15129   ins_cost(INSN_COST);
15130 
15131   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15132 
15133   ins_encode %{
15134     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15135   %}
15136 
15137   ins_pipe(fp_l2d);
15138 
15139 %}
15140 
15141 // ============================================================================
15142 // clearing of an array
15143 
15144 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
15145 %{
15146   match(Set dummy (ClearArray (Binary cnt base) zero));
15147   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15148 
15149   ins_cost(4 * INSN_COST);
15150   format %{ "ClearArray $cnt, $base" %}
15151 
15152   ins_encode %{
15153     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15154     if (tpc == nullptr) {
15155       ciEnv::current()->record_failure("CodeCache is full");
15156       return;
15157     }
15158   %}
15159 
15160   ins_pipe(pipe_class_memory);
15161 %}
15162 
15163 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
15164 %{
15165   predicate(((ClearArrayNode*)n)->word_copy_only());
15166   match(Set dummy (ClearArray (Binary cnt base) val));
15167   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15168 
15169   ins_cost(4 * INSN_COST);
15170   format %{ "ClearArray $cnt, $base, $val" %}
15171 
15172   ins_encode %{
15173     __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
15174   %}
15175 
15176   ins_pipe(pipe_class_memory);
15177 %}
15178 
15179 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15180 %{
15181   predicate((uint64_t)n->in(2)->get_long()
15182             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
15183             && !((ClearArrayNode*)n)->word_copy_only());
15184   match(Set dummy (ClearArray cnt base));
15185   effect(TEMP temp, USE_KILL base, KILL cr);
15186 
15187   ins_cost(4 * INSN_COST);
15188   format %{ "ClearArray $cnt, $base" %}
15189 
15190   ins_encode %{
15191     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15192     if (tpc == nullptr) {
15193       ciEnv::current()->record_failure("CodeCache is full");
15194       return;
15195     }
15196   %}
15197 
15198   ins_pipe(pipe_class_memory);
15199 %}
15200 
15201 // ============================================================================
15202 // Overflow Math Instructions
15203 
15204 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15205 %{
15206   match(Set cr (OverflowAddI op1 op2));
15207 
15208   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15209   ins_cost(INSN_COST);
15210   ins_encode %{
15211     __ cmnw($op1$$Register, $op2$$Register);
15212   %}
15213 
15214   ins_pipe(icmp_reg_reg);
15215 %}
15216 
15217 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15218 %{
15219   match(Set cr (OverflowAddI op1 op2));
15220 
15221   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15222   ins_cost(INSN_COST);
15223   ins_encode %{
15224     __ cmnw($op1$$Register, $op2$$constant);
15225   %}
15226 
15227   ins_pipe(icmp_reg_imm);
15228 %}
15229 
15230 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15231 %{
15232   match(Set cr (OverflowAddL op1 op2));
15233 
15234   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15235   ins_cost(INSN_COST);
15236   ins_encode %{
15237     __ cmn($op1$$Register, $op2$$Register);
15238   %}
15239 
15240   ins_pipe(icmp_reg_reg);
15241 %}
15242 
15243 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15244 %{
15245   match(Set cr (OverflowAddL op1 op2));
15246 
15247   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
15248   ins_cost(INSN_COST);
15249   ins_encode %{
15250     __ adds(zr, $op1$$Register, $op2$$constant);
15251   %}
15252 
15253   ins_pipe(icmp_reg_imm);
15254 %}
15255 
15256 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15257 %{
15258   match(Set cr (OverflowSubI op1 op2));
15259 
15260   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15261   ins_cost(INSN_COST);
15262   ins_encode %{
15263     __ cmpw($op1$$Register, $op2$$Register);
15264   %}
15265 
15266   ins_pipe(icmp_reg_reg);
15267 %}
15268 
15269 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15270 %{
15271   match(Set cr (OverflowSubI op1 op2));
15272 
15273   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15274   ins_cost(INSN_COST);
15275   ins_encode %{
15276     __ cmpw($op1$$Register, $op2$$constant);
15277   %}
15278 
15279   ins_pipe(icmp_reg_imm);
15280 %}
15281 
15282 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15283 %{
15284   match(Set cr (OverflowSubL op1 op2));
15285 
15286   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15287   ins_cost(INSN_COST);
15288   ins_encode %{
15289     __ cmp($op1$$Register, $op2$$Register);
15290   %}
15291 
15292   ins_pipe(icmp_reg_reg);
15293 %}
15294 
15295 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15296 %{
15297   match(Set cr (OverflowSubL op1 op2));
15298 
15299   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15300   ins_cost(INSN_COST);
15301   ins_encode %{
15302     __ subs(zr, $op1$$Register, $op2$$constant);
15303   %}
15304 
15305   ins_pipe(icmp_reg_imm);
15306 %}
15307 
15308 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15309 %{
15310   match(Set cr (OverflowSubI zero op1));
15311 
15312   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15313   ins_cost(INSN_COST);
15314   ins_encode %{
15315     __ cmpw(zr, $op1$$Register);
15316   %}
15317 
15318   ins_pipe(icmp_reg_imm);
15319 %}
15320 
15321 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15322 %{
15323   match(Set cr (OverflowSubL zero op1));
15324 
15325   format %{ "cmp   zr, $op1\t# overflow check long" %}
15326   ins_cost(INSN_COST);
15327   ins_encode %{
15328     __ cmp(zr, $op1$$Register);
15329   %}
15330 
15331   ins_pipe(icmp_reg_imm);
15332 %}
15333 
15334 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15335 %{
15336   match(Set cr (OverflowMulI op1 op2));
15337 
15338   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15339             "cmp   rscratch1, rscratch1, sxtw\n\t"
15340             "movw  rscratch1, #0x80000000\n\t"
15341             "cselw rscratch1, rscratch1, zr, NE\n\t"
15342             "cmpw  rscratch1, #1" %}
15343   ins_cost(5 * INSN_COST);
15344   ins_encode %{
15345     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15346     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15347     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15348     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15349     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15350   %}
15351 
15352   ins_pipe(pipe_slow);
15353 %}
15354 
15355 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15356 %{
15357   match(If cmp (OverflowMulI op1 op2));
15358   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15359             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15360   effect(USE labl, KILL cr);
15361 
15362   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15363             "cmp   rscratch1, rscratch1, sxtw\n\t"
15364             "b$cmp   $labl" %}
15365   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15366   ins_encode %{
15367     Label* L = $labl$$label;
15368     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15369     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15370     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15371     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15372   %}
15373 
15374   ins_pipe(pipe_serial);
15375 %}
15376 
15377 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15378 %{
15379   match(Set cr (OverflowMulL op1 op2));
15380 
15381   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15382             "smulh rscratch2, $op1, $op2\n\t"
15383             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15384             "movw  rscratch1, #0x80000000\n\t"
15385             "cselw rscratch1, rscratch1, zr, NE\n\t"
15386             "cmpw  rscratch1, #1" %}
15387   ins_cost(6 * INSN_COST);
15388   ins_encode %{
15389     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15390     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15391     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15392     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15393     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15394     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15395   %}
15396 
15397   ins_pipe(pipe_slow);
15398 %}
15399 
15400 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15401 %{
15402   match(If cmp (OverflowMulL op1 op2));
15403   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15404             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15405   effect(USE labl, KILL cr);
15406 
15407   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15408             "smulh rscratch2, $op1, $op2\n\t"
15409             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15410             "b$cmp $labl" %}
15411   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15412   ins_encode %{
15413     Label* L = $labl$$label;
15414     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15415     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15416     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15417     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15418     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15419   %}
15420 
15421   ins_pipe(pipe_serial);
15422 %}
15423 
15424 // ============================================================================
15425 // Compare Instructions
15426 
15427 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15428 %{
15429   match(Set cr (CmpI op1 op2));
15430 
15431   effect(DEF cr, USE op1, USE op2);
15432 
15433   ins_cost(INSN_COST);
15434   format %{ "cmpw  $op1, $op2" %}
15435 
15436   ins_encode(aarch64_enc_cmpw(op1, op2));
15437 
15438   ins_pipe(icmp_reg_reg);
15439 %}
15440 
15441 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15442 %{
15443   match(Set cr (CmpI op1 zero));
15444 
15445   effect(DEF cr, USE op1);
15446 
15447   ins_cost(INSN_COST);
15448   format %{ "cmpw $op1, 0" %}
15449 
15450   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15451 
15452   ins_pipe(icmp_reg_imm);
15453 %}
15454 
15455 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15456 %{
15457   match(Set cr (CmpI op1 op2));
15458 
15459   effect(DEF cr, USE op1);
15460 
15461   ins_cost(INSN_COST);
15462   format %{ "cmpw  $op1, $op2" %}
15463 
15464   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15465 
15466   ins_pipe(icmp_reg_imm);
15467 %}
15468 
15469 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15470 %{
15471   match(Set cr (CmpI op1 op2));
15472 
15473   effect(DEF cr, USE op1);
15474 
15475   ins_cost(INSN_COST * 2);
15476   format %{ "cmpw  $op1, $op2" %}
15477 
15478   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15479 
15480   ins_pipe(icmp_reg_imm);
15481 %}
15482 
15483 // Unsigned compare Instructions; really, same as signed compare
15484 // except it should only be used to feed an If or a CMovI which takes a
15485 // cmpOpU.
15486 
15487 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15488 %{
15489   match(Set cr (CmpU op1 op2));
15490 
15491   effect(DEF cr, USE op1, USE op2);
15492 
15493   ins_cost(INSN_COST);
15494   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15495 
15496   ins_encode(aarch64_enc_cmpw(op1, op2));
15497 
15498   ins_pipe(icmp_reg_reg);
15499 %}
15500 
15501 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15502 %{
15503   match(Set cr (CmpU op1 zero));
15504 
15505   effect(DEF cr, USE op1);
15506 
15507   ins_cost(INSN_COST);
15508   format %{ "cmpw $op1, #0\t# unsigned" %}
15509 
15510   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15511 
15512   ins_pipe(icmp_reg_imm);
15513 %}
15514 
15515 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15516 %{
15517   match(Set cr (CmpU op1 op2));
15518 
15519   effect(DEF cr, USE op1);
15520 
15521   ins_cost(INSN_COST);
15522   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15523 
15524   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15525 
15526   ins_pipe(icmp_reg_imm);
15527 %}
15528 
15529 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15530 %{
15531   match(Set cr (CmpU op1 op2));
15532 
15533   effect(DEF cr, USE op1);
15534 
15535   ins_cost(INSN_COST * 2);
15536   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15537 
15538   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15539 
15540   ins_pipe(icmp_reg_imm);
15541 %}
15542 
15543 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15544 %{
15545   match(Set cr (CmpL op1 op2));
15546 
15547   effect(DEF cr, USE op1, USE op2);
15548 
15549   ins_cost(INSN_COST);
15550   format %{ "cmp  $op1, $op2" %}
15551 
15552   ins_encode(aarch64_enc_cmp(op1, op2));
15553 
15554   ins_pipe(icmp_reg_reg);
15555 %}
15556 
15557 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15558 %{
15559   match(Set cr (CmpL op1 zero));
15560 
15561   effect(DEF cr, USE op1);
15562 
15563   ins_cost(INSN_COST);
15564   format %{ "tst  $op1" %}
15565 
15566   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15567 
15568   ins_pipe(icmp_reg_imm);
15569 %}
15570 
15571 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15572 %{
15573   match(Set cr (CmpL op1 op2));
15574 
15575   effect(DEF cr, USE op1);
15576 
15577   ins_cost(INSN_COST);
15578   format %{ "cmp  $op1, $op2" %}
15579 
15580   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15581 
15582   ins_pipe(icmp_reg_imm);
15583 %}
15584 
15585 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15586 %{
15587   match(Set cr (CmpL op1 op2));
15588 
15589   effect(DEF cr, USE op1);
15590 
15591   ins_cost(INSN_COST * 2);
15592   format %{ "cmp  $op1, $op2" %}
15593 
15594   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15595 
15596   ins_pipe(icmp_reg_imm);
15597 %}
15598 
15599 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15600 %{
15601   match(Set cr (CmpUL op1 op2));
15602 
15603   effect(DEF cr, USE op1, USE op2);
15604 
15605   ins_cost(INSN_COST);
15606   format %{ "cmp  $op1, $op2" %}
15607 
15608   ins_encode(aarch64_enc_cmp(op1, op2));
15609 
15610   ins_pipe(icmp_reg_reg);
15611 %}
15612 
15613 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15614 %{
15615   match(Set cr (CmpUL op1 zero));
15616 
15617   effect(DEF cr, USE op1);
15618 
15619   ins_cost(INSN_COST);
15620   format %{ "tst  $op1" %}
15621 
15622   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15623 
15624   ins_pipe(icmp_reg_imm);
15625 %}
15626 
15627 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15628 %{
15629   match(Set cr (CmpUL op1 op2));
15630 
15631   effect(DEF cr, USE op1);
15632 
15633   ins_cost(INSN_COST);
15634   format %{ "cmp  $op1, $op2" %}
15635 
15636   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15637 
15638   ins_pipe(icmp_reg_imm);
15639 %}
15640 
15641 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15642 %{
15643   match(Set cr (CmpUL op1 op2));
15644 
15645   effect(DEF cr, USE op1);
15646 
15647   ins_cost(INSN_COST * 2);
15648   format %{ "cmp  $op1, $op2" %}
15649 
15650   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15651 
15652   ins_pipe(icmp_reg_imm);
15653 %}
15654 
15655 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15656 %{
15657   match(Set cr (CmpP op1 op2));
15658 
15659   effect(DEF cr, USE op1, USE op2);
15660 
15661   ins_cost(INSN_COST);
15662   format %{ "cmp  $op1, $op2\t // ptr" %}
15663 
15664   ins_encode(aarch64_enc_cmpp(op1, op2));
15665 
15666   ins_pipe(icmp_reg_reg);
15667 %}
15668 
15669 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15670 %{
15671   match(Set cr (CmpN op1 op2));
15672 
15673   effect(DEF cr, USE op1, USE op2);
15674 
15675   ins_cost(INSN_COST);
15676   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15677 
15678   ins_encode(aarch64_enc_cmpn(op1, op2));
15679 
15680   ins_pipe(icmp_reg_reg);
15681 %}
15682 
15683 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15684 %{
15685   match(Set cr (CmpP op1 zero));
15686 
15687   effect(DEF cr, USE op1, USE zero);
15688 
15689   ins_cost(INSN_COST);
15690   format %{ "cmp  $op1, 0\t // ptr" %}
15691 
15692   ins_encode(aarch64_enc_testp(op1));
15693 
15694   ins_pipe(icmp_reg_imm);
15695 %}
15696 
15697 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15698 %{
15699   match(Set cr (CmpN op1 zero));
15700 
15701   effect(DEF cr, USE op1, USE zero);
15702 
15703   ins_cost(INSN_COST);
15704   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15705 
15706   ins_encode(aarch64_enc_testn(op1));
15707 
15708   ins_pipe(icmp_reg_imm);
15709 %}
15710 
15711 // FP comparisons
15712 //
15713 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15714 // using normal cmpOp. See declaration of rFlagsReg for details.
15715 
15716 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15717 %{
15718   match(Set cr (CmpF src1 src2));
15719 
15720   ins_cost(3 * INSN_COST);
15721   format %{ "fcmps $src1, $src2" %}
15722 
15723   ins_encode %{
15724     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15725   %}
15726 
15727   ins_pipe(pipe_class_compare);
15728 %}
15729 
15730 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15731 %{
15732   match(Set cr (CmpF src1 src2));
15733 
15734   ins_cost(3 * INSN_COST);
15735   format %{ "fcmps $src1, 0.0" %}
15736 
15737   ins_encode %{
15738     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15739   %}
15740 
15741   ins_pipe(pipe_class_compare);
15742 %}
15743 // FROM HERE
15744 
15745 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15746 %{
15747   match(Set cr (CmpD src1 src2));
15748 
15749   ins_cost(3 * INSN_COST);
15750   format %{ "fcmpd $src1, $src2" %}
15751 
15752   ins_encode %{
15753     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15754   %}
15755 
15756   ins_pipe(pipe_class_compare);
15757 %}
15758 
15759 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15760 %{
15761   match(Set cr (CmpD src1 src2));
15762 
15763   ins_cost(3 * INSN_COST);
15764   format %{ "fcmpd $src1, 0.0" %}
15765 
15766   ins_encode %{
15767     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15768   %}
15769 
15770   ins_pipe(pipe_class_compare);
15771 %}
15772 
15773 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15774 %{
15775   match(Set dst (CmpF3 src1 src2));
15776   effect(KILL cr);
15777 
15778   ins_cost(5 * INSN_COST);
15779   format %{ "fcmps $src1, $src2\n\t"
15780             "csinvw($dst, zr, zr, eq\n\t"
15781             "csnegw($dst, $dst, $dst, lt)"
15782   %}
15783 
15784   ins_encode %{
15785     Label done;
15786     FloatRegister s1 = as_FloatRegister($src1$$reg);
15787     FloatRegister s2 = as_FloatRegister($src2$$reg);
15788     Register d = as_Register($dst$$reg);
15789     __ fcmps(s1, s2);
15790     // installs 0 if EQ else -1
15791     __ csinvw(d, zr, zr, Assembler::EQ);
15792     // keeps -1 if less or unordered else installs 1
15793     __ csnegw(d, d, d, Assembler::LT);
15794     __ bind(done);
15795   %}
15796 
15797   ins_pipe(pipe_class_default);
15798 
15799 %}
15800 
15801 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15802 %{
15803   match(Set dst (CmpD3 src1 src2));
15804   effect(KILL cr);
15805 
15806   ins_cost(5 * INSN_COST);
15807   format %{ "fcmpd $src1, $src2\n\t"
15808             "csinvw($dst, zr, zr, eq\n\t"
15809             "csnegw($dst, $dst, $dst, lt)"
15810   %}
15811 
15812   ins_encode %{
15813     Label done;
15814     FloatRegister s1 = as_FloatRegister($src1$$reg);
15815     FloatRegister s2 = as_FloatRegister($src2$$reg);
15816     Register d = as_Register($dst$$reg);
15817     __ fcmpd(s1, s2);
15818     // installs 0 if EQ else -1
15819     __ csinvw(d, zr, zr, Assembler::EQ);
15820     // keeps -1 if less or unordered else installs 1
15821     __ csnegw(d, d, d, Assembler::LT);
15822     __ bind(done);
15823   %}
15824   ins_pipe(pipe_class_default);
15825 
15826 %}
15827 
15828 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15829 %{
15830   match(Set dst (CmpF3 src1 zero));
15831   effect(KILL cr);
15832 
15833   ins_cost(5 * INSN_COST);
15834   format %{ "fcmps $src1, 0.0\n\t"
15835             "csinvw($dst, zr, zr, eq\n\t"
15836             "csnegw($dst, $dst, $dst, lt)"
15837   %}
15838 
15839   ins_encode %{
15840     Label done;
15841     FloatRegister s1 = as_FloatRegister($src1$$reg);
15842     Register d = as_Register($dst$$reg);
15843     __ fcmps(s1, 0.0);
15844     // installs 0 if EQ else -1
15845     __ csinvw(d, zr, zr, Assembler::EQ);
15846     // keeps -1 if less or unordered else installs 1
15847     __ csnegw(d, d, d, Assembler::LT);
15848     __ bind(done);
15849   %}
15850 
15851   ins_pipe(pipe_class_default);
15852 
15853 %}
15854 
15855 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15856 %{
15857   match(Set dst (CmpD3 src1 zero));
15858   effect(KILL cr);
15859 
15860   ins_cost(5 * INSN_COST);
15861   format %{ "fcmpd $src1, 0.0\n\t"
15862             "csinvw($dst, zr, zr, eq\n\t"
15863             "csnegw($dst, $dst, $dst, lt)"
15864   %}
15865 
15866   ins_encode %{
15867     Label done;
15868     FloatRegister s1 = as_FloatRegister($src1$$reg);
15869     Register d = as_Register($dst$$reg);
15870     __ fcmpd(s1, 0.0);
15871     // installs 0 if EQ else -1
15872     __ csinvw(d, zr, zr, Assembler::EQ);
15873     // keeps -1 if less or unordered else installs 1
15874     __ csnegw(d, d, d, Assembler::LT);
15875     __ bind(done);
15876   %}
15877   ins_pipe(pipe_class_default);
15878 
15879 %}
15880 
15881 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15882 %{
15883   match(Set dst (CmpLTMask p q));
15884   effect(KILL cr);
15885 
15886   ins_cost(3 * INSN_COST);
15887 
15888   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15889             "csetw $dst, lt\n\t"
15890             "subw $dst, zr, $dst"
15891   %}
15892 
15893   ins_encode %{
15894     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15895     __ csetw(as_Register($dst$$reg), Assembler::LT);
15896     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15897   %}
15898 
15899   ins_pipe(ialu_reg_reg);
15900 %}
15901 
15902 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15903 %{
15904   match(Set dst (CmpLTMask src zero));
15905   effect(KILL cr);
15906 
15907   ins_cost(INSN_COST);
15908 
15909   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15910 
15911   ins_encode %{
15912     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15913   %}
15914 
15915   ins_pipe(ialu_reg_shift);
15916 %}
15917 
15918 // ============================================================================
15919 // Max and Min
15920 
15921 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15922 
15923 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15924 %{
15925   effect(DEF cr, USE src);
15926   ins_cost(INSN_COST);
15927   format %{ "cmpw $src, 0" %}
15928 
15929   ins_encode %{
15930     __ cmpw($src$$Register, 0);
15931   %}
15932   ins_pipe(icmp_reg_imm);
15933 %}
15934 
15935 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15936 %{
15937   match(Set dst (MinI src1 src2));
15938   ins_cost(INSN_COST * 3);
15939 
15940   expand %{
15941     rFlagsReg cr;
15942     compI_reg_reg(cr, src1, src2);
15943     cmovI_reg_reg_lt(dst, src1, src2, cr);
15944   %}
15945 %}
15946 
15947 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15948 %{
15949   match(Set dst (MaxI src1 src2));
15950   ins_cost(INSN_COST * 3);
15951 
15952   expand %{
15953     rFlagsReg cr;
15954     compI_reg_reg(cr, src1, src2);
15955     cmovI_reg_reg_gt(dst, src1, src2, cr);
15956   %}
15957 %}
15958 
15959 
15960 // ============================================================================
15961 // Branch Instructions
15962 
15963 // Direct Branch.
15964 instruct branch(label lbl)
15965 %{
15966   match(Goto);
15967 
15968   effect(USE lbl);
15969 
15970   ins_cost(BRANCH_COST);
15971   format %{ "b  $lbl" %}
15972 
15973   ins_encode(aarch64_enc_b(lbl));
15974 
15975   ins_pipe(pipe_branch);
15976 %}
15977 
15978 // Conditional Near Branch
15979 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15980 %{
15981   // Same match rule as `branchConFar'.
15982   match(If cmp cr);
15983 
15984   effect(USE lbl);
15985 
15986   ins_cost(BRANCH_COST);
15987   // If set to 1 this indicates that the current instruction is a
15988   // short variant of a long branch. This avoids using this
15989   // instruction in first-pass matching. It will then only be used in
15990   // the `Shorten_branches' pass.
15991   // ins_short_branch(1);
15992   format %{ "b$cmp  $lbl" %}
15993 
15994   ins_encode(aarch64_enc_br_con(cmp, lbl));
15995 
15996   ins_pipe(pipe_branch_cond);
15997 %}
15998 
15999 // Conditional Near Branch Unsigned
16000 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16001 %{
16002   // Same match rule as `branchConFar'.
16003   match(If cmp cr);
16004 
16005   effect(USE lbl);
16006 
16007   ins_cost(BRANCH_COST);
16008   // If set to 1 this indicates that the current instruction is a
16009   // short variant of a long branch. This avoids using this
16010   // instruction in first-pass matching. It will then only be used in
16011   // the `Shorten_branches' pass.
16012   // ins_short_branch(1);
16013   format %{ "b$cmp  $lbl\t# unsigned" %}
16014 
16015   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16016 
16017   ins_pipe(pipe_branch_cond);
16018 %}
16019 
16020 // Make use of CBZ and CBNZ.  These instructions, as well as being
16021 // shorter than (cmp; branch), have the additional benefit of not
16022 // killing the flags.
16023 
16024 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16025   match(If cmp (CmpI op1 op2));
16026   effect(USE labl);
16027 
16028   ins_cost(BRANCH_COST);
16029   format %{ "cbw$cmp   $op1, $labl" %}
16030   ins_encode %{
16031     Label* L = $labl$$label;
16032     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16033     if (cond == Assembler::EQ)
16034       __ cbzw($op1$$Register, *L);
16035     else
16036       __ cbnzw($op1$$Register, *L);
16037   %}
16038   ins_pipe(pipe_cmp_branch);
16039 %}
16040 
16041 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16042   match(If cmp (CmpL op1 op2));
16043   effect(USE labl);
16044 
16045   ins_cost(BRANCH_COST);
16046   format %{ "cb$cmp   $op1, $labl" %}
16047   ins_encode %{
16048     Label* L = $labl$$label;
16049     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16050     if (cond == Assembler::EQ)
16051       __ cbz($op1$$Register, *L);
16052     else
16053       __ cbnz($op1$$Register, *L);
16054   %}
16055   ins_pipe(pipe_cmp_branch);
16056 %}
16057 
16058 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16059   match(If cmp (CmpP op1 op2));
16060   effect(USE labl);
16061 
16062   ins_cost(BRANCH_COST);
16063   format %{ "cb$cmp   $op1, $labl" %}
16064   ins_encode %{
16065     Label* L = $labl$$label;
16066     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16067     if (cond == Assembler::EQ)
16068       __ cbz($op1$$Register, *L);
16069     else
16070       __ cbnz($op1$$Register, *L);
16071   %}
16072   ins_pipe(pipe_cmp_branch);
16073 %}
16074 
16075 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16076   match(If cmp (CmpN op1 op2));
16077   effect(USE labl);
16078 
16079   ins_cost(BRANCH_COST);
16080   format %{ "cbw$cmp   $op1, $labl" %}
16081   ins_encode %{
16082     Label* L = $labl$$label;
16083     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16084     if (cond == Assembler::EQ)
16085       __ cbzw($op1$$Register, *L);
16086     else
16087       __ cbnzw($op1$$Register, *L);
16088   %}
16089   ins_pipe(pipe_cmp_branch);
16090 %}
16091 
16092 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16093   match(If cmp (CmpP (DecodeN oop) zero));
16094   effect(USE labl);
16095 
16096   ins_cost(BRANCH_COST);
16097   format %{ "cb$cmp   $oop, $labl" %}
16098   ins_encode %{
16099     Label* L = $labl$$label;
16100     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16101     if (cond == Assembler::EQ)
16102       __ cbzw($oop$$Register, *L);
16103     else
16104       __ cbnzw($oop$$Register, *L);
16105   %}
16106   ins_pipe(pipe_cmp_branch);
16107 %}
16108 
16109 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16110   match(If cmp (CmpU op1 op2));
16111   effect(USE labl);
16112 
16113   ins_cost(BRANCH_COST);
16114   format %{ "cbw$cmp   $op1, $labl" %}
16115   ins_encode %{
16116     Label* L = $labl$$label;
16117     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16118     if (cond == Assembler::EQ || cond == Assembler::LS) {
16119       __ cbzw($op1$$Register, *L);
16120     } else {
16121       assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16122       __ cbnzw($op1$$Register, *L);
16123     }
16124   %}
16125   ins_pipe(pipe_cmp_branch);
16126 %}
16127 
16128 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
16129   match(If cmp (CmpUL op1 op2));
16130   effect(USE labl);
16131 
16132   ins_cost(BRANCH_COST);
16133   format %{ "cb$cmp   $op1, $labl" %}
16134   ins_encode %{
16135     Label* L = $labl$$label;
16136     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16137     if (cond == Assembler::EQ || cond == Assembler::LS) {
16138       __ cbz($op1$$Register, *L);
16139     } else {
16140       assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16141       __ cbnz($op1$$Register, *L);
16142     }
16143   %}
16144   ins_pipe(pipe_cmp_branch);
16145 %}
16146 
16147 // Test bit and Branch
16148 
16149 // Patterns for short (< 32KiB) variants
16150 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16151   match(If cmp (CmpL op1 op2));
16152   effect(USE labl);
16153 
16154   ins_cost(BRANCH_COST);
16155   format %{ "cb$cmp   $op1, $labl # long" %}
16156   ins_encode %{
16157     Label* L = $labl$$label;
16158     Assembler::Condition cond =
16159       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16160     __ tbr(cond, $op1$$Register, 63, *L);
16161   %}
16162   ins_pipe(pipe_cmp_branch);
16163   ins_short_branch(1);
16164 %}
16165 
16166 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16167   match(If cmp (CmpI op1 op2));
16168   effect(USE labl);
16169 
16170   ins_cost(BRANCH_COST);
16171   format %{ "cb$cmp   $op1, $labl # int" %}
16172   ins_encode %{
16173     Label* L = $labl$$label;
16174     Assembler::Condition cond =
16175       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16176     __ tbr(cond, $op1$$Register, 31, *L);
16177   %}
16178   ins_pipe(pipe_cmp_branch);
16179   ins_short_branch(1);
16180 %}
16181 
16182 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16183   match(If cmp (CmpL (AndL op1 op2) op3));
16184   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16185   effect(USE labl);
16186 
16187   ins_cost(BRANCH_COST);
16188   format %{ "tb$cmp   $op1, $op2, $labl" %}
16189   ins_encode %{
16190     Label* L = $labl$$label;
16191     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16192     int bit = exact_log2_long($op2$$constant);
16193     __ tbr(cond, $op1$$Register, bit, *L);
16194   %}
16195   ins_pipe(pipe_cmp_branch);
16196   ins_short_branch(1);
16197 %}
16198 
16199 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16200   match(If cmp (CmpI (AndI op1 op2) op3));
16201   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16202   effect(USE labl);
16203 
16204   ins_cost(BRANCH_COST);
16205   format %{ "tb$cmp   $op1, $op2, $labl" %}
16206   ins_encode %{
16207     Label* L = $labl$$label;
16208     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16209     int bit = exact_log2((juint)$op2$$constant);
16210     __ tbr(cond, $op1$$Register, bit, *L);
16211   %}
16212   ins_pipe(pipe_cmp_branch);
16213   ins_short_branch(1);
16214 %}
16215 
16216 // And far variants
16217 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16218   match(If cmp (CmpL op1 op2));
16219   effect(USE labl);
16220 
16221   ins_cost(BRANCH_COST);
16222   format %{ "cb$cmp   $op1, $labl # long" %}
16223   ins_encode %{
16224     Label* L = $labl$$label;
16225     Assembler::Condition cond =
16226       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16227     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16228   %}
16229   ins_pipe(pipe_cmp_branch);
16230 %}
16231 
16232 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16233   match(If cmp (CmpI op1 op2));
16234   effect(USE labl);
16235 
16236   ins_cost(BRANCH_COST);
16237   format %{ "cb$cmp   $op1, $labl # int" %}
16238   ins_encode %{
16239     Label* L = $labl$$label;
16240     Assembler::Condition cond =
16241       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16242     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16243   %}
16244   ins_pipe(pipe_cmp_branch);
16245 %}
16246 
16247 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16248   match(If cmp (CmpL (AndL op1 op2) op3));
16249   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16250   effect(USE labl);
16251 
16252   ins_cost(BRANCH_COST);
16253   format %{ "tb$cmp   $op1, $op2, $labl" %}
16254   ins_encode %{
16255     Label* L = $labl$$label;
16256     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16257     int bit = exact_log2_long($op2$$constant);
16258     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16259   %}
16260   ins_pipe(pipe_cmp_branch);
16261 %}
16262 
16263 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16264   match(If cmp (CmpI (AndI op1 op2) op3));
16265   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16266   effect(USE labl);
16267 
16268   ins_cost(BRANCH_COST);
16269   format %{ "tb$cmp   $op1, $op2, $labl" %}
16270   ins_encode %{
16271     Label* L = $labl$$label;
16272     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16273     int bit = exact_log2((juint)$op2$$constant);
16274     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16275   %}
16276   ins_pipe(pipe_cmp_branch);
16277 %}
16278 
16279 // Test bits
16280 
16281 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16282   match(Set cr (CmpL (AndL op1 op2) op3));
16283   predicate(Assembler::operand_valid_for_logical_immediate
16284             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16285 
16286   ins_cost(INSN_COST);
16287   format %{ "tst $op1, $op2 # long" %}
16288   ins_encode %{
16289     __ tst($op1$$Register, $op2$$constant);
16290   %}
16291   ins_pipe(ialu_reg_reg);
16292 %}
16293 
16294 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16295   match(Set cr (CmpI (AndI op1 op2) op3));
16296   predicate(Assembler::operand_valid_for_logical_immediate
16297             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16298 
16299   ins_cost(INSN_COST);
16300   format %{ "tst $op1, $op2 # int" %}
16301   ins_encode %{
16302     __ tstw($op1$$Register, $op2$$constant);
16303   %}
16304   ins_pipe(ialu_reg_reg);
16305 %}
16306 
16307 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16308   match(Set cr (CmpL (AndL op1 op2) op3));
16309 
16310   ins_cost(INSN_COST);
16311   format %{ "tst $op1, $op2 # long" %}
16312   ins_encode %{
16313     __ tst($op1$$Register, $op2$$Register);
16314   %}
16315   ins_pipe(ialu_reg_reg);
16316 %}
16317 
16318 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16319   match(Set cr (CmpI (AndI op1 op2) op3));
16320 
16321   ins_cost(INSN_COST);
16322   format %{ "tstw $op1, $op2 # int" %}
16323   ins_encode %{
16324     __ tstw($op1$$Register, $op2$$Register);
16325   %}
16326   ins_pipe(ialu_reg_reg);
16327 %}
16328 
16329 
16330 // Conditional Far Branch
16331 // Conditional Far Branch Unsigned
16332 // TODO: fixme
16333 
16334 // counted loop end branch near
16335 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16336 %{
16337   match(CountedLoopEnd cmp cr);
16338 
16339   effect(USE lbl);
16340 
16341   ins_cost(BRANCH_COST);
16342   // short variant.
16343   // ins_short_branch(1);
16344   format %{ "b$cmp $lbl \t// counted loop end" %}
16345 
16346   ins_encode(aarch64_enc_br_con(cmp, lbl));
16347 
16348   ins_pipe(pipe_branch);
16349 %}
16350 
16351 // counted loop end branch far
16352 // TODO: fixme
16353 
16354 // ============================================================================
16355 // inlined locking and unlocking
16356 
16357 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16358 %{
16359   match(Set cr (FastLock object box));
16360   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16361 
16362   ins_cost(5 * INSN_COST);
16363   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16364 
16365   ins_encode %{
16366     __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16367   %}
16368 
16369   ins_pipe(pipe_serial);
16370 %}
16371 
16372 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16373 %{
16374   match(Set cr (FastUnlock object box));
16375   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16376 
16377   ins_cost(5 * INSN_COST);
16378   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16379 
16380   ins_encode %{
16381     __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16382   %}
16383 
16384   ins_pipe(pipe_serial);
16385 %}
16386 
16387 // ============================================================================
16388 // Safepoint Instructions
16389 
16390 // TODO
16391 // provide a near and far version of this code
16392 
16393 instruct safePoint(rFlagsReg cr, iRegP poll)
16394 %{
16395   match(SafePoint poll);
16396   effect(KILL cr);
16397 
16398   format %{
16399     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16400   %}
16401   ins_encode %{
16402     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16403   %}
16404   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16405 %}
16406 
16407 
16408 // ============================================================================
16409 // Procedure Call/Return Instructions
16410 
16411 // Call Java Static Instruction
16412 
16413 instruct CallStaticJavaDirect(method meth)
16414 %{
16415   match(CallStaticJava);
16416 
16417   effect(USE meth);
16418 
16419   ins_cost(CALL_COST);
16420 
16421   format %{ "call,static $meth \t// ==> " %}
16422 
16423   ins_encode(aarch64_enc_java_static_call(meth),
16424              aarch64_enc_call_epilog);
16425 
16426   ins_pipe(pipe_class_call);
16427 %}
16428 
16429 // TO HERE
16430 
16431 // Call Java Dynamic Instruction
16432 instruct CallDynamicJavaDirect(method meth)
16433 %{
16434   match(CallDynamicJava);
16435 
16436   effect(USE meth);
16437 
16438   ins_cost(CALL_COST);
16439 
16440   format %{ "CALL,dynamic $meth \t// ==> " %}
16441 
16442   ins_encode(aarch64_enc_java_dynamic_call(meth),
16443              aarch64_enc_call_epilog);
16444 
16445   ins_pipe(pipe_class_call);
16446 %}
16447 
16448 // Call Runtime Instruction
16449 
16450 instruct CallRuntimeDirect(method meth)
16451 %{
16452   match(CallRuntime);
16453 
16454   effect(USE meth);
16455 
16456   ins_cost(CALL_COST);
16457 
16458   format %{ "CALL, runtime $meth" %}
16459 
16460   ins_encode( aarch64_enc_java_to_runtime(meth) );
16461 
16462   ins_pipe(pipe_class_call);
16463 %}
16464 
16465 // Call Runtime Instruction
16466 
16467 instruct CallLeafDirect(method meth)
16468 %{
16469   match(CallLeaf);
16470 
16471   effect(USE meth);
16472 
16473   ins_cost(CALL_COST);
16474 
16475   format %{ "CALL, runtime leaf $meth" %}
16476 
16477   ins_encode( aarch64_enc_java_to_runtime(meth) );
16478 
16479   ins_pipe(pipe_class_call);
16480 %}
16481 
16482 // Call Runtime Instruction without safepoint and with vector arguments
16483 instruct CallLeafDirectVector(method meth)
16484 %{
16485   match(CallLeafVector);
16486 
16487   effect(USE meth);
16488 
16489   ins_cost(CALL_COST);
16490 
16491   format %{ "CALL, runtime leaf vector $meth" %}
16492 
16493   ins_encode(aarch64_enc_java_to_runtime(meth));
16494 
16495   ins_pipe(pipe_class_call);
16496 %}
16497 
16498 // Call Runtime Instruction
16499 
16500 // entry point is null, target holds the address to call
16501 instruct CallLeafNoFPIndirect(iRegP target)
16502 %{
16503   predicate(n->as_Call()->entry_point() == nullptr);
16504 
16505   match(CallLeafNoFP target);
16506 
16507   ins_cost(CALL_COST);
16508 
16509   format %{ "CALL, runtime leaf nofp indirect $target" %}
16510 
16511   ins_encode %{
16512     __ blr($target$$Register);
16513   %}
16514 
16515   ins_pipe(pipe_class_call);
16516 %}
16517 
16518 instruct CallLeafNoFPDirect(method meth)
16519 %{
16520   predicate(n->as_Call()->entry_point() != nullptr);
16521 
16522   match(CallLeafNoFP);
16523 
16524   effect(USE meth);
16525 
16526   ins_cost(CALL_COST);
16527 
16528   format %{ "CALL, runtime leaf nofp $meth" %}
16529 
16530   ins_encode( aarch64_enc_java_to_runtime(meth) );
16531 
16532   ins_pipe(pipe_class_call);
16533 %}
16534 
16535 // Tail Call; Jump from runtime stub to Java code.
16536 // Also known as an 'interprocedural jump'.
16537 // Target of jump will eventually return to caller.
16538 // TailJump below removes the return address.
16539 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16540 // emitted just above the TailCall which has reset rfp to the caller state.
16541 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16542 %{
16543   match(TailCall jump_target method_ptr);
16544 
16545   ins_cost(CALL_COST);
16546 
16547   format %{ "br $jump_target\t# $method_ptr holds method" %}
16548 
16549   ins_encode(aarch64_enc_tail_call(jump_target));
16550 
16551   ins_pipe(pipe_class_call);
16552 %}
16553 
16554 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16555 %{
16556   match(TailJump jump_target ex_oop);
16557 
16558   ins_cost(CALL_COST);
16559 
16560   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16561 
16562   ins_encode(aarch64_enc_tail_jmp(jump_target));
16563 
16564   ins_pipe(pipe_class_call);
16565 %}
16566 
16567 // Forward exception.
16568 instruct ForwardExceptionjmp()
16569 %{
16570   match(ForwardException);
16571   ins_cost(CALL_COST);
16572 
16573   format %{ "b forward_exception_stub" %}
16574   ins_encode %{
16575     __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16576   %}
16577   ins_pipe(pipe_class_call);
16578 %}
16579 
16580 // Create exception oop: created by stack-crawling runtime code.
16581 // Created exception is now available to this handler, and is setup
16582 // just prior to jumping to this handler. No code emitted.
16583 // TODO check
16584 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16585 instruct CreateException(iRegP_R0 ex_oop)
16586 %{
16587   match(Set ex_oop (CreateEx));
16588 
16589   format %{ " -- \t// exception oop; no code emitted" %}
16590 
16591   size(0);
16592 
16593   ins_encode( /*empty*/ );
16594 
16595   ins_pipe(pipe_class_empty);
16596 %}
16597 
16598 // Rethrow exception: The exception oop will come in the first
16599 // argument position. Then JUMP (not call) to the rethrow stub code.
16600 instruct RethrowException() %{
16601   match(Rethrow);
16602   ins_cost(CALL_COST);
16603 
16604   format %{ "b rethrow_stub" %}
16605 
16606   ins_encode( aarch64_enc_rethrow() );
16607 
16608   ins_pipe(pipe_class_call);
16609 %}
16610 
16611 
16612 // Return Instruction
16613 // epilog node loads ret address into lr as part of frame pop
16614 instruct Ret()
16615 %{
16616   match(Return);
16617 
16618   format %{ "ret\t// return register" %}
16619 
16620   ins_encode( aarch64_enc_ret() );
16621 
16622   ins_pipe(pipe_branch);
16623 %}
16624 
16625 // Die now.
16626 instruct ShouldNotReachHere() %{
16627   match(Halt);
16628 
16629   ins_cost(CALL_COST);
16630   format %{ "ShouldNotReachHere" %}
16631 
16632   ins_encode %{
16633     if (is_reachable()) {
16634       const char* str = __ code_string(_halt_reason);
16635       __ stop(str);
16636     }
16637   %}
16638 
16639   ins_pipe(pipe_class_default);
16640 %}
16641 
16642 // ============================================================================
16643 // Partial Subtype Check
16644 //
16645 // superklass array for an instance of the superklass.  Set a hidden
16646 // internal cache on a hit (cache is checked with exposed code in
16647 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16648 // encoding ALSO sets flags.
16649 
16650 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16651 %{
16652   match(Set result (PartialSubtypeCheck sub super));
16653   predicate(!UseSecondarySupersTable);
16654   effect(KILL cr, KILL temp);
16655 
16656   ins_cost(20 * INSN_COST);  // slightly larger than the next version
16657   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16658 
16659   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16660 
16661   opcode(0x1); // Force zero of result reg on hit
16662 
16663   ins_pipe(pipe_class_memory);
16664 %}
16665 
16666 // Two versions of partialSubtypeCheck, both used when we need to
16667 // search for a super class in the secondary supers array. The first
16668 // is used when we don't know _a priori_ the class being searched
16669 // for. The second, far more common, is used when we do know: this is
16670 // used for instanceof, checkcast, and any case where C2 can determine
16671 // it by constant propagation.
16672 
16673 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16674                                      iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16675                                      rFlagsReg cr)
16676 %{
16677   match(Set result (PartialSubtypeCheck sub super));
16678   predicate(UseSecondarySupersTable);
16679   effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16680 
16681   ins_cost(10 * INSN_COST);  // slightly larger than the next version
16682   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16683 
16684   ins_encode %{
16685     __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16686                                          $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16687                                          $vtemp$$FloatRegister,
16688                                          $result$$Register, /*L_success*/nullptr);
16689   %}
16690 
16691   ins_pipe(pipe_class_memory);
16692 %}
16693 
16694 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16695                                        iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16696                                        rFlagsReg cr)
16697 %{
16698   match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16699   predicate(UseSecondarySupersTable);
16700   effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16701 
16702   ins_cost(5 * INSN_COST);  // smaller than the next version
16703   format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16704 
16705   ins_encode %{
16706     bool success = false;
16707     u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16708     if (InlineSecondarySupersTest) {
16709       success =
16710         __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16711                                                $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16712                                                $vtemp$$FloatRegister,
16713                                                $result$$Register,
16714                                                super_klass_slot);
16715     } else {
16716       address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16717       success = (call != nullptr);
16718     }
16719     if (!success) {
16720       ciEnv::current()->record_failure("CodeCache is full");
16721       return;
16722     }
16723   %}
16724 
16725   ins_pipe(pipe_class_memory);
16726 %}
16727 
16728 // Intrisics for String.compareTo()
16729 
16730 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16731                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16732 %{
16733   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16734   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16735   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16736 
16737   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16738   ins_encode %{
16739     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16740     __ string_compare($str1$$Register, $str2$$Register,
16741                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16742                       $tmp1$$Register, $tmp2$$Register,
16743                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16744   %}
16745   ins_pipe(pipe_class_memory);
16746 %}
16747 
16748 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16749                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16750 %{
16751   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16752   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16753   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16754 
16755   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16756   ins_encode %{
16757     __ string_compare($str1$$Register, $str2$$Register,
16758                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16759                       $tmp1$$Register, $tmp2$$Register,
16760                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16761   %}
16762   ins_pipe(pipe_class_memory);
16763 %}
16764 
16765 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16766                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16767                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16768 %{
16769   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16770   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16771   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16772          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16773 
16774   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16775   ins_encode %{
16776     __ string_compare($str1$$Register, $str2$$Register,
16777                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16778                       $tmp1$$Register, $tmp2$$Register,
16779                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16780                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16781   %}
16782   ins_pipe(pipe_class_memory);
16783 %}
16784 
16785 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16786                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16787                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16788 %{
16789   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16790   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16791   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16792          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16793 
16794   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16795   ins_encode %{
16796     __ string_compare($str1$$Register, $str2$$Register,
16797                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16798                       $tmp1$$Register, $tmp2$$Register,
16799                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16800                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16801   %}
16802   ins_pipe(pipe_class_memory);
16803 %}
16804 
16805 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16806 // these string_compare variants as NEON register type for convenience so that the prototype of
16807 // string_compare can be shared with all variants.
16808 
16809 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16810                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16811                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16812                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16813 %{
16814   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16815   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16816   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16817          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16818 
16819   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16820   ins_encode %{
16821     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16822     __ string_compare($str1$$Register, $str2$$Register,
16823                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16824                       $tmp1$$Register, $tmp2$$Register,
16825                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16826                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16827                       StrIntrinsicNode::LL);
16828   %}
16829   ins_pipe(pipe_class_memory);
16830 %}
16831 
16832 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16833                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16834                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16835                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16836 %{
16837   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16838   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16839   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16840          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16841 
16842   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16843   ins_encode %{
16844     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16845     __ string_compare($str1$$Register, $str2$$Register,
16846                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16847                       $tmp1$$Register, $tmp2$$Register,
16848                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16849                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16850                       StrIntrinsicNode::LU);
16851   %}
16852   ins_pipe(pipe_class_memory);
16853 %}
16854 
16855 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16856                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16857                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16858                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16859 %{
16860   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16861   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16862   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16863          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16864 
16865   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16866   ins_encode %{
16867     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16868     __ string_compare($str1$$Register, $str2$$Register,
16869                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16870                       $tmp1$$Register, $tmp2$$Register,
16871                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16872                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16873                       StrIntrinsicNode::UL);
16874   %}
16875   ins_pipe(pipe_class_memory);
16876 %}
16877 
16878 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16879                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16880                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16881                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16882 %{
16883   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16884   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16885   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16886          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16887 
16888   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16889   ins_encode %{
16890     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16891     __ string_compare($str1$$Register, $str2$$Register,
16892                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16893                       $tmp1$$Register, $tmp2$$Register,
16894                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16895                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16896                       StrIntrinsicNode::UU);
16897   %}
16898   ins_pipe(pipe_class_memory);
16899 %}
16900 
16901 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16902                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16903                           iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16904                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16905 %{
16906   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16907   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16908   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16909          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16910          TEMP vtmp0, TEMP vtmp1, KILL cr);
16911   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16912             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16913 
16914   ins_encode %{
16915     __ string_indexof($str1$$Register, $str2$$Register,
16916                       $cnt1$$Register, $cnt2$$Register,
16917                       $tmp1$$Register, $tmp2$$Register,
16918                       $tmp3$$Register, $tmp4$$Register,
16919                       $tmp5$$Register, $tmp6$$Register,
16920                       -1, $result$$Register, StrIntrinsicNode::UU);
16921   %}
16922   ins_pipe(pipe_class_memory);
16923 %}
16924 
16925 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16926                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16927                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16928                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16929 %{
16930   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16931   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16932   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16933          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16934          TEMP vtmp0, TEMP vtmp1, KILL cr);
16935   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16936             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16937 
16938   ins_encode %{
16939     __ string_indexof($str1$$Register, $str2$$Register,
16940                       $cnt1$$Register, $cnt2$$Register,
16941                       $tmp1$$Register, $tmp2$$Register,
16942                       $tmp3$$Register, $tmp4$$Register,
16943                       $tmp5$$Register, $tmp6$$Register,
16944                       -1, $result$$Register, StrIntrinsicNode::LL);
16945   %}
16946   ins_pipe(pipe_class_memory);
16947 %}
16948 
16949 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16950                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16951                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16952                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16953 %{
16954   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16955   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16956   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16957          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16958          TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16959   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16960             "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16961 
16962   ins_encode %{
16963     __ string_indexof($str1$$Register, $str2$$Register,
16964                       $cnt1$$Register, $cnt2$$Register,
16965                       $tmp1$$Register, $tmp2$$Register,
16966                       $tmp3$$Register, $tmp4$$Register,
16967                       $tmp5$$Register, $tmp6$$Register,
16968                       -1, $result$$Register, StrIntrinsicNode::UL);
16969   %}
16970   ins_pipe(pipe_class_memory);
16971 %}
16972 
16973 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16974                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16975                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16976 %{
16977   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16978   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16979   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16980          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16981   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16982             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16983 
16984   ins_encode %{
16985     int icnt2 = (int)$int_cnt2$$constant;
16986     __ string_indexof($str1$$Register, $str2$$Register,
16987                       $cnt1$$Register, zr,
16988                       $tmp1$$Register, $tmp2$$Register,
16989                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16990                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16991   %}
16992   ins_pipe(pipe_class_memory);
16993 %}
16994 
16995 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16996                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16997                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16998 %{
16999   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17000   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17001   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17002          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17003   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
17004             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17005 
17006   ins_encode %{
17007     int icnt2 = (int)$int_cnt2$$constant;
17008     __ string_indexof($str1$$Register, $str2$$Register,
17009                       $cnt1$$Register, zr,
17010                       $tmp1$$Register, $tmp2$$Register,
17011                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17012                       icnt2, $result$$Register, StrIntrinsicNode::LL);
17013   %}
17014   ins_pipe(pipe_class_memory);
17015 %}
17016 
17017 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17018                               immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17019                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17020 %{
17021   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17022   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17023   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17024          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17025   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
17026             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17027 
17028   ins_encode %{
17029     int icnt2 = (int)$int_cnt2$$constant;
17030     __ string_indexof($str1$$Register, $str2$$Register,
17031                       $cnt1$$Register, zr,
17032                       $tmp1$$Register, $tmp2$$Register,
17033                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17034                       icnt2, $result$$Register, StrIntrinsicNode::UL);
17035   %}
17036   ins_pipe(pipe_class_memory);
17037 %}
17038 
17039 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17040                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17041                              iRegINoSp tmp3, rFlagsReg cr)
17042 %{
17043   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17044   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17045   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17046          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17047 
17048   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17049 
17050   ins_encode %{
17051     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17052                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
17053                            $tmp3$$Register);
17054   %}
17055   ins_pipe(pipe_class_memory);
17056 %}
17057 
17058 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17059                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17060                               iRegINoSp tmp3, rFlagsReg cr)
17061 %{
17062   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17063   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17064   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17065          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17066 
17067   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17068 
17069   ins_encode %{
17070     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17071                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
17072                             $tmp3$$Register);
17073   %}
17074   ins_pipe(pipe_class_memory);
17075 %}
17076 
17077 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17078                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17079                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17080   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17081   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17082   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17083   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17084   ins_encode %{
17085     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17086                                $result$$Register, $ztmp1$$FloatRegister,
17087                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17088                                $ptmp$$PRegister, true /* isL */);
17089   %}
17090   ins_pipe(pipe_class_memory);
17091 %}
17092 
17093 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17094                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17095                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17096   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17097   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17098   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17099   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17100   ins_encode %{
17101     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17102                                $result$$Register, $ztmp1$$FloatRegister,
17103                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17104                                $ptmp$$PRegister, false /* isL */);
17105   %}
17106   ins_pipe(pipe_class_memory);
17107 %}
17108 
17109 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17110                         iRegI_R0 result, rFlagsReg cr)
17111 %{
17112   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17113   match(Set result (StrEquals (Binary str1 str2) cnt));
17114   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17115 
17116   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17117   ins_encode %{
17118     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17119     __ string_equals($str1$$Register, $str2$$Register,
17120                      $result$$Register, $cnt$$Register);
17121   %}
17122   ins_pipe(pipe_class_memory);
17123 %}
17124 
17125 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17126                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17127                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17128                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17129                        iRegP_R10 tmp, rFlagsReg cr)
17130 %{
17131   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17132   match(Set result (AryEq ary1 ary2));
17133   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17134          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17135          TEMP vtmp6, TEMP vtmp7, KILL cr);
17136 
17137   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17138   ins_encode %{
17139     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17140                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17141                                    $result$$Register, $tmp$$Register, 1);
17142     if (tpc == nullptr) {
17143       ciEnv::current()->record_failure("CodeCache is full");
17144       return;
17145     }
17146   %}
17147   ins_pipe(pipe_class_memory);
17148 %}
17149 
17150 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17151                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17152                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17153                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17154                        iRegP_R10 tmp, rFlagsReg cr)
17155 %{
17156   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17157   match(Set result (AryEq ary1 ary2));
17158   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17159          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17160          TEMP vtmp6, TEMP vtmp7, KILL cr);
17161 
17162   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17163   ins_encode %{
17164     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17165                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17166                                    $result$$Register, $tmp$$Register, 2);
17167     if (tpc == nullptr) {
17168       ciEnv::current()->record_failure("CodeCache is full");
17169       return;
17170     }
17171   %}
17172   ins_pipe(pipe_class_memory);
17173 %}
17174 
17175 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
17176                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17177                          vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17178                          vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
17179 %{
17180   match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
17181   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
17182          TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
17183 
17184   format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result   // KILL all" %}
17185   ins_encode %{
17186     address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
17187                                      $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
17188                                      $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
17189                                      $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
17190                                      $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
17191                                      $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
17192                                      (BasicType)$basic_type$$constant);
17193     if (tpc == nullptr) {
17194       ciEnv::current()->record_failure("CodeCache is full");
17195       return;
17196     }
17197   %}
17198   ins_pipe(pipe_class_memory);
17199 %}
17200 
17201 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17202 %{
17203   match(Set result (CountPositives ary1 len));
17204   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17205   format %{ "count positives byte[] $ary1,$len -> $result" %}
17206   ins_encode %{
17207     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17208     if (tpc == nullptr) {
17209       ciEnv::current()->record_failure("CodeCache is full");
17210       return;
17211     }
17212   %}
17213   ins_pipe( pipe_slow );
17214 %}
17215 
17216 // fast char[] to byte[] compression
17217 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17218                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17219                          vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17220                          iRegI_R0 result, rFlagsReg cr)
17221 %{
17222   match(Set result (StrCompressedCopy src (Binary dst len)));
17223   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17224          USE_KILL src, USE_KILL dst, USE len, KILL cr);
17225 
17226   format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17227   ins_encode %{
17228     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17229                            $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17230                            $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17231                            $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17232   %}
17233   ins_pipe(pipe_slow);
17234 %}
17235 
17236 // fast byte[] to char[] inflation
17237 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17238                         vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17239                         vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17240 %{
17241   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17242   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17243          TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17244          USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17245 
17246   format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17247   ins_encode %{
17248     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17249                                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17250                                         $vtmp2$$FloatRegister, $tmp$$Register);
17251     if (tpc == nullptr) {
17252       ciEnv::current()->record_failure("CodeCache is full");
17253       return;
17254     }
17255   %}
17256   ins_pipe(pipe_class_memory);
17257 %}
17258 
17259 // encode char[] to byte[] in ISO_8859_1
17260 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17261                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17262                           vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17263                           iRegI_R0 result, rFlagsReg cr)
17264 %{
17265   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17266   match(Set result (EncodeISOArray src (Binary dst len)));
17267   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17268          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17269 
17270   format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17271   ins_encode %{
17272     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17273                         $result$$Register, false,
17274                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17275                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17276                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17277   %}
17278   ins_pipe(pipe_class_memory);
17279 %}
17280 
17281 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17282                             vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17283                             vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17284                             iRegI_R0 result, rFlagsReg cr)
17285 %{
17286   predicate(((EncodeISOArrayNode*)n)->is_ascii());
17287   match(Set result (EncodeISOArray src (Binary dst len)));
17288   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17289          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17290 
17291   format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17292   ins_encode %{
17293     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17294                         $result$$Register, true,
17295                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17296                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17297                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17298   %}
17299   ins_pipe(pipe_class_memory);
17300 %}
17301 
17302 //----------------------------- CompressBits/ExpandBits ------------------------
17303 
17304 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17305                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17306   match(Set dst (CompressBits src mask));
17307   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17308   format %{ "mov    $tsrc, $src\n\t"
17309             "mov    $tmask, $mask\n\t"
17310             "bext   $tdst, $tsrc, $tmask\n\t"
17311             "mov    $dst, $tdst"
17312           %}
17313   ins_encode %{
17314     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17315     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17316     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17317     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17318   %}
17319   ins_pipe(pipe_slow);
17320 %}
17321 
17322 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17323                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17324   match(Set dst (CompressBits (LoadI mem) mask));
17325   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17326   format %{ "ldrs   $tsrc, $mem\n\t"
17327             "ldrs   $tmask, $mask\n\t"
17328             "bext   $tdst, $tsrc, $tmask\n\t"
17329             "mov    $dst, $tdst"
17330           %}
17331   ins_encode %{
17332     loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17333               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17334     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17335     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17336     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17337   %}
17338   ins_pipe(pipe_slow);
17339 %}
17340 
17341 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17342                            vRegD tdst, vRegD tsrc, vRegD tmask) %{
17343   match(Set dst (CompressBits src mask));
17344   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17345   format %{ "mov    $tsrc, $src\n\t"
17346             "mov    $tmask, $mask\n\t"
17347             "bext   $tdst, $tsrc, $tmask\n\t"
17348             "mov    $dst, $tdst"
17349           %}
17350   ins_encode %{
17351     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17352     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17353     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17354     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17355   %}
17356   ins_pipe(pipe_slow);
17357 %}
17358 
17359 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17360                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17361   match(Set dst (CompressBits (LoadL mem) mask));
17362   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17363   format %{ "ldrd   $tsrc, $mem\n\t"
17364             "ldrd   $tmask, $mask\n\t"
17365             "bext   $tdst, $tsrc, $tmask\n\t"
17366             "mov    $dst, $tdst"
17367           %}
17368   ins_encode %{
17369     loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17370               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17371     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17372     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17373     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17374   %}
17375   ins_pipe(pipe_slow);
17376 %}
17377 
17378 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17379                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17380   match(Set dst (ExpandBits src mask));
17381   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17382   format %{ "mov    $tsrc, $src\n\t"
17383             "mov    $tmask, $mask\n\t"
17384             "bdep   $tdst, $tsrc, $tmask\n\t"
17385             "mov    $dst, $tdst"
17386           %}
17387   ins_encode %{
17388     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17389     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17390     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17391     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17392   %}
17393   ins_pipe(pipe_slow);
17394 %}
17395 
17396 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17397                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17398   match(Set dst (ExpandBits (LoadI mem) mask));
17399   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17400   format %{ "ldrs   $tsrc, $mem\n\t"
17401             "ldrs   $tmask, $mask\n\t"
17402             "bdep   $tdst, $tsrc, $tmask\n\t"
17403             "mov    $dst, $tdst"
17404           %}
17405   ins_encode %{
17406     loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17407               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17408     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17409     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17410     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17411   %}
17412   ins_pipe(pipe_slow);
17413 %}
17414 
17415 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17416                          vRegD tdst, vRegD tsrc, vRegD tmask) %{
17417   match(Set dst (ExpandBits src mask));
17418   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17419   format %{ "mov    $tsrc, $src\n\t"
17420             "mov    $tmask, $mask\n\t"
17421             "bdep   $tdst, $tsrc, $tmask\n\t"
17422             "mov    $dst, $tdst"
17423           %}
17424   ins_encode %{
17425     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17426     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17427     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17428     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17429   %}
17430   ins_pipe(pipe_slow);
17431 %}
17432 
17433 
17434 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17435                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17436   match(Set dst (ExpandBits (LoadL mem) mask));
17437   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17438   format %{ "ldrd   $tsrc, $mem\n\t"
17439             "ldrd   $tmask, $mask\n\t"
17440             "bdep   $tdst, $tsrc, $tmask\n\t"
17441             "mov    $dst, $tdst"
17442           %}
17443   ins_encode %{
17444     loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17445               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17446     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17447     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17448     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17449   %}
17450   ins_pipe(pipe_slow);
17451 %}
17452 
17453 //----------------------------- Reinterpret ----------------------------------
17454 // Reinterpret a half-precision float value in a floating point register to a general purpose register
17455 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
17456   match(Set dst (ReinterpretHF2S src));
17457   format %{ "reinterpretHF2S $dst, $src" %}
17458   ins_encode %{
17459     __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
17460   %}
17461   ins_pipe(pipe_slow);
17462 %}
17463 
17464 // Reinterpret a half-precision float value in a general purpose register to a floating point register
17465 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
17466   match(Set dst (ReinterpretS2HF src));
17467   format %{ "reinterpretS2HF $dst, $src" %}
17468   ins_encode %{
17469     __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
17470   %}
17471   ins_pipe(pipe_slow);
17472 %}
17473 
17474 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
17475 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
17476 // fcvt $tmp1_fpr, $src_fpr    // Convert float to half-precision float
17477 // mov  $tmp2_gpr, $tmp1_fpr   // Move half-precision float in FPR to a GPR
17478 // mov  $dst_fpr,  $tmp2_gpr   // Move the result from a GPR to an FPR
17479 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
17480 // can be omitted in this pattern, resulting in -
17481 // fcvt $dst, $src  // Convert float to half-precision float
17482 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
17483 %{
17484   match(Set dst (ReinterpretS2HF (ConvF2HF src)));
17485   format %{ "convF2HFAndS2HF $dst, $src" %}
17486   ins_encode %{
17487     __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
17488   %}
17489   ins_pipe(pipe_slow);
17490 %}
17491 
17492 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
17493 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
17494 // mov  $tmp1_gpr, $src_fpr  // Move the half-precision float from an FPR to a GPR
17495 // mov  $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
17496 // fcvt $dst_fpr,  $tmp2_fpr // Convert the half-precision float to 32-bit float
17497 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
17498 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
17499 // resulting in -
17500 // fcvt $dst, $src  // Convert half-precision float to a 32-bit float
17501 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
17502 %{
17503   match(Set dst (ConvHF2F (ReinterpretHF2S src)));
17504   format %{ "convHF2SAndHF2F $dst, $src" %}
17505   ins_encode %{
17506     __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
17507   %}
17508   ins_pipe(pipe_slow);
17509 %}
17510 
17511 // ============================================================================
17512 // This name is KNOWN by the ADLC and cannot be changed.
17513 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17514 // for this guy.
17515 instruct tlsLoadP(thread_RegP dst)
17516 %{
17517   match(Set dst (ThreadLocal));
17518 
17519   ins_cost(0);
17520 
17521   format %{ " -- \t// $dst=Thread::current(), empty" %}
17522 
17523   size(0);
17524 
17525   ins_encode( /*empty*/ );
17526 
17527   ins_pipe(pipe_class_empty);
17528 %}
17529 
17530 //----------PEEPHOLE RULES-----------------------------------------------------
17531 // These must follow all instruction definitions as they use the names
17532 // defined in the instructions definitions.
17533 //
17534 // peepmatch ( root_instr_name [preceding_instruction]* );
17535 //
17536 // peepconstraint %{
17537 // (instruction_number.operand_name relational_op instruction_number.operand_name
17538 //  [, ...] );
17539 // // instruction numbers are zero-based using left to right order in peepmatch
17540 //
17541 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17542 // // provide an instruction_number.operand_name for each operand that appears
17543 // // in the replacement instruction's match rule
17544 //
17545 // ---------VM FLAGS---------------------------------------------------------
17546 //
17547 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17548 //
17549 // Each peephole rule is given an identifying number starting with zero and
17550 // increasing by one in the order seen by the parser.  An individual peephole
17551 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17552 // on the command-line.
17553 //
17554 // ---------CURRENT LIMITATIONS----------------------------------------------
17555 //
17556 // Only match adjacent instructions in same basic block
17557 // Only equality constraints
17558 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17559 // Only one replacement instruction
17560 //
17561 // ---------EXAMPLE----------------------------------------------------------
17562 //
17563 // // pertinent parts of existing instructions in architecture description
17564 // instruct movI(iRegINoSp dst, iRegI src)
17565 // %{
17566 //   match(Set dst (CopyI src));
17567 // %}
17568 //
17569 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17570 // %{
17571 //   match(Set dst (AddI dst src));
17572 //   effect(KILL cr);
17573 // %}
17574 //
17575 // // Change (inc mov) to lea
17576 // peephole %{
17577 //   // increment preceded by register-register move
17578 //   peepmatch ( incI_iReg movI );
17579 //   // require that the destination register of the increment
17580 //   // match the destination register of the move
17581 //   peepconstraint ( 0.dst == 1.dst );
17582 //   // construct a replacement instruction that sets
17583 //   // the destination to ( move's source register + one )
17584 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17585 // %}
17586 //
17587 
17588 // Implementation no longer uses movX instructions since
17589 // machine-independent system no longer uses CopyX nodes.
17590 //
17591 // peephole
17592 // %{
17593 //   peepmatch (incI_iReg movI);
17594 //   peepconstraint (0.dst == 1.dst);
17595 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17596 // %}
17597 
17598 // peephole
17599 // %{
17600 //   peepmatch (decI_iReg movI);
17601 //   peepconstraint (0.dst == 1.dst);
17602 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17603 // %}
17604 
17605 // peephole
17606 // %{
17607 //   peepmatch (addI_iReg_imm movI);
17608 //   peepconstraint (0.dst == 1.dst);
17609 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17610 // %}
17611 
17612 // peephole
17613 // %{
17614 //   peepmatch (incL_iReg movL);
17615 //   peepconstraint (0.dst == 1.dst);
17616 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17617 // %}
17618 
17619 // peephole
17620 // %{
17621 //   peepmatch (decL_iReg movL);
17622 //   peepconstraint (0.dst == 1.dst);
17623 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17624 // %}
17625 
17626 // peephole
17627 // %{
17628 //   peepmatch (addL_iReg_imm movL);
17629 //   peepconstraint (0.dst == 1.dst);
17630 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17631 // %}
17632 
17633 // peephole
17634 // %{
17635 //   peepmatch (addP_iReg_imm movP);
17636 //   peepconstraint (0.dst == 1.dst);
17637 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17638 // %}
17639 
17640 // // Change load of spilled value to only a spill
17641 // instruct storeI(memory mem, iRegI src)
17642 // %{
17643 //   match(Set mem (StoreI mem src));
17644 // %}
17645 //
17646 // instruct loadI(iRegINoSp dst, memory mem)
17647 // %{
17648 //   match(Set dst (LoadI mem));
17649 // %}
17650 //
17651 
17652 //----------SMARTSPILL RULES---------------------------------------------------
17653 // These must follow all instruction definitions as they use the names
17654 // defined in the instructions definitions.
17655 
17656 // Local Variables:
17657 // mode: c++
17658 // End: