1 //
   2 // Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
   3 // Copyright (c) 2014, 2019, Red Hat, Inc. All rights reserved.
   4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5 //
   6 // This code is free software; you can redistribute it and/or modify it
   7 // under the terms of the GNU General Public License version 2 only, as
   8 // published by the Free Software Foundation.
   9 //
  10 // This code is distributed in the hope that it will be useful, but WITHOUT
  11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 // version 2 for more details (a copy is included in the LICENSE file that
  14 // accompanied this code).
  15 //
  16 // You should have received a copy of the GNU General Public License version
  17 // 2 along with this work; if not, write to the Free Software Foundation,
  18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19 //
  20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21 // or visit www.oracle.com if you need additional information or have any
  22 // questions.
  23 //
  24 //
  25 
  26 // AArch64 Architecture Description File
  27 
  28 //----------REGISTER DEFINITION BLOCK------------------------------------------
  29 // This information is used by the matcher and the register allocator to
  30 // describe individual registers and classes of registers within the target
  31 // archtecture.
  32 
  33 register %{
  34 //----------Architecture Description Register Definitions----------------------
  35 // General Registers
  36 // "reg_def"  name ( register save type, C convention save type,
  37 //                   ideal register type, encoding );
  38 // Register Save Types:
  39 //
  40 // NS  = No-Save:       The register allocator assumes that these registers
  41 //                      can be used without saving upon entry to the method, &
  42 //                      that they do not need to be saved at call sites.
  43 //
  44 // SOC = Save-On-Call:  The register allocator assumes that these registers
  45 //                      can be used without saving upon entry to the method,
  46 //                      but that they must be saved at call sites.
  47 //
  48 // SOE = Save-On-Entry: The register allocator assumes that these registers
  49 //                      must be saved before using them upon entry to the
  50 //                      method, but they do not need to be saved at call
  51 //                      sites.
  52 //
  53 // AS  = Always-Save:   The register allocator assumes that these registers
  54 //                      must be saved before using them upon entry to the
  55 //                      method, & that they must be saved at call sites.
  56 //
  57 // Ideal Register Type is used to determine how to save & restore a
  58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  60 //
  61 // The encoding number is the actual bit-pattern placed into the opcodes.
  62 
  63 // We must define the 64 bit int registers in two 32 bit halves, the
  64 // real lower register and a virtual upper half register. upper halves
  65 // are used by the register allocator but are not actually supplied as
  66 // operands to memory ops.
  67 //
  68 // follow the C1 compiler in making registers
  69 //
  70 //   r0-r7,r10-r26 volatile (caller save)
  71 //   r27-r32 system (no save, no allocate)
  72 //   r8-r9 invisible to the allocator (so we can use them as scratch regs)
  73 //
  74 // as regards Java usage. we don't use any callee save registers
  75 // because this makes it difficult to de-optimise a frame (see comment
  76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
  77 //
  78 
  79 // General Registers
  80 
  81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
  82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
  83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
  84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
  85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
  86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
  87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
  88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
  89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
  90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
  91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
  92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
  93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
  94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
  95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
  96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
  97 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  98 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  99 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
 100 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 101 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
 102 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
 103 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
 104 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
 105 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
 106 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
 107 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
 108 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
 109 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
 110 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
 111 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
 112 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
 113 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18->as_VMReg()        );
 114 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
 115 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
 116 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
 117 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
 118 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
 119 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
 120 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
 121 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
 122 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
 123 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
 124 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
 125 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
 126 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
 127 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
 128 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
 129 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
 130 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
 131 reg_def R27     (  NS, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
 132 reg_def R27_H   (  NS, SOE, Op_RegI, 27, r27->as_VMReg()->next());
 133 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
 134 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
 135 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
 136 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
 137 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
 138 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
 139 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
 140 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
 141 
 142 // ----------------------------
 143 // Float/Double Registers
 144 // ----------------------------
 145 
 146 // Double Registers
 147 
 148 // The rules of ADL require that double registers be defined in pairs.
 149 // Each pair must be two 32-bit values, but not necessarily a pair of
 150 // single float registers. In each pair, ADLC-assigned register numbers
 151 // must be adjacent, with the lower number even. Finally, when the
 152 // CPU stores such a register pair to memory, the word associated with
 153 // the lower ADLC-assigned number must be stored to the lower address.
 154 
 155 // AArch64 has 32 floating-point registers. Each can store a vector of
 156 // single or double precision floating-point values up to 8 * 32
 157 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
 158 // use the first float or double element of the vector.
 159 
 160 // for Java use float registers v0-v15 are always save on call whereas
 161 // the platform ABI treats v8-v15 as callee save). float registers
 162 // v16-v31 are SOC as per the platform spec
 163 
 164   reg_def V0   ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()          );
 165   reg_def V0_H ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()->next()  );
 166   reg_def V0_J ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()->next(2) );
 167   reg_def V0_K ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()->next(3) );
 168 
 169   reg_def V1   ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()          );
 170   reg_def V1_H ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()->next()  );
 171   reg_def V1_J ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()->next(2) );
 172   reg_def V1_K ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()->next(3) );
 173 
 174   reg_def V2   ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()          );
 175   reg_def V2_H ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()->next()  );
 176   reg_def V2_J ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()->next(2) );
 177   reg_def V2_K ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()->next(3) );
 178 
 179   reg_def V3   ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()          );
 180   reg_def V3_H ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()->next()  );
 181   reg_def V3_J ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()->next(2) );
 182   reg_def V3_K ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()->next(3) );
 183 
 184   reg_def V4   ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()          );
 185   reg_def V4_H ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()->next()  );
 186   reg_def V4_J ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()->next(2) );
 187   reg_def V4_K ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()->next(3) );
 188 
 189   reg_def V5   ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()          );
 190   reg_def V5_H ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()->next()  );
 191   reg_def V5_J ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()->next(2) );
 192   reg_def V5_K ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()->next(3) );
 193 
 194   reg_def V6   ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()          );
 195   reg_def V6_H ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()->next()  );
 196   reg_def V6_J ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()->next(2) );
 197   reg_def V6_K ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()->next(3) );
 198 
 199   reg_def V7   ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()          );
 200   reg_def V7_H ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()->next()  );
 201   reg_def V7_J ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()->next(2) );
 202   reg_def V7_K ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()->next(3) );
 203 
 204   reg_def V8   ( SOC, SOC, Op_RegF,  8, v8->as_VMReg()          );
 205   reg_def V8_H ( SOC, SOC, Op_RegF,  8, v8->as_VMReg()->next()  );
 206   reg_def V8_J ( SOC, SOC, Op_RegF,  8, v8->as_VMReg()->next(2) );
 207   reg_def V8_K ( SOC, SOC, Op_RegF,  8, v8->as_VMReg()->next(3) );
 208 
 209   reg_def V9   ( SOC, SOC, Op_RegF,  9, v9->as_VMReg()          );
 210   reg_def V9_H ( SOC, SOC, Op_RegF,  9, v9->as_VMReg()->next()  );
 211   reg_def V9_J ( SOC, SOC, Op_RegF,  9, v9->as_VMReg()->next(2) );
 212   reg_def V9_K ( SOC, SOC, Op_RegF,  9, v9->as_VMReg()->next(3) );
 213 
 214   reg_def V10  ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()         );
 215   reg_def V10_H( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next() );
 216   reg_def V10_J( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2));
 217   reg_def V10_K( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3));
 218 
 219   reg_def V11  ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()         );
 220   reg_def V11_H( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next() );
 221   reg_def V11_J( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2));
 222   reg_def V11_K( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3));
 223 
 224   reg_def V12  ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()         );
 225   reg_def V12_H( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next() );
 226   reg_def V12_J( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2));
 227   reg_def V12_K( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3));
 228 
 229   reg_def V13  ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()         );
 230   reg_def V13_H( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next() );
 231   reg_def V13_J( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2));
 232   reg_def V13_K( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3));
 233 
 234   reg_def V14  ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()         );
 235   reg_def V14_H( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next() );
 236   reg_def V14_J( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2));
 237   reg_def V14_K( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3));
 238 
 239   reg_def V15  ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()         );
 240   reg_def V15_H( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next() );
 241   reg_def V15_J( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2));
 242   reg_def V15_K( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3));
 243 
 244   reg_def V16  ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()         );
 245   reg_def V16_H( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
 246   reg_def V16_J( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2));
 247   reg_def V16_K( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3));
 248 
 249   reg_def V17  ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()         );
 250   reg_def V17_H( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
 251   reg_def V17_J( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2));
 252   reg_def V17_K( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3));
 253 
 254   reg_def V18  ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()         );
 255   reg_def V18_H( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
 256   reg_def V18_J( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2));
 257   reg_def V18_K( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3));
 258 
 259   reg_def V19  ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()         );
 260   reg_def V19_H( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
 261   reg_def V19_J( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2));
 262   reg_def V19_K( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3));
 263 
 264   reg_def V20  ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()         );
 265   reg_def V20_H( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
 266   reg_def V20_J( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2));
 267   reg_def V20_K( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3));
 268 
 269   reg_def V21  ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()         );
 270   reg_def V21_H( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
 271   reg_def V21_J( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2));
 272   reg_def V21_K( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3));
 273 
 274   reg_def V22  ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()         );
 275   reg_def V22_H( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
 276   reg_def V22_J( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2));
 277   reg_def V22_K( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3));
 278 
 279   reg_def V23  ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()         );
 280   reg_def V23_H( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
 281   reg_def V23_J( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2));
 282   reg_def V23_K( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3));
 283 
 284   reg_def V24  ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()         );
 285   reg_def V24_H( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
 286   reg_def V24_J( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2));
 287   reg_def V24_K( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3));
 288 
 289   reg_def V25  ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()         );
 290   reg_def V25_H( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
 291   reg_def V25_J( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2));
 292   reg_def V25_K( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3));
 293 
 294   reg_def V26  ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()         );
 295   reg_def V26_H( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
 296   reg_def V26_J( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2));
 297   reg_def V26_K( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3));
 298 
 299   reg_def V27  ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()         );
 300   reg_def V27_H( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
 301   reg_def V27_J( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2));
 302   reg_def V27_K( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3));
 303 
 304   reg_def V28  ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()         );
 305   reg_def V28_H( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
 306   reg_def V28_J( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2));
 307   reg_def V28_K( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3));
 308 
 309   reg_def V29  ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()         );
 310   reg_def V29_H( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
 311   reg_def V29_J( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2));
 312   reg_def V29_K( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3));
 313 
 314   reg_def V30  ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()         );
 315   reg_def V30_H( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
 316   reg_def V30_J( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2));
 317   reg_def V30_K( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3));
 318 
 319   reg_def V31  ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()         );
 320   reg_def V31_H( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
 321   reg_def V31_J( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2));
 322   reg_def V31_K( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3));
 323 
 324 // ----------------------------
 325 // Special Registers
 326 // ----------------------------
 327 
 328 // the AArch64 CSPR status flag register is not directly acessible as
 329 // instruction operand. the FPSR status flag register is a system
 330 // register which can be written/read using MSR/MRS but again does not
 331 // appear as an operand (a code identifying the FSPR occurs as an
 332 // immediate value in the instruction).
 333 
 334 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
 335 
 336 
 337 // Specify priority of register selection within phases of register
 338 // allocation.  Highest priority is first.  A useful heuristic is to
 339 // give registers a low priority when they are required by machine
 340 // instructions, like EAX and EDX on I486, and choose no-save registers
 341 // before save-on-call, & save-on-call before save-on-entry.  Registers
 342 // which participate in fixed calling sequences should come last.
 343 // Registers which are used as pairs must fall on an even boundary.
 344 
 345 alloc_class chunk0(
 346     // volatiles
 347     R10, R10_H,
 348     R11, R11_H,
 349     R12, R12_H,
 350     R13, R13_H,
 351     R14, R14_H,
 352     R15, R15_H,
 353     R16, R16_H,
 354     R17, R17_H,
 355     R18, R18_H,
 356 
 357     // arg registers
 358     R0, R0_H,
 359     R1, R1_H,
 360     R2, R2_H,
 361     R3, R3_H,
 362     R4, R4_H,
 363     R5, R5_H,
 364     R6, R6_H,
 365     R7, R7_H,
 366 
 367     // non-volatiles
 368     R19, R19_H,
 369     R20, R20_H,
 370     R21, R21_H,
 371     R22, R22_H,
 372     R23, R23_H,
 373     R24, R24_H,
 374     R25, R25_H,
 375     R26, R26_H,
 376 
 377     // non-allocatable registers
 378 
 379     R27, R27_H, // heapbase
 380     R28, R28_H, // thread
 381     R29, R29_H, // fp
 382     R30, R30_H, // lr
 383     R31, R31_H, // sp
 384 );
 385 
 386 alloc_class chunk1(
 387 
 388     // no save
 389     V16, V16_H, V16_J, V16_K,
 390     V17, V17_H, V17_J, V17_K,
 391     V18, V18_H, V18_J, V18_K,
 392     V19, V19_H, V19_J, V19_K,
 393     V20, V20_H, V20_J, V20_K,
 394     V21, V21_H, V21_J, V21_K,
 395     V22, V22_H, V22_J, V22_K,
 396     V23, V23_H, V23_J, V23_K,
 397     V24, V24_H, V24_J, V24_K,
 398     V25, V25_H, V25_J, V25_K,
 399     V26, V26_H, V26_J, V26_K,
 400     V27, V27_H, V27_J, V27_K,
 401     V28, V28_H, V28_J, V28_K,
 402     V29, V29_H, V29_J, V29_K,
 403     V30, V30_H, V30_J, V30_K,
 404     V31, V31_H, V31_J, V31_K,
 405 
 406     // arg registers
 407     V0, V0_H, V0_J, V0_K,
 408     V1, V1_H, V1_J, V1_K,
 409     V2, V2_H, V2_J, V2_K,
 410     V3, V3_H, V3_J, V3_K,
 411     V4, V4_H, V4_J, V4_K,
 412     V5, V5_H, V5_J, V5_K,
 413     V6, V6_H, V6_J, V6_K,
 414     V7, V7_H, V7_J, V7_K,
 415 
 416     // non-volatiles
 417     V8, V8_H, V8_J, V8_K,
 418     V9, V9_H, V9_J, V9_K,
 419     V10, V10_H, V10_J, V10_K,
 420     V11, V11_H, V11_J, V11_K,
 421     V12, V12_H, V12_J, V12_K,
 422     V13, V13_H, V13_J, V13_K,
 423     V14, V14_H, V14_J, V14_K,
 424     V15, V15_H, V15_J, V15_K,
 425 );
 426 
 427 alloc_class chunk2(RFLAGS);
 428 
 429 //----------Architecture Description Register Classes--------------------------
 430 // Several register classes are automatically defined based upon information in
 431 // this architecture description.
 432 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 433 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 434 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 435 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 436 //
 437 
 438 // Class for all 32 bit integer registers -- excludes SP which will
 439 // never be used as an integer register
 440 reg_class any_reg32(
 441     R0,
 442     R1,
 443     R2,
 444     R3,
 445     R4,
 446     R5,
 447     R6,
 448     R7,
 449     R10,
 450     R11,
 451     R12,
 452     R13,
 453     R14,
 454     R15,
 455     R16,
 456     R17,
 457     R18,
 458     R19,
 459     R20,
 460     R21,
 461     R22,
 462     R23,
 463     R24,
 464     R25,
 465     R26,
 466     R27,
 467     R28,
 468     R29,
 469     R30
 470 );
 471 
 472 // Singleton class for R0 int register
 473 reg_class int_r0_reg(R0);
 474 
 475 // Singleton class for R2 int register
 476 reg_class int_r2_reg(R2);
 477 
 478 // Singleton class for R3 int register
 479 reg_class int_r3_reg(R3);
 480 
 481 // Singleton class for R4 int register
 482 reg_class int_r4_reg(R4);
 483 
 484 // Class for all long integer registers (including RSP)
 485 reg_class any_reg(
 486     R0, R0_H,
 487     R1, R1_H,
 488     R2, R2_H,
 489     R3, R3_H,
 490     R4, R4_H,
 491     R5, R5_H,
 492     R6, R6_H,
 493     R7, R7_H,
 494     R10, R10_H,
 495     R11, R11_H,
 496     R12, R12_H,
 497     R13, R13_H,
 498     R14, R14_H,
 499     R15, R15_H,
 500     R16, R16_H,
 501     R17, R17_H,
 502     R18, R18_H,
 503     R19, R19_H,
 504     R20, R20_H,
 505     R21, R21_H,
 506     R22, R22_H,
 507     R23, R23_H,
 508     R24, R24_H,
 509     R25, R25_H,
 510     R26, R26_H,
 511     R27, R27_H,
 512     R28, R28_H,
 513     R29, R29_H,
 514     R30, R30_H,
 515     R31, R31_H
 516 );
 517 
 518 // Class for all non-special integer registers
 519 reg_class no_special_reg32_no_fp(
 520     R0,
 521     R1,
 522     R2,
 523     R3,
 524     R4,
 525     R5,
 526     R6,
 527     R7,
 528     R10,
 529     R11,
 530     R12,                        // rmethod
 531     R13,
 532     R14,
 533     R15,
 534     R16,
 535     R17,
 536     R18,
 537     R19,
 538     R20,
 539     R21,
 540     R22,
 541     R23,
 542     R24,
 543     R25,
 544     R26
 545  /* R27, */                     // heapbase
 546  /* R28, */                     // thread
 547  /* R29, */                     // fp
 548  /* R30, */                     // lr
 549  /* R31 */                      // sp
 550 );
 551 
 552 reg_class no_special_reg32_with_fp(
 553     R0,
 554     R1,
 555     R2,
 556     R3,
 557     R4,
 558     R5,
 559     R6,
 560     R7,
 561     R10,
 562     R11,
 563     R12,                        // rmethod
 564     R13,
 565     R14,
 566     R15,
 567     R16,
 568     R17,
 569     R18,
 570     R19,
 571     R20,
 572     R21,
 573     R22,
 574     R23,
 575     R24,
 576     R25,
 577     R26
 578  /* R27, */                     // heapbase
 579  /* R28, */                     // thread
 580     R29,                        // fp
 581  /* R30, */                     // lr
 582  /* R31 */                      // sp
 583 );
 584 
 585 reg_class_dynamic no_special_reg32(no_special_reg32_no_fp, no_special_reg32_with_fp, %{ PreserveFramePointer %});
 586 
 587 // Class for all non-special long integer registers
 588 reg_class no_special_reg_no_fp(
 589     R0, R0_H,
 590     R1, R1_H,
 591     R2, R2_H,
 592     R3, R3_H,
 593     R4, R4_H,
 594     R5, R5_H,
 595     R6, R6_H,
 596     R7, R7_H,
 597     R10, R10_H,
 598     R11, R11_H,
 599     R12, R12_H,                 // rmethod
 600     R13, R13_H,
 601     R14, R14_H,
 602     R15, R15_H,
 603     R16, R16_H,
 604     R17, R17_H,
 605     R18, R18_H,
 606     R19, R19_H,
 607     R20, R20_H,
 608     R21, R21_H,
 609     R22, R22_H,
 610     R23, R23_H,
 611     R24, R24_H,
 612     R25, R25_H,
 613     R26, R26_H,
 614  /* R27, R27_H, */              // heapbase
 615  /* R28, R28_H, */              // thread
 616  /* R29, R29_H, */              // fp
 617  /* R30, R30_H, */              // lr
 618  /* R31, R31_H */               // sp
 619 );
 620 
 621 reg_class no_special_reg_with_fp(
 622     R0, R0_H,
 623     R1, R1_H,
 624     R2, R2_H,
 625     R3, R3_H,
 626     R4, R4_H,
 627     R5, R5_H,
 628     R6, R6_H,
 629     R7, R7_H,
 630     R10, R10_H,
 631     R11, R11_H,
 632     R12, R12_H,                 // rmethod
 633     R13, R13_H,
 634     R14, R14_H,
 635     R15, R15_H,
 636     R16, R16_H,
 637     R17, R17_H,
 638     R18, R18_H,
 639     R19, R19_H,
 640     R20, R20_H,
 641     R21, R21_H,
 642     R22, R22_H,
 643     R23, R23_H,
 644     R24, R24_H,
 645     R25, R25_H,
 646     R26, R26_H,
 647  /* R27, R27_H, */              // heapbase
 648  /* R28, R28_H, */              // thread
 649     R29, R29_H,                 // fp
 650  /* R30, R30_H, */              // lr
 651  /* R31, R31_H */               // sp
 652 );
 653 
 654 reg_class_dynamic no_special_reg(no_special_reg_no_fp, no_special_reg_with_fp, %{ PreserveFramePointer %});
 655 
 656 // Class for 64 bit register r0
 657 reg_class r0_reg(
 658     R0, R0_H
 659 );
 660 
 661 // Class for 64 bit register r1
 662 reg_class r1_reg(
 663     R1, R1_H
 664 );
 665 
 666 // Class for 64 bit register r2
 667 reg_class r2_reg(
 668     R2, R2_H
 669 );
 670 
 671 // Class for 64 bit register r3
 672 reg_class r3_reg(
 673     R3, R3_H
 674 );
 675 
 676 // Class for 64 bit register r4
 677 reg_class r4_reg(
 678     R4, R4_H
 679 );
 680 
 681 // Class for 64 bit register r5
 682 reg_class r5_reg(
 683     R5, R5_H
 684 );
 685 
 686 // Class for 64 bit register r10
 687 reg_class r10_reg(
 688     R10, R10_H
 689 );
 690 
 691 // Class for 64 bit register r11
 692 reg_class r11_reg(
 693     R11, R11_H
 694 );
 695 
 696 // Class for method register
 697 reg_class method_reg(
 698     R12, R12_H
 699 );
 700 
 701 // Class for heapbase register
 702 reg_class heapbase_reg(
 703     R27, R27_H
 704 );
 705 
 706 // Class for thread register
 707 reg_class thread_reg(
 708     R28, R28_H
 709 );
 710 
 711 // Class for frame pointer register
 712 reg_class fp_reg(
 713     R29, R29_H
 714 );
 715 
 716 // Class for link register
 717 reg_class lr_reg(
 718     R30, R30_H
 719 );
 720 
 721 // Class for long sp register
 722 reg_class sp_reg(
 723   R31, R31_H
 724 );
 725 
 726 // Class for all pointer registers
 727 reg_class ptr_reg(
 728     R0, R0_H,
 729     R1, R1_H,
 730     R2, R2_H,
 731     R3, R3_H,
 732     R4, R4_H,
 733     R5, R5_H,
 734     R6, R6_H,
 735     R7, R7_H,
 736     R10, R10_H,
 737     R11, R11_H,
 738     R12, R12_H,
 739     R13, R13_H,
 740     R14, R14_H,
 741     R15, R15_H,
 742     R16, R16_H,
 743     R17, R17_H,
 744     R18, R18_H,
 745     R19, R19_H,
 746     R20, R20_H,
 747     R21, R21_H,
 748     R22, R22_H,
 749     R23, R23_H,
 750     R24, R24_H,
 751     R25, R25_H,
 752     R26, R26_H,
 753     R27, R27_H,
 754     R28, R28_H,
 755     R29, R29_H,
 756     R30, R30_H,
 757     R31, R31_H
 758 );
 759 
 760 // Class for all non_special pointer registers
 761 reg_class no_special_ptr_reg(
 762     R0, R0_H,
 763     R1, R1_H,
 764     R2, R2_H,
 765     R3, R3_H,
 766     R4, R4_H,
 767     R5, R5_H,
 768     R6, R6_H,
 769     R7, R7_H,
 770     R10, R10_H,
 771     R11, R11_H,
 772     R12, R12_H,
 773     R13, R13_H,
 774     R14, R14_H,
 775     R15, R15_H,
 776     R16, R16_H,
 777     R17, R17_H,
 778     R18, R18_H,
 779     R19, R19_H,
 780     R20, R20_H,
 781     R21, R21_H,
 782     R22, R22_H,
 783     R23, R23_H,
 784     R24, R24_H,
 785     R25, R25_H,
 786     R26, R26_H,
 787  /* R27, R27_H, */              // heapbase
 788  /* R28, R28_H, */              // thread
 789  /* R29, R29_H, */              // fp
 790  /* R30, R30_H, */              // lr
 791  /* R31, R31_H */               // sp
 792 );
 793 
 794 // Class for all float registers
 795 reg_class float_reg(
 796     V0,
 797     V1,
 798     V2,
 799     V3,
 800     V4,
 801     V5,
 802     V6,
 803     V7,
 804     V8,
 805     V9,
 806     V10,
 807     V11,
 808     V12,
 809     V13,
 810     V14,
 811     V15,
 812     V16,
 813     V17,
 814     V18,
 815     V19,
 816     V20,
 817     V21,
 818     V22,
 819     V23,
 820     V24,
 821     V25,
 822     V26,
 823     V27,
 824     V28,
 825     V29,
 826     V30,
 827     V31
 828 );
 829 
 830 // Double precision float registers have virtual `high halves' that
 831 // are needed by the allocator.
 832 // Class for all double registers
 833 reg_class double_reg(
 834     V0, V0_H,
 835     V1, V1_H,
 836     V2, V2_H,
 837     V3, V3_H,
 838     V4, V4_H,
 839     V5, V5_H,
 840     V6, V6_H,
 841     V7, V7_H,
 842     V8, V8_H,
 843     V9, V9_H,
 844     V10, V10_H,
 845     V11, V11_H,
 846     V12, V12_H,
 847     V13, V13_H,
 848     V14, V14_H,
 849     V15, V15_H,
 850     V16, V16_H,
 851     V17, V17_H,
 852     V18, V18_H,
 853     V19, V19_H,
 854     V20, V20_H,
 855     V21, V21_H,
 856     V22, V22_H,
 857     V23, V23_H,
 858     V24, V24_H,
 859     V25, V25_H,
 860     V26, V26_H,
 861     V27, V27_H,
 862     V28, V28_H,
 863     V29, V29_H,
 864     V30, V30_H,
 865     V31, V31_H
 866 );
 867 
 868 // Class for all 64bit vector registers
 869 reg_class vectord_reg(
 870     V0, V0_H,
 871     V1, V1_H,
 872     V2, V2_H,
 873     V3, V3_H,
 874     V4, V4_H,
 875     V5, V5_H,
 876     V6, V6_H,
 877     V7, V7_H,
 878     V8, V8_H,
 879     V9, V9_H,
 880     V10, V10_H,
 881     V11, V11_H,
 882     V12, V12_H,
 883     V13, V13_H,
 884     V14, V14_H,
 885     V15, V15_H,
 886     V16, V16_H,
 887     V17, V17_H,
 888     V18, V18_H,
 889     V19, V19_H,
 890     V20, V20_H,
 891     V21, V21_H,
 892     V22, V22_H,
 893     V23, V23_H,
 894     V24, V24_H,
 895     V25, V25_H,
 896     V26, V26_H,
 897     V27, V27_H,
 898     V28, V28_H,
 899     V29, V29_H,
 900     V30, V30_H,
 901     V31, V31_H
 902 );
 903 
 904 // Class for all 128bit vector registers
 905 reg_class vectorx_reg(
 906     V0, V0_H, V0_J, V0_K,
 907     V1, V1_H, V1_J, V1_K,
 908     V2, V2_H, V2_J, V2_K,
 909     V3, V3_H, V3_J, V3_K,
 910     V4, V4_H, V4_J, V4_K,
 911     V5, V5_H, V5_J, V5_K,
 912     V6, V6_H, V6_J, V6_K,
 913     V7, V7_H, V7_J, V7_K,
 914     V8, V8_H, V8_J, V8_K,
 915     V9, V9_H, V9_J, V9_K,
 916     V10, V10_H, V10_J, V10_K,
 917     V11, V11_H, V11_J, V11_K,
 918     V12, V12_H, V12_J, V12_K,
 919     V13, V13_H, V13_J, V13_K,
 920     V14, V14_H, V14_J, V14_K,
 921     V15, V15_H, V15_J, V15_K,
 922     V16, V16_H, V16_J, V16_K,
 923     V17, V17_H, V17_J, V17_K,
 924     V18, V18_H, V18_J, V18_K,
 925     V19, V19_H, V19_J, V19_K,
 926     V20, V20_H, V20_J, V20_K,
 927     V21, V21_H, V21_J, V21_K,
 928     V22, V22_H, V22_J, V22_K,
 929     V23, V23_H, V23_J, V23_K,
 930     V24, V24_H, V24_J, V24_K,
 931     V25, V25_H, V25_J, V25_K,
 932     V26, V26_H, V26_J, V26_K,
 933     V27, V27_H, V27_J, V27_K,
 934     V28, V28_H, V28_J, V28_K,
 935     V29, V29_H, V29_J, V29_K,
 936     V30, V30_H, V30_J, V30_K,
 937     V31, V31_H, V31_J, V31_K
 938 );
 939 
 940 // Class for 128 bit register v0
 941 reg_class v0_reg(
 942     V0, V0_H
 943 );
 944 
 945 // Class for 128 bit register v1
 946 reg_class v1_reg(
 947     V1, V1_H
 948 );
 949 
 950 // Class for 128 bit register v2
 951 reg_class v2_reg(
 952     V2, V2_H
 953 );
 954 
 955 // Class for 128 bit register v3
 956 reg_class v3_reg(
 957     V3, V3_H
 958 );
 959 
 960 // Class for 128 bit register v4
 961 reg_class v4_reg(
 962     V4, V4_H
 963 );
 964 
 965 // Class for 128 bit register v5
 966 reg_class v5_reg(
 967     V5, V5_H
 968 );
 969 
 970 // Class for 128 bit register v6
 971 reg_class v6_reg(
 972     V6, V6_H
 973 );
 974 
 975 // Class for 128 bit register v7
 976 reg_class v7_reg(
 977     V7, V7_H
 978 );
 979 
 980 // Class for 128 bit register v8
 981 reg_class v8_reg(
 982     V8, V8_H
 983 );
 984 
 985 // Class for 128 bit register v9
 986 reg_class v9_reg(
 987     V9, V9_H
 988 );
 989 
 990 // Class for 128 bit register v10
 991 reg_class v10_reg(
 992     V10, V10_H
 993 );
 994 
 995 // Class for 128 bit register v11
 996 reg_class v11_reg(
 997     V11, V11_H
 998 );
 999 
1000 // Class for 128 bit register v12
1001 reg_class v12_reg(
1002     V12, V12_H
1003 );
1004 
1005 // Class for 128 bit register v13
1006 reg_class v13_reg(
1007     V13, V13_H
1008 );
1009 
1010 // Class for 128 bit register v14
1011 reg_class v14_reg(
1012     V14, V14_H
1013 );
1014 
1015 // Class for 128 bit register v15
1016 reg_class v15_reg(
1017     V15, V15_H
1018 );
1019 
1020 // Class for 128 bit register v16
1021 reg_class v16_reg(
1022     V16, V16_H
1023 );
1024 
1025 // Class for 128 bit register v17
1026 reg_class v17_reg(
1027     V17, V17_H
1028 );
1029 
1030 // Class for 128 bit register v18
1031 reg_class v18_reg(
1032     V18, V18_H
1033 );
1034 
1035 // Class for 128 bit register v19
1036 reg_class v19_reg(
1037     V19, V19_H
1038 );
1039 
1040 // Class for 128 bit register v20
1041 reg_class v20_reg(
1042     V20, V20_H
1043 );
1044 
1045 // Class for 128 bit register v21
1046 reg_class v21_reg(
1047     V21, V21_H
1048 );
1049 
1050 // Class for 128 bit register v22
1051 reg_class v22_reg(
1052     V22, V22_H
1053 );
1054 
1055 // Class for 128 bit register v23
1056 reg_class v23_reg(
1057     V23, V23_H
1058 );
1059 
1060 // Class for 128 bit register v24
1061 reg_class v24_reg(
1062     V24, V24_H
1063 );
1064 
1065 // Class for 128 bit register v25
1066 reg_class v25_reg(
1067     V25, V25_H
1068 );
1069 
1070 // Class for 128 bit register v26
1071 reg_class v26_reg(
1072     V26, V26_H
1073 );
1074 
1075 // Class for 128 bit register v27
1076 reg_class v27_reg(
1077     V27, V27_H
1078 );
1079 
1080 // Class for 128 bit register v28
1081 reg_class v28_reg(
1082     V28, V28_H
1083 );
1084 
1085 // Class for 128 bit register v29
1086 reg_class v29_reg(
1087     V29, V29_H
1088 );
1089 
1090 // Class for 128 bit register v30
1091 reg_class v30_reg(
1092     V30, V30_H
1093 );
1094 
1095 // Class for 128 bit register v31
1096 reg_class v31_reg(
1097     V31, V31_H
1098 );
1099 
1100 // Singleton class for condition codes
1101 reg_class int_flags(RFLAGS);
1102 
1103 %}
1104 
1105 //----------DEFINITION BLOCK---------------------------------------------------
1106 // Define name --> value mappings to inform the ADLC of an integer valued name
1107 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1108 // Format:
1109 //        int_def  <name>         ( <int_value>, <expression>);
1110 // Generated Code in ad_<arch>.hpp
1111 //        #define  <name>   (<expression>)
1112 //        // value == <int_value>
1113 // Generated code in ad_<arch>.cpp adlc_verification()
1114 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1115 //
1116 
1117 // we follow the ppc-aix port in using a simple cost model which ranks
1118 // register operations as cheap, memory ops as more expensive and
1119 // branches as most expensive. the first two have a low as well as a
1120 // normal cost. huge cost appears to be a way of saying don't do
1121 // something
1122 
1123 definitions %{
1124   // The default cost (of a register move instruction).
1125   int_def INSN_COST            (    100,     100);
1126   int_def BRANCH_COST          (    200,     2 * INSN_COST);
1127   int_def CALL_COST            (    200,     2 * INSN_COST);
1128   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
1129 %}
1130 
1131 
1132 //----------SOURCE BLOCK-------------------------------------------------------
1133 // This is a block of C++ code which provides values, functions, and
1134 // definitions necessary in the rest of the architecture description
1135 
1136 source_hpp %{
1137 
1138 #include "asm/macroAssembler.hpp"
1139 #include "gc/shared/cardTable.hpp"
1140 #include "gc/shared/cardTableBarrierSet.hpp"
1141 #include "gc/shared/collectedHeap.hpp"
1142 #include "opto/addnode.hpp"
1143 
1144 class CallStubImpl {
1145 
1146   //--------------------------------------------------------------
1147   //---<  Used for optimization in Compile::shorten_branches  >---
1148   //--------------------------------------------------------------
1149 
1150  public:
1151   // Size of call trampoline stub.
1152   static uint size_call_trampoline() {
1153     return 0; // no call trampolines on this platform
1154   }
1155 
1156   // number of relocations needed by a call trampoline stub
1157   static uint reloc_call_trampoline() {
1158     return 0; // no call trampolines on this platform
1159   }
1160 };
1161 
1162 class HandlerImpl {
1163 
1164  public:
1165 
1166   static int emit_exception_handler(CodeBuffer &cbuf);
1167   static int emit_deopt_handler(CodeBuffer& cbuf);
1168 
1169   static uint size_exception_handler() {
1170     return MacroAssembler::far_branch_size();
1171   }
1172 
1173   static uint size_deopt_handler() {
1174     // count one adr and one far branch instruction
1175     return 4 * NativeInstruction::instruction_size;
1176   }
1177 };
1178 
1179  bool is_CAS(int opcode, bool maybe_volatile);
1180 
1181   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1182 
1183   bool unnecessary_acquire(const Node *barrier);
1184   bool needs_acquiring_load(const Node *load);
1185 
1186   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1187 
1188   bool unnecessary_release(const Node *barrier);
1189   bool unnecessary_volatile(const Node *barrier);
1190   bool needs_releasing_store(const Node *store);
1191 
1192   // predicate controlling translation of CompareAndSwapX
1193   bool needs_acquiring_load_exclusive(const Node *load);
1194 
1195   // predicate controlling translation of StoreCM
1196   bool unnecessary_storestore(const Node *storecm);
1197 
1198   // predicate controlling addressing modes
1199   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1200 %}
1201 
1202 source %{
1203 
1204   // Optimizaton of volatile gets and puts
1205   // -------------------------------------
1206   //
1207   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1208   // use to implement volatile reads and writes. For a volatile read
1209   // we simply need
1210   //
1211   //   ldar<x>
1212   //
1213   // and for a volatile write we need
1214   //
1215   //   stlr<x>
1216   //
1217   // Alternatively, we can implement them by pairing a normal
1218   // load/store with a memory barrier. For a volatile read we need
1219   //
1220   //   ldr<x>
1221   //   dmb ishld
1222   //
1223   // for a volatile write
1224   //
1225   //   dmb ish
1226   //   str<x>
1227   //   dmb ish
1228   //
1229   // We can also use ldaxr and stlxr to implement compare and swap CAS
1230   // sequences. These are normally translated to an instruction
1231   // sequence like the following
1232   //
1233   //   dmb      ish
1234   // retry:
1235   //   ldxr<x>   rval raddr
1236   //   cmp       rval rold
1237   //   b.ne done
1238   //   stlxr<x>  rval, rnew, rold
1239   //   cbnz      rval retry
1240   // done:
1241   //   cset      r0, eq
1242   //   dmb ishld
1243   //
1244   // Note that the exclusive store is already using an stlxr
1245   // instruction. That is required to ensure visibility to other
1246   // threads of the exclusive write (assuming it succeeds) before that
1247   // of any subsequent writes.
1248   //
1249   // The following instruction sequence is an improvement on the above
1250   //
1251   // retry:
1252   //   ldaxr<x>  rval raddr
1253   //   cmp       rval rold
1254   //   b.ne done
1255   //   stlxr<x>  rval, rnew, rold
1256   //   cbnz      rval retry
1257   // done:
1258   //   cset      r0, eq
1259   //
1260   // We don't need the leading dmb ish since the stlxr guarantees
1261   // visibility of prior writes in the case that the swap is
1262   // successful. Crucially we don't have to worry about the case where
1263   // the swap is not successful since no valid program should be
1264   // relying on visibility of prior changes by the attempting thread
1265   // in the case where the CAS fails.
1266   //
1267   // Similarly, we don't need the trailing dmb ishld if we substitute
1268   // an ldaxr instruction since that will provide all the guarantees we
1269   // require regarding observation of changes made by other threads
1270   // before any change to the CAS address observed by the load.
1271   //
1272   // In order to generate the desired instruction sequence we need to
1273   // be able to identify specific 'signature' ideal graph node
1274   // sequences which i) occur as a translation of a volatile reads or
1275   // writes or CAS operations and ii) do not occur through any other
1276   // translation or graph transformation. We can then provide
1277   // alternative aldc matching rules which translate these node
1278   // sequences to the desired machine code sequences. Selection of the
1279   // alternative rules can be implemented by predicates which identify
1280   // the relevant node sequences.
1281   //
1282   // The ideal graph generator translates a volatile read to the node
1283   // sequence
1284   //
1285   //   LoadX[mo_acquire]
1286   //   MemBarAcquire
1287   //
1288   // As a special case when using the compressed oops optimization we
1289   // may also see this variant
1290   //
1291   //   LoadN[mo_acquire]
1292   //   DecodeN
1293   //   MemBarAcquire
1294   //
1295   // A volatile write is translated to the node sequence
1296   //
1297   //   MemBarRelease
1298   //   StoreX[mo_release] {CardMark}-optional
1299   //   MemBarVolatile
1300   //
1301   // n.b. the above node patterns are generated with a strict
1302   // 'signature' configuration of input and output dependencies (see
1303   // the predicates below for exact details). The card mark may be as
1304   // simple as a few extra nodes or, in a few GC configurations, may
1305   // include more complex control flow between the leading and
1306   // trailing memory barriers. However, whatever the card mark
1307   // configuration these signatures are unique to translated volatile
1308   // reads/stores -- they will not appear as a result of any other
1309   // bytecode translation or inlining nor as a consequence of
1310   // optimizing transforms.
1311   //
1312   // We also want to catch inlined unsafe volatile gets and puts and
1313   // be able to implement them using either ldar<x>/stlr<x> or some
1314   // combination of ldr<x>/stlr<x> and dmb instructions.
1315   //
1316   // Inlined unsafe volatiles puts manifest as a minor variant of the
1317   // normal volatile put node sequence containing an extra cpuorder
1318   // membar
1319   //
1320   //   MemBarRelease
1321   //   MemBarCPUOrder
1322   //   StoreX[mo_release] {CardMark}-optional
1323   //   MemBarCPUOrder
1324   //   MemBarVolatile
1325   //
1326   // n.b. as an aside, a cpuorder membar is not itself subject to
1327   // matching and translation by adlc rules.  However, the rule
1328   // predicates need to detect its presence in order to correctly
1329   // select the desired adlc rules.
1330   //
1331   // Inlined unsafe volatile gets manifest as a slightly different
1332   // node sequence to a normal volatile get because of the
1333   // introduction of some CPUOrder memory barriers to bracket the
1334   // Load. However, but the same basic skeleton of a LoadX feeding a
1335   // MemBarAcquire, possibly thorugh an optional DecodeN, is still
1336   // present
1337   //
1338   //   MemBarCPUOrder
1339   //        ||       \\
1340   //   MemBarCPUOrder LoadX[mo_acquire]
1341   //        ||            |
1342   //        ||       {DecodeN} optional
1343   //        ||       /
1344   //     MemBarAcquire
1345   //
1346   // In this case the acquire membar does not directly depend on the
1347   // load. However, we can be sure that the load is generated from an
1348   // inlined unsafe volatile get if we see it dependent on this unique
1349   // sequence of membar nodes. Similarly, given an acquire membar we
1350   // can know that it was added because of an inlined unsafe volatile
1351   // get if it is fed and feeds a cpuorder membar and if its feed
1352   // membar also feeds an acquiring load.
1353   //
1354   // Finally an inlined (Unsafe) CAS operation is translated to the
1355   // following ideal graph
1356   //
1357   //   MemBarRelease
1358   //   MemBarCPUOrder
1359   //   CompareAndSwapX {CardMark}-optional
1360   //   MemBarCPUOrder
1361   //   MemBarAcquire
1362   //
1363   // So, where we can identify these volatile read and write
1364   // signatures we can choose to plant either of the above two code
1365   // sequences. For a volatile read we can simply plant a normal
1366   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1367   // also choose to inhibit translation of the MemBarAcquire and
1368   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1369   //
1370   // When we recognise a volatile store signature we can choose to
1371   // plant at a dmb ish as a translation for the MemBarRelease, a
1372   // normal str<x> and then a dmb ish for the MemBarVolatile.
1373   // Alternatively, we can inhibit translation of the MemBarRelease
1374   // and MemBarVolatile and instead plant a simple stlr<x>
1375   // instruction.
1376   //
1377   // when we recognise a CAS signature we can choose to plant a dmb
1378   // ish as a translation for the MemBarRelease, the conventional
1379   // macro-instruction sequence for the CompareAndSwap node (which
1380   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1381   // Alternatively, we can elide generation of the dmb instructions
1382   // and plant the alternative CompareAndSwap macro-instruction
1383   // sequence (which uses ldaxr<x>).
1384   //
1385   // Of course, the above only applies when we see these signature
1386   // configurations. We still want to plant dmb instructions in any
1387   // other cases where we may see a MemBarAcquire, MemBarRelease or
1388   // MemBarVolatile. For example, at the end of a constructor which
1389   // writes final/volatile fields we will see a MemBarRelease
1390   // instruction and this needs a 'dmb ish' lest we risk the
1391   // constructed object being visible without making the
1392   // final/volatile field writes visible.
1393   //
1394   // n.b. the translation rules below which rely on detection of the
1395   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1396   // If we see anything other than the signature configurations we
1397   // always just translate the loads and stores to ldr<x> and str<x>
1398   // and translate acquire, release and volatile membars to the
1399   // relevant dmb instructions.
1400   //
1401 
1402   // is_CAS(int opcode, bool maybe_volatile)
1403   //
1404   // return true if opcode is one of the possible CompareAndSwapX
1405   // values otherwise false.
1406 
1407   bool is_CAS(int opcode, bool maybe_volatile)
1408   {
1409     switch(opcode) {
1410       // We handle these
1411     case Op_CompareAndSwapI:
1412     case Op_CompareAndSwapL:
1413     case Op_CompareAndSwapP:
1414     case Op_CompareAndSwapN:
1415     case Op_ShenandoahCompareAndSwapP:
1416     case Op_ShenandoahCompareAndSwapN:
1417     case Op_CompareAndSwapB:
1418     case Op_CompareAndSwapS:
1419     case Op_GetAndSetI:
1420     case Op_GetAndSetL:
1421     case Op_GetAndSetP:
1422     case Op_GetAndSetN:
1423     case Op_GetAndAddI:
1424     case Op_GetAndAddL:
1425       return true;
1426     case Op_CompareAndExchangeI:
1427     case Op_CompareAndExchangeN:
1428     case Op_CompareAndExchangeB:
1429     case Op_CompareAndExchangeS:
1430     case Op_CompareAndExchangeL:
1431     case Op_CompareAndExchangeP:
1432     case Op_WeakCompareAndSwapB:
1433     case Op_WeakCompareAndSwapS:
1434     case Op_WeakCompareAndSwapI:
1435     case Op_WeakCompareAndSwapL:
1436     case Op_WeakCompareAndSwapP:
1437     case Op_WeakCompareAndSwapN:
1438     case Op_ShenandoahWeakCompareAndSwapP:
1439     case Op_ShenandoahWeakCompareAndSwapN:
1440     case Op_ShenandoahCompareAndExchangeP:
1441     case Op_ShenandoahCompareAndExchangeN:
1442       return maybe_volatile;
1443     default:
1444       return false;
1445     }
1446   }
1447 
1448   // helper to determine the maximum number of Phi nodes we may need to
1449   // traverse when searching from a card mark membar for the merge mem
1450   // feeding a trailing membar or vice versa
1451 
1452 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1453 
1454 bool unnecessary_acquire(const Node *barrier)
1455 {
1456   assert(barrier->is_MemBar(), "expecting a membar");
1457 
1458   if (UseBarriersForVolatile) {
1459     // we need to plant a dmb
1460     return false;
1461   }
1462 
1463   MemBarNode* mb = barrier->as_MemBar();
1464 
1465   if (mb->trailing_load()) {
1466     return true;
1467   }
1468 
1469   if (mb->trailing_load_store()) {
1470     Node* load_store = mb->in(MemBarNode::Precedent);
1471     assert(load_store->is_LoadStore(), "unexpected graph shape");
1472     return is_CAS(load_store->Opcode(), true);
1473   }
1474 
1475   return false;
1476 }
1477 
1478 bool needs_acquiring_load(const Node *n)
1479 {
1480   assert(n->is_Load(), "expecting a load");
1481   if (UseBarriersForVolatile) {
1482     // we use a normal load and a dmb
1483     return false;
1484   }
1485 
1486   LoadNode *ld = n->as_Load();
1487 
1488   return ld->is_acquire();
1489 }
1490 
1491 bool unnecessary_release(const Node *n)
1492 {
1493   assert((n->is_MemBar() &&
1494           n->Opcode() == Op_MemBarRelease),
1495          "expecting a release membar");
1496 
1497   if (UseBarriersForVolatile) {
1498     // we need to plant a dmb
1499     return false;
1500   }
1501 
1502   MemBarNode *barrier = n->as_MemBar();
1503   if (!barrier->leading()) {
1504     return false;
1505   } else {
1506     Node* trailing = barrier->trailing_membar();
1507     MemBarNode* trailing_mb = trailing->as_MemBar();
1508     assert(trailing_mb->trailing(), "Not a trailing membar?");
1509     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1510 
1511     Node* mem = trailing_mb->in(MemBarNode::Precedent);
1512     if (mem->is_Store()) {
1513       assert(mem->as_Store()->is_release(), "");
1514       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1515       return true;
1516     } else {
1517       assert(mem->is_LoadStore(), "");
1518       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1519       return is_CAS(mem->Opcode(), true);
1520     }
1521   }
1522   return false;
1523 }
1524 
1525 bool unnecessary_volatile(const Node *n)
1526 {
1527   // assert n->is_MemBar();
1528   if (UseBarriersForVolatile) {
1529     // we need to plant a dmb
1530     return false;
1531   }
1532 
1533   MemBarNode *mbvol = n->as_MemBar();
1534 
1535   bool release = mbvol->trailing_store();
1536   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1537 #ifdef ASSERT
1538   if (release) {
1539     Node* leading = mbvol->leading_membar();
1540     assert(leading->Opcode() == Op_MemBarRelease, "");
1541     assert(leading->as_MemBar()->leading_store(), "");
1542     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1543   }
1544 #endif
1545 
1546   return release;
1547 }
1548 
1549 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1550 
1551 bool needs_releasing_store(const Node *n)
1552 {
1553   // assert n->is_Store();
1554   if (UseBarriersForVolatile) {
1555     // we use a normal store and dmb combination
1556     return false;
1557   }
1558 
1559   StoreNode *st = n->as_Store();
1560 
1561   return st->trailing_membar() != NULL;
1562 }
1563 
1564 // predicate controlling translation of CAS
1565 //
1566 // returns true if CAS needs to use an acquiring load otherwise false
1567 
1568 bool needs_acquiring_load_exclusive(const Node *n)
1569 {
1570   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1571   if (UseBarriersForVolatile) {
1572     return false;
1573   }
1574 
1575   LoadStoreNode* ldst = n->as_LoadStore();
1576   if (is_CAS(n->Opcode(), false)) {
1577     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
1578   } else {
1579     return ldst->trailing_membar() != NULL;
1580   }
1581 
1582   // so we can just return true here
1583   return true;
1584 }
1585 
1586 // predicate controlling translation of StoreCM
1587 //
1588 // returns true if a StoreStore must precede the card write otherwise
1589 // false
1590 
1591 bool unnecessary_storestore(const Node *storecm)
1592 {
1593   assert(storecm->Opcode()  == Op_StoreCM, "expecting a StoreCM");
1594 
1595   // we need to generate a dmb ishst between an object put and the
1596   // associated card mark when we are using CMS without conditional
1597   // card marking
1598 
1599   if (UseConcMarkSweepGC && !UseCondCardMark) {
1600     return false;
1601   }
1602 
1603   // a storestore is unnecesary in all other cases
1604 
1605   return true;
1606 }
1607 
1608 
1609 #define __ _masm.
1610 
1611 // advance declarations for helper functions to convert register
1612 // indices to register objects
1613 
1614 // the ad file has to provide implementations of certain methods
1615 // expected by the generic code
1616 //
1617 // REQUIRED FUNCTIONALITY
1618 
1619 //=============================================================================
1620 
1621 // !!!!! Special hack to get all types of calls to specify the byte offset
1622 //       from the start of the call to the point where the return address
1623 //       will point.
1624 
1625 int MachCallStaticJavaNode::ret_addr_offset()
1626 {
1627   // call should be a simple bl
1628   int off = 4;
1629   return off;
1630 }
1631 
1632 int MachCallDynamicJavaNode::ret_addr_offset()
1633 {
1634   return 16; // movz, movk, movk, bl
1635 }
1636 
1637 int MachCallRuntimeNode::ret_addr_offset() {
1638   // for generated stubs the call will be
1639   //   far_call(addr)
1640   // for real runtime callouts it will be six instructions
1641   // see aarch64_enc_java_to_runtime
1642   //   adr(rscratch2, retaddr)
1643   //   lea(rscratch1, RuntimeAddress(addr)
1644   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1645   //   blrt rscratch1
1646   CodeBlob *cb = CodeCache::find_blob(_entry_point);
1647   if (cb) {
1648     return MacroAssembler::far_branch_size();
1649   } else {
1650     return 6 * NativeInstruction::instruction_size;
1651   }
1652 }
1653 
1654 // Indicate if the safepoint node needs the polling page as an input
1655 
1656 // the shared code plants the oop data at the start of the generated
1657 // code for the safepoint node and that needs ot be at the load
1658 // instruction itself. so we cannot plant a mov of the safepoint poll
1659 // address followed by a load. setting this to true means the mov is
1660 // scheduled as a prior instruction. that's better for scheduling
1661 // anyway.
1662 
1663 bool SafePointNode::needs_polling_address_input()
1664 {
1665   return true;
1666 }
1667 
1668 //=============================================================================
1669 
1670 #ifndef PRODUCT
1671 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1672   st->print("BREAKPOINT");
1673 }
1674 #endif
1675 
1676 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1677   MacroAssembler _masm(&cbuf);
1678   __ brk(0);
1679 }
1680 
1681 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1682   return MachNode::size(ra_);
1683 }
1684 
1685 //=============================================================================
1686 
1687 #ifndef PRODUCT
1688   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1689     st->print("nop \t# %d bytes pad for loops and calls", _count);
1690   }
1691 #endif
1692 
1693   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1694     MacroAssembler _masm(&cbuf);
1695     for (int i = 0; i < _count; i++) {
1696       __ nop();
1697     }
1698   }
1699 
1700   uint MachNopNode::size(PhaseRegAlloc*) const {
1701     return _count * NativeInstruction::instruction_size;
1702   }
1703 
1704 //=============================================================================
1705 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1706 
1707 int Compile::ConstantTable::calculate_table_base_offset() const {
1708   return 0;  // absolute addressing, no offset
1709 }
1710 
1711 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1712 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1713   ShouldNotReachHere();
1714 }
1715 
1716 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1717   // Empty encoding
1718 }
1719 
1720 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1721   return 0;
1722 }
1723 
1724 #ifndef PRODUCT
1725 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1726   st->print("-- \t// MachConstantBaseNode (empty encoding)");
1727 }
1728 #endif
1729 
1730 #ifndef PRODUCT
1731 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1732   Compile* C = ra_->C;
1733 
1734   int framesize = C->frame_slots() << LogBytesPerInt;
1735 
1736   if (C->need_stack_bang(framesize))
1737     st->print("# stack bang size=%d\n\t", framesize);
1738 
1739   if (framesize < ((1 << 9) + 2 * wordSize)) {
1740     st->print("sub  sp, sp, #%d\n\t", framesize);
1741     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1742     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
1743   } else {
1744     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1745     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
1746     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
1747     st->print("sub  sp, sp, rscratch1");
1748   }
1749 }
1750 #endif
1751 
1752 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1753   Compile* C = ra_->C;
1754   MacroAssembler _masm(&cbuf);
1755 
1756   // n.b. frame size includes space for return pc and rfp
1757   const long framesize = C->frame_size_in_bytes();
1758   assert(framesize%(2*wordSize) == 0, "must preserve 2*wordSize alignment");
1759 
1760   // insert a nop at the start of the prolog so we can patch in a
1761   // branch if we need to invalidate the method later
1762   __ nop();
1763 
1764   int bangsize = C->bang_size_in_bytes();
1765   if (C->need_stack_bang(bangsize) && UseStackBanging)
1766     __ generate_stack_overflow_check(bangsize);
1767 
1768   __ build_frame(framesize);
1769 
1770   if (NotifySimulator) {
1771     __ notify(Assembler::method_entry);
1772   }
1773 
1774   if (VerifyStackAtCalls) {
1775     Unimplemented();
1776   }
1777 
1778   C->set_frame_complete(cbuf.insts_size());
1779 
1780   if (C->has_mach_constant_base_node()) {
1781     // NOTE: We set the table base offset here because users might be
1782     // emitted before MachConstantBaseNode.
1783     Compile::ConstantTable& constant_table = C->constant_table();
1784     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1785   }
1786 }
1787 
1788 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1789 {
1790   return MachNode::size(ra_); // too many variables; just compute it
1791                               // the hard way
1792 }
1793 
1794 int MachPrologNode::reloc() const
1795 {
1796   return 0;
1797 }
1798 
1799 //=============================================================================
1800 
1801 #ifndef PRODUCT
1802 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1803   Compile* C = ra_->C;
1804   int framesize = C->frame_slots() << LogBytesPerInt;
1805 
1806   st->print("# pop frame %d\n\t",framesize);
1807 
1808   if (framesize == 0) {
1809     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1810   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1811     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1812     st->print("add  sp, sp, #%d\n\t", framesize);
1813   } else {
1814     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
1815     st->print("add  sp, sp, rscratch1\n\t");
1816     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1817   }
1818 
1819   if (do_polling() && C->is_method_compilation()) {
1820     st->print("# touch polling page\n\t");
1821     st->print("mov  rscratch1, #0x%lx\n\t", p2i(os::get_polling_page()));
1822     st->print("ldr zr, [rscratch1]");
1823   }
1824 }
1825 #endif
1826 
1827 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1828   Compile* C = ra_->C;
1829   MacroAssembler _masm(&cbuf);
1830   int framesize = C->frame_slots() << LogBytesPerInt;
1831 
1832   __ remove_frame(framesize);
1833 
1834   if (NotifySimulator) {
1835     __ notify(Assembler::method_reentry);
1836   }
1837 
1838   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1839     __ reserved_stack_check();
1840   }
1841 
1842   if (do_polling() && C->is_method_compilation()) {
1843     __ read_polling_page(rscratch1, os::get_polling_page(), relocInfo::poll_return_type);
1844   }
1845 }
1846 
1847 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1848   // Variable size. Determine dynamically.
1849   return MachNode::size(ra_);
1850 }
1851 
1852 int MachEpilogNode::reloc() const {
1853   // Return number of relocatable values contained in this instruction.
1854   return 1; // 1 for polling page.
1855 }
1856 
1857 const Pipeline * MachEpilogNode::pipeline() const {
1858   return MachNode::pipeline_class();
1859 }
1860 
1861 // This method seems to be obsolete. It is declared in machnode.hpp
1862 // and defined in all *.ad files, but it is never called. Should we
1863 // get rid of it?
1864 int MachEpilogNode::safepoint_offset() const {
1865   assert(do_polling(), "no return for this epilog node");
1866   return 4;
1867 }
1868 
1869 //=============================================================================
1870 
1871 // Figure out which register class each belongs in: rc_int, rc_float or
1872 // rc_stack.
1873 enum RC { rc_bad, rc_int, rc_float, rc_stack };
1874 
1875 static enum RC rc_class(OptoReg::Name reg) {
1876 
1877   if (reg == OptoReg::Bad) {
1878     return rc_bad;
1879   }
1880 
1881   // we have 30 int registers * 2 halves
1882   // (rscratch1 and rscratch2 are omitted)
1883 
1884   if (reg < 60) {
1885     return rc_int;
1886   }
1887 
1888   // we have 32 float register * 2 halves
1889   if (reg < 60 + 128) {
1890     return rc_float;
1891   }
1892 
1893   // Between float regs & stack is the flags regs.
1894   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1895 
1896   return rc_stack;
1897 }
1898 
1899 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1900   Compile* C = ra_->C;
1901 
1902   // Get registers to move.
1903   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1904   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1905   OptoReg::Name dst_hi = ra_->get_reg_second(this);
1906   OptoReg::Name dst_lo = ra_->get_reg_first(this);
1907 
1908   enum RC src_hi_rc = rc_class(src_hi);
1909   enum RC src_lo_rc = rc_class(src_lo);
1910   enum RC dst_hi_rc = rc_class(dst_hi);
1911   enum RC dst_lo_rc = rc_class(dst_lo);
1912 
1913   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1914 
1915   if (src_hi != OptoReg::Bad) {
1916     assert((src_lo&1)==0 && src_lo+1==src_hi &&
1917            (dst_lo&1)==0 && dst_lo+1==dst_hi,
1918            "expected aligned-adjacent pairs");
1919   }
1920 
1921   if (src_lo == dst_lo && src_hi == dst_hi) {
1922     return 0;            // Self copy, no move.
1923   }
1924 
1925   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1926               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1927   int src_offset = ra_->reg2offset(src_lo);
1928   int dst_offset = ra_->reg2offset(dst_lo);
1929 
1930   if (bottom_type()->isa_vect() != NULL) {
1931     uint ireg = ideal_reg();
1932     assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1933     if (cbuf) {
1934       MacroAssembler _masm(cbuf);
1935       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1936       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1937         // stack->stack
1938         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1939         if (ireg == Op_VecD) {
1940           __ unspill(rscratch1, true, src_offset);
1941           __ spill(rscratch1, true, dst_offset);
1942         } else {
1943           __ spill_copy128(src_offset, dst_offset);
1944         }
1945       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1946         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1947                ireg == Op_VecD ? __ T8B : __ T16B,
1948                as_FloatRegister(Matcher::_regEncode[src_lo]));
1949       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1950         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1951                        ireg == Op_VecD ? __ D : __ Q,
1952                        ra_->reg2offset(dst_lo));
1953       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1954         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1955                        ireg == Op_VecD ? __ D : __ Q,
1956                        ra_->reg2offset(src_lo));
1957       } else {
1958         ShouldNotReachHere();
1959       }
1960     }
1961   } else if (cbuf) {
1962     MacroAssembler _masm(cbuf);
1963     switch (src_lo_rc) {
1964     case rc_int:
1965       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
1966         if (is64) {
1967             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
1968                    as_Register(Matcher::_regEncode[src_lo]));
1969         } else {
1970             MacroAssembler _masm(cbuf);
1971             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
1972                     as_Register(Matcher::_regEncode[src_lo]));
1973         }
1974       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
1975         if (is64) {
1976             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1977                      as_Register(Matcher::_regEncode[src_lo]));
1978         } else {
1979             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1980                      as_Register(Matcher::_regEncode[src_lo]));
1981         }
1982       } else {                    // gpr --> stack spill
1983         assert(dst_lo_rc == rc_stack, "spill to bad register class");
1984         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
1985       }
1986       break;
1987     case rc_float:
1988       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
1989         if (is64) {
1990             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
1991                      as_FloatRegister(Matcher::_regEncode[src_lo]));
1992         } else {
1993             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
1994                      as_FloatRegister(Matcher::_regEncode[src_lo]));
1995         }
1996       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
1997           if (cbuf) {
1998             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1999                      as_FloatRegister(Matcher::_regEncode[src_lo]));
2000         } else {
2001             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2002                      as_FloatRegister(Matcher::_regEncode[src_lo]));
2003         }
2004       } else {                    // fpr --> stack spill
2005         assert(dst_lo_rc == rc_stack, "spill to bad register class");
2006         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2007                  is64 ? __ D : __ S, dst_offset);
2008       }
2009       break;
2010     case rc_stack:
2011       if (dst_lo_rc == rc_int) {  // stack --> gpr load
2012         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2013       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2014         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2015                    is64 ? __ D : __ S, src_offset);
2016       } else {                    // stack --> stack copy
2017         assert(dst_lo_rc == rc_stack, "spill to bad register class");
2018         __ unspill(rscratch1, is64, src_offset);
2019         __ spill(rscratch1, is64, dst_offset);
2020       }
2021       break;
2022     default:
2023       assert(false, "bad rc_class for spill");
2024       ShouldNotReachHere();
2025     }
2026   }
2027 
2028   if (st) {
2029     st->print("spill ");
2030     if (src_lo_rc == rc_stack) {
2031       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2032     } else {
2033       st->print("%s -> ", Matcher::regName[src_lo]);
2034     }
2035     if (dst_lo_rc == rc_stack) {
2036       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2037     } else {
2038       st->print("%s", Matcher::regName[dst_lo]);
2039     }
2040     if (bottom_type()->isa_vect() != NULL) {
2041       st->print("\t# vector spill size = %d", ideal_reg()==Op_VecD ? 64:128);
2042     } else {
2043       st->print("\t# spill size = %d", is64 ? 64:32);
2044     }
2045   }
2046 
2047   return 0;
2048 
2049 }
2050 
2051 #ifndef PRODUCT
2052 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2053   if (!ra_)
2054     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2055   else
2056     implementation(NULL, ra_, false, st);
2057 }
2058 #endif
2059 
2060 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2061   implementation(&cbuf, ra_, false, NULL);
2062 }
2063 
2064 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2065   return MachNode::size(ra_);
2066 }
2067 
2068 //=============================================================================
2069 
2070 #ifndef PRODUCT
2071 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2072   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2073   int reg = ra_->get_reg_first(this);
2074   st->print("add %s, rsp, #%d]\t# box lock",
2075             Matcher::regName[reg], offset);
2076 }
2077 #endif
2078 
2079 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2080   MacroAssembler _masm(&cbuf);
2081 
2082   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2083   int reg    = ra_->get_encode(this);
2084 
2085   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2086     __ add(as_Register(reg), sp, offset);
2087   } else {
2088     ShouldNotReachHere();
2089   }
2090 }
2091 
2092 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2093   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2094   return 4;
2095 }
2096 
2097 //=============================================================================
2098 
2099 #ifndef PRODUCT
2100 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2101 {
2102   st->print_cr("# MachUEPNode");
2103   if (UseCompressedClassPointers) {
2104     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2105     if (CompressedKlassPointers::shift() != 0) {
2106       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
2107     }
2108   } else {
2109    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2110   }
2111   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
2112   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2113 }
2114 #endif
2115 
2116 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
2117 {
2118   // This is the unverified entry point.
2119   MacroAssembler _masm(&cbuf);
2120 
2121   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2122   Label skip;
2123   // TODO
2124   // can we avoid this skip and still use a reloc?
2125   __ br(Assembler::EQ, skip);
2126   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2127   __ bind(skip);
2128 }
2129 
2130 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2131 {
2132   return MachNode::size(ra_);
2133 }
2134 
2135 // REQUIRED EMIT CODE
2136 
2137 //=============================================================================
2138 
2139 // Emit exception handler code.
2140 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2141 {
2142   // mov rscratch1 #exception_blob_entry_point
2143   // br rscratch1
2144   // Note that the code buffer's insts_mark is always relative to insts.
2145   // That's why we must use the macroassembler to generate a handler.
2146   MacroAssembler _masm(&cbuf);
2147   address base = __ start_a_stub(size_exception_handler());
2148   if (base == NULL) {
2149     ciEnv::current()->record_failure("CodeCache is full");
2150     return 0;  // CodeBuffer::expand failed
2151   }
2152   int offset = __ offset();
2153   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2154   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2155   __ end_a_stub();
2156   return offset;
2157 }
2158 
2159 // Emit deopt handler code.
2160 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2161 {
2162   // Note that the code buffer's insts_mark is always relative to insts.
2163   // That's why we must use the macroassembler to generate a handler.
2164   MacroAssembler _masm(&cbuf);
2165   address base = __ start_a_stub(size_deopt_handler());
2166   if (base == NULL) {
2167     ciEnv::current()->record_failure("CodeCache is full");
2168     return 0;  // CodeBuffer::expand failed
2169   }
2170   int offset = __ offset();
2171 
2172   __ adr(lr, __ pc());
2173   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2174 
2175   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
2176   __ end_a_stub();
2177   return offset;
2178 }
2179 
2180 // REQUIRED MATCHER CODE
2181 
2182 //=============================================================================
2183 
2184 const bool Matcher::match_rule_supported(int opcode) {
2185 
2186   switch (opcode) {
2187   default:
2188     break;
2189   }
2190 
2191   if (!has_match_rule(opcode)) {
2192     return false;
2193   }
2194 
2195   return true;  // Per default match rules are supported.
2196 }
2197 
2198 const bool Matcher::match_rule_supported_vector(int opcode, int vlen) {
2199 
2200   // TODO
2201   // identify extra cases that we might want to provide match rules for
2202   // e.g. Op_ vector nodes and other intrinsics while guarding with vlen
2203   bool ret_value = match_rule_supported(opcode);
2204   // Add rules here.
2205 
2206   return ret_value;  // Per default match rules are supported.
2207 }
2208 
2209 const bool Matcher::has_predicated_vectors(void) {
2210   return false;
2211 }
2212 
2213 const int Matcher::float_pressure(int default_pressure_threshold) {
2214   return default_pressure_threshold;
2215 }
2216 
2217 int Matcher::regnum_to_fpu_offset(int regnum)
2218 {
2219   Unimplemented();
2220   return 0;
2221 }
2222 
2223 // Is this branch offset short enough that a short branch can be used?
2224 //
2225 // NOTE: If the platform does not provide any short branch variants, then
2226 //       this method should return false for offset 0.
2227 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2228   // The passed offset is relative to address of the branch.
2229 
2230   return (-32768 <= offset && offset < 32768);
2231 }
2232 
2233 const bool Matcher::isSimpleConstant64(jlong value) {
2234   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
2235   // Probably always true, even if a temp register is required.
2236   return true;
2237 }
2238 
2239 // true just means we have fast l2f conversion
2240 const bool Matcher::convL2FSupported(void) {
2241   return true;
2242 }
2243 
2244 // Vector width in bytes.
2245 const int Matcher::vector_width_in_bytes(BasicType bt) {
2246   int size = MIN2(16,(int)MaxVectorSize);
2247   // Minimum 2 values in vector
2248   if (size < 2*type2aelembytes(bt)) size = 0;
2249   // But never < 4
2250   if (size < 4) size = 0;
2251   return size;
2252 }
2253 
2254 // Limits on vector size (number of elements) loaded into vector.
2255 const int Matcher::max_vector_size(const BasicType bt) {
2256   return vector_width_in_bytes(bt)/type2aelembytes(bt);
2257 }
2258 const int Matcher::min_vector_size(const BasicType bt) {
2259 //  For the moment limit the vector size to 8 bytes
2260     int size = 8 / type2aelembytes(bt);
2261     if (size < 2) size = 2;
2262     return size;
2263 }
2264 
2265 // Vector ideal reg.
2266 const uint Matcher::vector_ideal_reg(int len) {
2267   switch(len) {
2268     case  8: return Op_VecD;
2269     case 16: return Op_VecX;
2270   }
2271   ShouldNotReachHere();
2272   return 0;
2273 }
2274 
2275 const uint Matcher::vector_shift_count_ideal_reg(int size) {
2276   switch(size) {
2277     case  8: return Op_VecD;
2278     case 16: return Op_VecX;
2279   }
2280   ShouldNotReachHere();
2281   return 0;
2282 }
2283 
2284 // AES support not yet implemented
2285 const bool Matcher::pass_original_key_for_aes() {
2286   return false;
2287 }
2288 
2289 // aarch64 supports misaligned vectors store/load.
2290 const bool Matcher::misaligned_vectors_ok() {
2291   return true;
2292 }
2293 
2294 // false => size gets scaled to BytesPerLong, ok.
2295 const bool Matcher::init_array_count_is_in_bytes = false;
2296 
2297 // Use conditional move (CMOVL)
2298 const int Matcher::long_cmove_cost() {
2299   // long cmoves are no more expensive than int cmoves
2300   return 0;
2301 }
2302 
2303 const int Matcher::float_cmove_cost() {
2304   // float cmoves are no more expensive than int cmoves
2305   return 0;
2306 }
2307 
2308 // Does the CPU require late expand (see block.cpp for description of late expand)?
2309 const bool Matcher::require_postalloc_expand = false;
2310 
2311 // Do we need to mask the count passed to shift instructions or does
2312 // the cpu only look at the lower 5/6 bits anyway?
2313 const bool Matcher::need_masked_shift_count = false;
2314 
2315 // This affects two different things:
2316 //  - how Decode nodes are matched
2317 //  - how ImplicitNullCheck opportunities are recognized
2318 // If true, the matcher will try to remove all Decodes and match them
2319 // (as operands) into nodes. NullChecks are not prepared to deal with
2320 // Decodes by final_graph_reshaping().
2321 // If false, final_graph_reshaping() forces the decode behind the Cmp
2322 // for a NullCheck. The matcher matches the Decode node into a register.
2323 // Implicit_null_check optimization moves the Decode along with the
2324 // memory operation back up before the NullCheck.
2325 bool Matcher::narrow_oop_use_complex_address() {
2326   return CompressedOops::shift() == 0;
2327 }
2328 
2329 bool Matcher::narrow_klass_use_complex_address() {
2330 // TODO
2331 // decide whether we need to set this to true
2332   return false;
2333 }
2334 
2335 bool Matcher::const_oop_prefer_decode() {
2336   // Prefer ConN+DecodeN over ConP in simple compressed oops mode.
2337   return CompressedOops::base() == NULL;
2338 }
2339 
2340 bool Matcher::const_klass_prefer_decode() {
2341   // Prefer ConNKlass+DecodeNKlass over ConP in simple compressed klass mode.
2342   return CompressedKlassPointers::base() == NULL;
2343 }
2344 
2345 // Is it better to copy float constants, or load them directly from
2346 // memory?  Intel can load a float constant from a direct address,
2347 // requiring no extra registers.  Most RISCs will have to materialize
2348 // an address into a register first, so they would do better to copy
2349 // the constant from stack.
2350 const bool Matcher::rematerialize_float_constants = false;
2351 
2352 // If CPU can load and store mis-aligned doubles directly then no
2353 // fixup is needed.  Else we split the double into 2 integer pieces
2354 // and move it piece-by-piece.  Only happens when passing doubles into
2355 // C code as the Java calling convention forces doubles to be aligned.
2356 const bool Matcher::misaligned_doubles_ok = true;
2357 
2358 // No-op on amd64
2359 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {
2360   Unimplemented();
2361 }
2362 
2363 // Advertise here if the CPU requires explicit rounding operations to
2364 // implement the UseStrictFP mode.
2365 const bool Matcher::strict_fp_requires_explicit_rounding = false;
2366 
2367 // Are floats converted to double when stored to stack during
2368 // deoptimization?
2369 bool Matcher::float_in_double() { return false; }
2370 
2371 // Do ints take an entire long register or just half?
2372 // The relevant question is how the int is callee-saved:
2373 // the whole long is written but de-opt'ing will have to extract
2374 // the relevant 32 bits.
2375 const bool Matcher::int_in_long = true;
2376 
2377 // Return whether or not this register is ever used as an argument.
2378 // This function is used on startup to build the trampoline stubs in
2379 // generateOptoStub.  Registers not mentioned will be killed by the VM
2380 // call in the trampoline, and arguments in those registers not be
2381 // available to the callee.
2382 bool Matcher::can_be_java_arg(int reg)
2383 {
2384   return
2385     reg ==  R0_num || reg == R0_H_num ||
2386     reg ==  R1_num || reg == R1_H_num ||
2387     reg ==  R2_num || reg == R2_H_num ||
2388     reg ==  R3_num || reg == R3_H_num ||
2389     reg ==  R4_num || reg == R4_H_num ||
2390     reg ==  R5_num || reg == R5_H_num ||
2391     reg ==  R6_num || reg == R6_H_num ||
2392     reg ==  R7_num || reg == R7_H_num ||
2393     reg ==  V0_num || reg == V0_H_num ||
2394     reg ==  V1_num || reg == V1_H_num ||
2395     reg ==  V2_num || reg == V2_H_num ||
2396     reg ==  V3_num || reg == V3_H_num ||
2397     reg ==  V4_num || reg == V4_H_num ||
2398     reg ==  V5_num || reg == V5_H_num ||
2399     reg ==  V6_num || reg == V6_H_num ||
2400     reg ==  V7_num || reg == V7_H_num;
2401 }
2402 
2403 bool Matcher::is_spillable_arg(int reg)
2404 {
2405   return can_be_java_arg(reg);
2406 }
2407 
2408 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2409   return false;
2410 }
2411 
2412 RegMask Matcher::divI_proj_mask() {
2413   ShouldNotReachHere();
2414   return RegMask();
2415 }
2416 
2417 // Register for MODI projection of divmodI.
2418 RegMask Matcher::modI_proj_mask() {
2419   ShouldNotReachHere();
2420   return RegMask();
2421 }
2422 
2423 // Register for DIVL projection of divmodL.
2424 RegMask Matcher::divL_proj_mask() {
2425   ShouldNotReachHere();
2426   return RegMask();
2427 }
2428 
2429 // Register for MODL projection of divmodL.
2430 RegMask Matcher::modL_proj_mask() {
2431   ShouldNotReachHere();
2432   return RegMask();
2433 }
2434 
2435 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2436   return FP_REG_mask();
2437 }
2438 
2439 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2440   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2441     Node* u = addp->fast_out(i);
2442     if (u->is_Mem()) {
2443       int opsize = u->as_Mem()->memory_size();
2444       assert(opsize > 0, "unexpected memory operand size");
2445       if (u->as_Mem()->memory_size() != (1<<shift)) {
2446         return false;
2447       }
2448     }
2449   }
2450   return true;
2451 }
2452 
2453 const bool Matcher::convi2l_type_required = false;
2454 
2455 // Should the Matcher clone shifts on addressing modes, expecting them
2456 // to be subsumed into complex addressing expressions or compute them
2457 // into registers?
2458 bool Matcher::clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2459   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2460     return true;
2461   }
2462 
2463   Node *off = m->in(AddPNode::Offset);
2464   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2465       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2466       // Are there other uses besides address expressions?
2467       !is_visited(off)) {
2468     address_visited.set(off->_idx); // Flag as address_visited
2469     mstack.push(off->in(2), Visit);
2470     Node *conv = off->in(1);
2471     if (conv->Opcode() == Op_ConvI2L &&
2472         // Are there other uses besides address expressions?
2473         !is_visited(conv)) {
2474       address_visited.set(conv->_idx); // Flag as address_visited
2475       mstack.push(conv->in(1), Pre_Visit);
2476     } else {
2477       mstack.push(conv, Pre_Visit);
2478     }
2479     address_visited.test_set(m->_idx); // Flag as address_visited
2480     mstack.push(m->in(AddPNode::Address), Pre_Visit);
2481     mstack.push(m->in(AddPNode::Base), Pre_Visit);
2482     return true;
2483   } else if (off->Opcode() == Op_ConvI2L &&
2484              // Are there other uses besides address expressions?
2485              !is_visited(off)) {
2486     address_visited.test_set(m->_idx); // Flag as address_visited
2487     address_visited.set(off->_idx); // Flag as address_visited
2488     mstack.push(off->in(1), Pre_Visit);
2489     mstack.push(m->in(AddPNode::Address), Pre_Visit);
2490     mstack.push(m->in(AddPNode::Base), Pre_Visit);
2491     return true;
2492   }
2493   return false;
2494 }
2495 
2496 void Compile::reshape_address(AddPNode* addp) {
2497 }
2498 
2499 // helper for encoding java_to_runtime calls on sim
2500 //
2501 // this is needed to compute the extra arguments required when
2502 // planting a call to the simulator blrt instruction. the TypeFunc
2503 // can be queried to identify the counts for integral, and floating
2504 // arguments and the return type
2505 
2506 static void getCallInfo(const TypeFunc *tf, int &gpcnt, int &fpcnt, int &rtype)
2507 {
2508   int gps = 0;
2509   int fps = 0;
2510   const TypeTuple *domain = tf->domain();
2511   int max = domain->cnt();
2512   for (int i = TypeFunc::Parms; i < max; i++) {
2513     const Type *t = domain->field_at(i);
2514     switch(t->basic_type()) {
2515     case T_FLOAT:
2516     case T_DOUBLE:
2517       fps++;
2518     default:
2519       gps++;
2520     }
2521   }
2522   gpcnt = gps;
2523   fpcnt = fps;
2524   BasicType rt = tf->return_type();
2525   switch (rt) {
2526   case T_VOID:
2527     rtype = MacroAssembler::ret_type_void;
2528     break;
2529   default:
2530     rtype = MacroAssembler::ret_type_integral;
2531     break;
2532   case T_FLOAT:
2533     rtype = MacroAssembler::ret_type_float;
2534     break;
2535   case T_DOUBLE:
2536     rtype = MacroAssembler::ret_type_double;
2537     break;
2538   }
2539 }
2540 
2541 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
2542   MacroAssembler _masm(&cbuf);                                          \
2543   {                                                                     \
2544     guarantee(INDEX == -1, "mode not permitted for volatile");          \
2545     guarantee(DISP == 0, "mode not permitted for volatile");            \
2546     guarantee(SCALE == 0, "mode not permitted for volatile");           \
2547     __ INSN(REG, as_Register(BASE));                                    \
2548   }
2549 
2550 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2551 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2552 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2553                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
2554 
2555   // Used for all non-volatile memory accesses.  The use of
2556   // $mem->opcode() to discover whether this pattern uses sign-extended
2557   // offsets is something of a kludge.
2558   static void loadStore(MacroAssembler masm, mem_insn insn,
2559                          Register reg, int opcode,
2560                          Register base, int index, int size, int disp)
2561   {
2562     Address::extend scale;
2563 
2564     // Hooboy, this is fugly.  We need a way to communicate to the
2565     // encoder that the index needs to be sign extended, so we have to
2566     // enumerate all the cases.
2567     switch (opcode) {
2568     case INDINDEXSCALEDI2L:
2569     case INDINDEXSCALEDI2LN:
2570     case INDINDEXI2L:
2571     case INDINDEXI2LN:
2572       scale = Address::sxtw(size);
2573       break;
2574     default:
2575       scale = Address::lsl(size);
2576     }
2577 
2578     if (index == -1) {
2579       (masm.*insn)(reg, Address(base, disp));
2580     } else {
2581       assert(disp == 0, "unsupported address mode: disp = %d", disp);
2582       (masm.*insn)(reg, Address(base, as_Register(index), scale));
2583     }
2584   }
2585 
2586   static void loadStore(MacroAssembler masm, mem_float_insn insn,
2587                          FloatRegister reg, int opcode,
2588                          Register base, int index, int size, int disp)
2589   {
2590     Address::extend scale;
2591 
2592     switch (opcode) {
2593     case INDINDEXSCALEDI2L:
2594     case INDINDEXSCALEDI2LN:
2595       scale = Address::sxtw(size);
2596       break;
2597     default:
2598       scale = Address::lsl(size);
2599     }
2600 
2601      if (index == -1) {
2602       (masm.*insn)(reg, Address(base, disp));
2603     } else {
2604       assert(disp == 0, "unsupported address mode: disp = %d", disp);
2605       (masm.*insn)(reg, Address(base, as_Register(index), scale));
2606     }
2607   }
2608 
2609   static void loadStore(MacroAssembler masm, mem_vector_insn insn,
2610                          FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2611                          int opcode, Register base, int index, int size, int disp)
2612   {
2613     if (index == -1) {
2614       (masm.*insn)(reg, T, Address(base, disp));
2615     } else {
2616       assert(disp == 0, "unsupported address mode");
2617       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2618     }
2619   }
2620 
2621 %}
2622 
2623 
2624 
2625 //----------ENCODING BLOCK-----------------------------------------------------
2626 // This block specifies the encoding classes used by the compiler to
2627 // output byte streams.  Encoding classes are parameterized macros
2628 // used by Machine Instruction Nodes in order to generate the bit
2629 // encoding of the instruction.  Operands specify their base encoding
2630 // interface with the interface keyword.  There are currently
2631 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2632 // COND_INTER.  REG_INTER causes an operand to generate a function
2633 // which returns its register number when queried.  CONST_INTER causes
2634 // an operand to generate a function which returns the value of the
2635 // constant when queried.  MEMORY_INTER causes an operand to generate
2636 // four functions which return the Base Register, the Index Register,
2637 // the Scale Value, and the Offset Value of the operand when queried.
2638 // COND_INTER causes an operand to generate six functions which return
2639 // the encoding code (ie - encoding bits for the instruction)
2640 // associated with each basic boolean condition for a conditional
2641 // instruction.
2642 //
2643 // Instructions specify two basic values for encoding.  Again, a
2644 // function is available to check if the constant displacement is an
2645 // oop. They use the ins_encode keyword to specify their encoding
2646 // classes (which must be a sequence of enc_class names, and their
2647 // parameters, specified in the encoding block), and they use the
2648 // opcode keyword to specify, in order, their primary, secondary, and
2649 // tertiary opcode.  Only the opcode sections which a particular
2650 // instruction needs for encoding need to be specified.
2651 encode %{
2652   // Build emit functions for each basic byte or larger field in the
2653   // intel encoding scheme (opcode, rm, sib, immediate), and call them
2654   // from C++ code in the enc_class source block.  Emit functions will
2655   // live in the main source block for now.  In future, we can
2656   // generalize this by adding a syntax that specifies the sizes of
2657   // fields in an order, so that the adlc can build the emit functions
2658   // automagically
2659 
2660   // catch all for unimplemented encodings
2661   enc_class enc_unimplemented %{
2662     MacroAssembler _masm(&cbuf);
2663     __ unimplemented("C2 catch all");
2664   %}
2665 
2666   // BEGIN Non-volatile memory access
2667 
2668   enc_class aarch64_enc_ldrsbw(iRegI dst, memory mem) %{
2669     Register dst_reg = as_Register($dst$$reg);
2670     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2671                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2672   %}
2673 
2674   enc_class aarch64_enc_ldrsb(iRegI dst, memory mem) %{
2675     Register dst_reg = as_Register($dst$$reg);
2676     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2677                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2678   %}
2679 
2680   enc_class aarch64_enc_ldrb(iRegI dst, memory mem) %{
2681     Register dst_reg = as_Register($dst$$reg);
2682     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2683                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2684   %}
2685 
2686   enc_class aarch64_enc_ldrb(iRegL dst, memory mem) %{
2687     Register dst_reg = as_Register($dst$$reg);
2688     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2689                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2690   %}
2691 
2692   enc_class aarch64_enc_ldrshw(iRegI dst, memory mem) %{
2693     Register dst_reg = as_Register($dst$$reg);
2694     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2695                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2696   %}
2697 
2698   enc_class aarch64_enc_ldrsh(iRegI dst, memory mem) %{
2699     Register dst_reg = as_Register($dst$$reg);
2700     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2701                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2702   %}
2703 
2704   enc_class aarch64_enc_ldrh(iRegI dst, memory mem) %{
2705     Register dst_reg = as_Register($dst$$reg);
2706     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2707                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2708   %}
2709 
2710   enc_class aarch64_enc_ldrh(iRegL dst, memory mem) %{
2711     Register dst_reg = as_Register($dst$$reg);
2712     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2713                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2714   %}
2715 
2716   enc_class aarch64_enc_ldrw(iRegI dst, memory mem) %{
2717     Register dst_reg = as_Register($dst$$reg);
2718     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2719                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2720   %}
2721 
2722   enc_class aarch64_enc_ldrw(iRegL dst, memory mem) %{
2723     Register dst_reg = as_Register($dst$$reg);
2724     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2725                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2726   %}
2727 
2728   enc_class aarch64_enc_ldrsw(iRegL dst, memory mem) %{
2729     Register dst_reg = as_Register($dst$$reg);
2730     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2731                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2732   %}
2733 
2734   enc_class aarch64_enc_ldr(iRegL dst, memory mem) %{
2735     Register dst_reg = as_Register($dst$$reg);
2736     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2737                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2738   %}
2739 
2740   enc_class aarch64_enc_ldrs(vRegF dst, memory mem) %{
2741     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2742     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2743                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2744   %}
2745 
2746   enc_class aarch64_enc_ldrd(vRegD dst, memory mem) %{
2747     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2748     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2749                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2750   %}
2751 
2752   enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
2753     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2754     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
2755        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2756   %}
2757 
2758   enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
2759     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2760     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
2761        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2762   %}
2763 
2764   enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
2765     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2766     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
2767        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2768   %}
2769 
2770   enc_class aarch64_enc_strb(iRegI src, memory mem) %{
2771     Register src_reg = as_Register($src$$reg);
2772     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2773                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2774   %}
2775 
2776   enc_class aarch64_enc_strb0(memory mem) %{
2777     MacroAssembler _masm(&cbuf);
2778     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2779                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2780   %}
2781 
2782   enc_class aarch64_enc_strb0_ordered(memory mem) %{
2783     MacroAssembler _masm(&cbuf);
2784     __ membar(Assembler::StoreStore);
2785     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2786                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2787   %}
2788 
2789   enc_class aarch64_enc_strh(iRegI src, memory mem) %{
2790     Register src_reg = as_Register($src$$reg);
2791     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2792                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2793   %}
2794 
2795   enc_class aarch64_enc_strh0(memory mem) %{
2796     MacroAssembler _masm(&cbuf);
2797     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2798                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2799   %}
2800 
2801   enc_class aarch64_enc_strw(iRegI src, memory mem) %{
2802     Register src_reg = as_Register($src$$reg);
2803     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2804                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2805   %}
2806 
2807   enc_class aarch64_enc_strw0(memory mem) %{
2808     MacroAssembler _masm(&cbuf);
2809     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2810                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2811   %}
2812 
2813   enc_class aarch64_enc_str(iRegL src, memory mem) %{
2814     Register src_reg = as_Register($src$$reg);
2815     // we sometimes get asked to store the stack pointer into the
2816     // current thread -- we cannot do that directly on AArch64
2817     if (src_reg == r31_sp) {
2818       MacroAssembler _masm(&cbuf);
2819       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2820       __ mov(rscratch2, sp);
2821       src_reg = rscratch2;
2822     }
2823     loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
2824                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2825   %}
2826 
2827   enc_class aarch64_enc_str0(memory mem) %{
2828     MacroAssembler _masm(&cbuf);
2829     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
2830                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2831   %}
2832 
2833   enc_class aarch64_enc_strs(vRegF src, memory mem) %{
2834     FloatRegister src_reg = as_FloatRegister($src$$reg);
2835     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
2836                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2837   %}
2838 
2839   enc_class aarch64_enc_strd(vRegD src, memory mem) %{
2840     FloatRegister src_reg = as_FloatRegister($src$$reg);
2841     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
2842                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2843   %}
2844 
2845   enc_class aarch64_enc_strvS(vecD src, memory mem) %{
2846     FloatRegister src_reg = as_FloatRegister($src$$reg);
2847     loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
2848        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2849   %}
2850 
2851   enc_class aarch64_enc_strvD(vecD src, memory mem) %{
2852     FloatRegister src_reg = as_FloatRegister($src$$reg);
2853     loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
2854        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2855   %}
2856 
2857   enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
2858     FloatRegister src_reg = as_FloatRegister($src$$reg);
2859     loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
2860        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2861   %}
2862 
2863   // END Non-volatile memory access
2864 
2865   // volatile loads and stores
2866 
2867   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
2868     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2869                  rscratch1, stlrb);
2870   %}
2871 
2872   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
2873     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2874                  rscratch1, stlrh);
2875   %}
2876 
2877   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
2878     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2879                  rscratch1, stlrw);
2880   %}
2881 
2882 
2883   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
2884     Register dst_reg = as_Register($dst$$reg);
2885     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2886              rscratch1, ldarb);
2887     __ sxtbw(dst_reg, dst_reg);
2888   %}
2889 
2890   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
2891     Register dst_reg = as_Register($dst$$reg);
2892     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2893              rscratch1, ldarb);
2894     __ sxtb(dst_reg, dst_reg);
2895   %}
2896 
2897   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
2898     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2899              rscratch1, ldarb);
2900   %}
2901 
2902   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
2903     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2904              rscratch1, ldarb);
2905   %}
2906 
2907   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
2908     Register dst_reg = as_Register($dst$$reg);
2909     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2910              rscratch1, ldarh);
2911     __ sxthw(dst_reg, dst_reg);
2912   %}
2913 
2914   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
2915     Register dst_reg = as_Register($dst$$reg);
2916     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2917              rscratch1, ldarh);
2918     __ sxth(dst_reg, dst_reg);
2919   %}
2920 
2921   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
2922     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2923              rscratch1, ldarh);
2924   %}
2925 
2926   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
2927     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2928              rscratch1, ldarh);
2929   %}
2930 
2931   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
2932     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2933              rscratch1, ldarw);
2934   %}
2935 
2936   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
2937     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2938              rscratch1, ldarw);
2939   %}
2940 
2941   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
2942     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2943              rscratch1, ldar);
2944   %}
2945 
2946   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
2947     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2948              rscratch1, ldarw);
2949     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
2950   %}
2951 
2952   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
2953     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2954              rscratch1, ldar);
2955     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
2956   %}
2957 
2958   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
2959     Register src_reg = as_Register($src$$reg);
2960     // we sometimes get asked to store the stack pointer into the
2961     // current thread -- we cannot do that directly on AArch64
2962     if (src_reg == r31_sp) {
2963         MacroAssembler _masm(&cbuf);
2964       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2965       __ mov(rscratch2, sp);
2966       src_reg = rscratch2;
2967     }
2968     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2969                  rscratch1, stlr);
2970   %}
2971 
2972   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
2973     {
2974       MacroAssembler _masm(&cbuf);
2975       FloatRegister src_reg = as_FloatRegister($src$$reg);
2976       __ fmovs(rscratch2, src_reg);
2977     }
2978     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2979                  rscratch1, stlrw);
2980   %}
2981 
2982   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
2983     {
2984       MacroAssembler _masm(&cbuf);
2985       FloatRegister src_reg = as_FloatRegister($src$$reg);
2986       __ fmovd(rscratch2, src_reg);
2987     }
2988     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2989                  rscratch1, stlr);
2990   %}
2991 
2992   // synchronized read/update encodings
2993 
2994   enc_class aarch64_enc_ldaxr(iRegL dst, memory mem) %{
2995     MacroAssembler _masm(&cbuf);
2996     Register dst_reg = as_Register($dst$$reg);
2997     Register base = as_Register($mem$$base);
2998     int index = $mem$$index;
2999     int scale = $mem$$scale;
3000     int disp = $mem$$disp;
3001     if (index == -1) {
3002        if (disp != 0) {
3003         __ lea(rscratch1, Address(base, disp));
3004         __ ldaxr(dst_reg, rscratch1);
3005       } else {
3006         // TODO
3007         // should we ever get anything other than this case?
3008         __ ldaxr(dst_reg, base);
3009       }
3010     } else {
3011       Register index_reg = as_Register(index);
3012       if (disp == 0) {
3013         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3014         __ ldaxr(dst_reg, rscratch1);
3015       } else {
3016         __ lea(rscratch1, Address(base, disp));
3017         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3018         __ ldaxr(dst_reg, rscratch1);
3019       }
3020     }
3021   %}
3022 
3023   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory mem) %{
3024     MacroAssembler _masm(&cbuf);
3025     Register src_reg = as_Register($src$$reg);
3026     Register base = as_Register($mem$$base);
3027     int index = $mem$$index;
3028     int scale = $mem$$scale;
3029     int disp = $mem$$disp;
3030     if (index == -1) {
3031        if (disp != 0) {
3032         __ lea(rscratch2, Address(base, disp));
3033         __ stlxr(rscratch1, src_reg, rscratch2);
3034       } else {
3035         // TODO
3036         // should we ever get anything other than this case?
3037         __ stlxr(rscratch1, src_reg, base);
3038       }
3039     } else {
3040       Register index_reg = as_Register(index);
3041       if (disp == 0) {
3042         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3043         __ stlxr(rscratch1, src_reg, rscratch2);
3044       } else {
3045         __ lea(rscratch2, Address(base, disp));
3046         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3047         __ stlxr(rscratch1, src_reg, rscratch2);
3048       }
3049     }
3050     __ cmpw(rscratch1, zr);
3051   %}
3052 
3053   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3054     MacroAssembler _masm(&cbuf);
3055     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3056     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3057                Assembler::xword, /*acquire*/ false, /*release*/ true,
3058                /*weak*/ false, noreg);
3059   %}
3060 
3061   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3062     MacroAssembler _masm(&cbuf);
3063     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3064     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3065                Assembler::word, /*acquire*/ false, /*release*/ true,
3066                /*weak*/ false, noreg);
3067   %}
3068 
3069   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3070     MacroAssembler _masm(&cbuf);
3071     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3072     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3073                Assembler::halfword, /*acquire*/ false, /*release*/ true,
3074                /*weak*/ false, noreg);
3075   %}
3076 
3077   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3078     MacroAssembler _masm(&cbuf);
3079     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3080     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3081                Assembler::byte, /*acquire*/ false, /*release*/ true,
3082                /*weak*/ false, noreg);
3083   %}
3084 
3085 
3086   // The only difference between aarch64_enc_cmpxchg and
3087   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3088   // CompareAndSwap sequence to serve as a barrier on acquiring a
3089   // lock.
3090   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3091     MacroAssembler _masm(&cbuf);
3092     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3093     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3094                Assembler::xword, /*acquire*/ true, /*release*/ true,
3095                /*weak*/ false, noreg);
3096   %}
3097 
3098   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3099     MacroAssembler _masm(&cbuf);
3100     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3101     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3102                Assembler::word, /*acquire*/ true, /*release*/ true,
3103                /*weak*/ false, noreg);
3104   %}
3105 
3106   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3107     MacroAssembler _masm(&cbuf);
3108     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3109     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3110                Assembler::halfword, /*acquire*/ true, /*release*/ true,
3111                /*weak*/ false, noreg);
3112   %}
3113 
3114   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3115     MacroAssembler _masm(&cbuf);
3116     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3117     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3118                Assembler::byte, /*acquire*/ true, /*release*/ true,
3119                /*weak*/ false, noreg);
3120   %}
3121 
3122   // auxiliary used for CompareAndSwapX to set result register
3123   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3124     MacroAssembler _masm(&cbuf);
3125     Register res_reg = as_Register($res$$reg);
3126     __ cset(res_reg, Assembler::EQ);
3127   %}
3128 
3129   // prefetch encodings
3130 
3131   enc_class aarch64_enc_prefetchw(memory mem) %{
3132     MacroAssembler _masm(&cbuf);
3133     Register base = as_Register($mem$$base);
3134     int index = $mem$$index;
3135     int scale = $mem$$scale;
3136     int disp = $mem$$disp;
3137     if (index == -1) {
3138       __ prfm(Address(base, disp), PSTL1KEEP);
3139     } else {
3140       Register index_reg = as_Register(index);
3141       if (disp == 0) {
3142         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3143       } else {
3144         __ lea(rscratch1, Address(base, disp));
3145         __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3146       }
3147     }
3148   %}
3149 
3150   /// mov envcodings
3151 
3152   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3153     MacroAssembler _masm(&cbuf);
3154     u_int32_t con = (u_int32_t)$src$$constant;
3155     Register dst_reg = as_Register($dst$$reg);
3156     if (con == 0) {
3157       __ movw(dst_reg, zr);
3158     } else {
3159       __ movw(dst_reg, con);
3160     }
3161   %}
3162 
3163   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3164     MacroAssembler _masm(&cbuf);
3165     Register dst_reg = as_Register($dst$$reg);
3166     u_int64_t con = (u_int64_t)$src$$constant;
3167     if (con == 0) {
3168       __ mov(dst_reg, zr);
3169     } else {
3170       __ mov(dst_reg, con);
3171     }
3172   %}
3173 
3174   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3175     MacroAssembler _masm(&cbuf);
3176     Register dst_reg = as_Register($dst$$reg);
3177     address con = (address)$src$$constant;
3178     if (con == NULL || con == (address)1) {
3179       ShouldNotReachHere();
3180     } else {
3181       relocInfo::relocType rtype = $src->constant_reloc();
3182       if (rtype == relocInfo::oop_type) {
3183         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
3184       } else if (rtype == relocInfo::metadata_type) {
3185         __ mov_metadata(dst_reg, (Metadata*)con);
3186       } else {
3187         assert(rtype == relocInfo::none, "unexpected reloc type");
3188         if (con < (address)(uintptr_t)os::vm_page_size()) {
3189           __ mov(dst_reg, con);
3190         } else {
3191           unsigned long offset;
3192           __ adrp(dst_reg, con, offset);
3193           __ add(dst_reg, dst_reg, offset);
3194         }
3195       }
3196     }
3197   %}
3198 
3199   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3200     MacroAssembler _masm(&cbuf);
3201     Register dst_reg = as_Register($dst$$reg);
3202     __ mov(dst_reg, zr);
3203   %}
3204 
3205   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3206     MacroAssembler _masm(&cbuf);
3207     Register dst_reg = as_Register($dst$$reg);
3208     __ mov(dst_reg, (u_int64_t)1);
3209   %}
3210 
3211   enc_class aarch64_enc_mov_poll_page(iRegP dst, immPollPage src) %{
3212     MacroAssembler _masm(&cbuf);
3213     address page = (address)$src$$constant;
3214     Register dst_reg = as_Register($dst$$reg);
3215     unsigned long off;
3216     __ adrp(dst_reg, Address(page, relocInfo::poll_type), off);
3217     assert(off == 0, "assumed offset == 0");
3218   %}
3219 
3220   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3221     MacroAssembler _masm(&cbuf);
3222     __ load_byte_map_base($dst$$Register);
3223   %}
3224 
3225   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3226     MacroAssembler _masm(&cbuf);
3227     Register dst_reg = as_Register($dst$$reg);
3228     address con = (address)$src$$constant;
3229     if (con == NULL) {
3230       ShouldNotReachHere();
3231     } else {
3232       relocInfo::relocType rtype = $src->constant_reloc();
3233       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3234       __ set_narrow_oop(dst_reg, (jobject)con);
3235     }
3236   %}
3237 
3238   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3239     MacroAssembler _masm(&cbuf);
3240     Register dst_reg = as_Register($dst$$reg);
3241     __ mov(dst_reg, zr);
3242   %}
3243 
3244   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3245     MacroAssembler _masm(&cbuf);
3246     Register dst_reg = as_Register($dst$$reg);
3247     address con = (address)$src$$constant;
3248     if (con == NULL) {
3249       ShouldNotReachHere();
3250     } else {
3251       relocInfo::relocType rtype = $src->constant_reloc();
3252       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3253       __ set_narrow_klass(dst_reg, (Klass *)con);
3254     }
3255   %}
3256 
3257   // arithmetic encodings
3258 
3259   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3260     MacroAssembler _masm(&cbuf);
3261     Register dst_reg = as_Register($dst$$reg);
3262     Register src_reg = as_Register($src1$$reg);
3263     int32_t con = (int32_t)$src2$$constant;
3264     // add has primary == 0, subtract has primary == 1
3265     if ($primary) { con = -con; }
3266     if (con < 0) {
3267       __ subw(dst_reg, src_reg, -con);
3268     } else {
3269       __ addw(dst_reg, src_reg, con);
3270     }
3271   %}
3272 
3273   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3274     MacroAssembler _masm(&cbuf);
3275     Register dst_reg = as_Register($dst$$reg);
3276     Register src_reg = as_Register($src1$$reg);
3277     int32_t con = (int32_t)$src2$$constant;
3278     // add has primary == 0, subtract has primary == 1
3279     if ($primary) { con = -con; }
3280     if (con < 0) {
3281       __ sub(dst_reg, src_reg, -con);
3282     } else {
3283       __ add(dst_reg, src_reg, con);
3284     }
3285   %}
3286 
3287   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3288     MacroAssembler _masm(&cbuf);
3289    Register dst_reg = as_Register($dst$$reg);
3290    Register src1_reg = as_Register($src1$$reg);
3291    Register src2_reg = as_Register($src2$$reg);
3292     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3293   %}
3294 
3295   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3296     MacroAssembler _masm(&cbuf);
3297    Register dst_reg = as_Register($dst$$reg);
3298    Register src1_reg = as_Register($src1$$reg);
3299    Register src2_reg = as_Register($src2$$reg);
3300     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3301   %}
3302 
3303   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3304     MacroAssembler _masm(&cbuf);
3305    Register dst_reg = as_Register($dst$$reg);
3306    Register src1_reg = as_Register($src1$$reg);
3307    Register src2_reg = as_Register($src2$$reg);
3308     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3309   %}
3310 
3311   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3312     MacroAssembler _masm(&cbuf);
3313    Register dst_reg = as_Register($dst$$reg);
3314    Register src1_reg = as_Register($src1$$reg);
3315    Register src2_reg = as_Register($src2$$reg);
3316     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3317   %}
3318 
3319   // compare instruction encodings
3320 
3321   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3322     MacroAssembler _masm(&cbuf);
3323     Register reg1 = as_Register($src1$$reg);
3324     Register reg2 = as_Register($src2$$reg);
3325     __ cmpw(reg1, reg2);
3326   %}
3327 
3328   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3329     MacroAssembler _masm(&cbuf);
3330     Register reg = as_Register($src1$$reg);
3331     int32_t val = $src2$$constant;
3332     if (val >= 0) {
3333       __ subsw(zr, reg, val);
3334     } else {
3335       __ addsw(zr, reg, -val);
3336     }
3337   %}
3338 
3339   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3340     MacroAssembler _masm(&cbuf);
3341     Register reg1 = as_Register($src1$$reg);
3342     u_int32_t val = (u_int32_t)$src2$$constant;
3343     __ movw(rscratch1, val);
3344     __ cmpw(reg1, rscratch1);
3345   %}
3346 
3347   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3348     MacroAssembler _masm(&cbuf);
3349     Register reg1 = as_Register($src1$$reg);
3350     Register reg2 = as_Register($src2$$reg);
3351     __ cmp(reg1, reg2);
3352   %}
3353 
3354   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3355     MacroAssembler _masm(&cbuf);
3356     Register reg = as_Register($src1$$reg);
3357     int64_t val = $src2$$constant;
3358     if (val >= 0) {
3359       __ subs(zr, reg, val);
3360     } else if (val != -val) {
3361       __ adds(zr, reg, -val);
3362     } else {
3363     // aargh, Long.MIN_VALUE is a special case
3364       __ orr(rscratch1, zr, (u_int64_t)val);
3365       __ subs(zr, reg, rscratch1);
3366     }
3367   %}
3368 
3369   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3370     MacroAssembler _masm(&cbuf);
3371     Register reg1 = as_Register($src1$$reg);
3372     u_int64_t val = (u_int64_t)$src2$$constant;
3373     __ mov(rscratch1, val);
3374     __ cmp(reg1, rscratch1);
3375   %}
3376 
3377   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3378     MacroAssembler _masm(&cbuf);
3379     Register reg1 = as_Register($src1$$reg);
3380     Register reg2 = as_Register($src2$$reg);
3381     __ cmp(reg1, reg2);
3382   %}
3383 
3384   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3385     MacroAssembler _masm(&cbuf);
3386     Register reg1 = as_Register($src1$$reg);
3387     Register reg2 = as_Register($src2$$reg);
3388     __ cmpw(reg1, reg2);
3389   %}
3390 
3391   enc_class aarch64_enc_testp(iRegP src) %{
3392     MacroAssembler _masm(&cbuf);
3393     Register reg = as_Register($src$$reg);
3394     __ cmp(reg, zr);
3395   %}
3396 
3397   enc_class aarch64_enc_testn(iRegN src) %{
3398     MacroAssembler _masm(&cbuf);
3399     Register reg = as_Register($src$$reg);
3400     __ cmpw(reg, zr);
3401   %}
3402 
3403   enc_class aarch64_enc_b(label lbl) %{
3404     MacroAssembler _masm(&cbuf);
3405     Label *L = $lbl$$label;
3406     __ b(*L);
3407   %}
3408 
3409   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3410     MacroAssembler _masm(&cbuf);
3411     Label *L = $lbl$$label;
3412     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3413   %}
3414 
3415   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3416     MacroAssembler _masm(&cbuf);
3417     Label *L = $lbl$$label;
3418     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3419   %}
3420 
3421   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3422   %{
3423      Register sub_reg = as_Register($sub$$reg);
3424      Register super_reg = as_Register($super$$reg);
3425      Register temp_reg = as_Register($temp$$reg);
3426      Register result_reg = as_Register($result$$reg);
3427 
3428      Label miss;
3429      MacroAssembler _masm(&cbuf);
3430      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3431                                      NULL, &miss,
3432                                      /*set_cond_codes:*/ true);
3433      if ($primary) {
3434        __ mov(result_reg, zr);
3435      }
3436      __ bind(miss);
3437   %}
3438 
3439   enc_class aarch64_enc_java_static_call(method meth) %{
3440     MacroAssembler _masm(&cbuf);
3441 
3442     address addr = (address)$meth$$method;
3443     address call;
3444     if (!_method) {
3445       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3446       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
3447     } else {
3448       int method_index = resolved_method_index(cbuf);
3449       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3450                                                   : static_call_Relocation::spec(method_index);
3451       call = __ trampoline_call(Address(addr, rspec), &cbuf);
3452 
3453       // Emit stub for static call
3454       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
3455       if (stub == NULL) {
3456         ciEnv::current()->record_failure("CodeCache is full");
3457         return;
3458       }
3459     }
3460     if (call == NULL) {
3461       ciEnv::current()->record_failure("CodeCache is full");
3462       return;
3463     }
3464   %}
3465 
3466   enc_class aarch64_enc_java_dynamic_call(method meth) %{
3467     MacroAssembler _masm(&cbuf);
3468     int method_index = resolved_method_index(cbuf);
3469     address call = __ ic_call((address)$meth$$method, method_index);
3470     if (call == NULL) {
3471       ciEnv::current()->record_failure("CodeCache is full");
3472       return;
3473     }
3474   %}
3475 
3476   enc_class aarch64_enc_call_epilog() %{
3477     MacroAssembler _masm(&cbuf);
3478     if (VerifyStackAtCalls) {
3479       // Check that stack depth is unchanged: find majik cookie on stack
3480       __ call_Unimplemented();
3481     }
3482   %}
3483 
3484   enc_class aarch64_enc_java_to_runtime(method meth) %{
3485     MacroAssembler _masm(&cbuf);
3486 
3487     // some calls to generated routines (arraycopy code) are scheduled
3488     // by C2 as runtime calls. if so we can call them using a br (they
3489     // will be in a reachable segment) otherwise we have to use a blrt
3490     // which loads the absolute address into a register.
3491     address entry = (address)$meth$$method;
3492     CodeBlob *cb = CodeCache::find_blob(entry);
3493     if (cb) {
3494       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3495       if (call == NULL) {
3496         ciEnv::current()->record_failure("CodeCache is full");
3497         return;
3498       }
3499     } else {
3500       int gpcnt;
3501       int fpcnt;
3502       int rtype;
3503       getCallInfo(tf(), gpcnt, fpcnt, rtype);
3504       Label retaddr;
3505       __ adr(rscratch2, retaddr);
3506       __ lea(rscratch1, RuntimeAddress(entry));
3507       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3508       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3509       __ blrt(rscratch1, gpcnt, fpcnt, rtype);
3510       __ bind(retaddr);
3511       __ add(sp, sp, 2 * wordSize);
3512     }
3513   %}
3514 
3515   enc_class aarch64_enc_rethrow() %{
3516     MacroAssembler _masm(&cbuf);
3517     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3518   %}
3519 
3520   enc_class aarch64_enc_ret() %{
3521     MacroAssembler _masm(&cbuf);
3522     __ ret(lr);
3523   %}
3524 
3525   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3526     MacroAssembler _masm(&cbuf);
3527     Register target_reg = as_Register($jump_target$$reg);
3528     __ br(target_reg);
3529   %}
3530 
3531   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3532     MacroAssembler _masm(&cbuf);
3533     Register target_reg = as_Register($jump_target$$reg);
3534     // exception oop should be in r0
3535     // ret addr has been popped into lr
3536     // callee expects it in r3
3537     __ mov(r3, lr);
3538     __ br(target_reg);
3539   %}
3540 
3541   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3542     MacroAssembler _masm(&cbuf);
3543     Register oop = as_Register($object$$reg);
3544     Register box = as_Register($box$$reg);
3545     Register disp_hdr = as_Register($tmp$$reg);
3546     Register tmp = as_Register($tmp2$$reg);
3547     Label cont;
3548     Label object_has_monitor;
3549     Label cas_failed;
3550 
3551     assert_different_registers(oop, box, tmp, disp_hdr);
3552 
3553     // Load markOop from object into displaced_header.
3554     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3555 
3556     if (UseBiasedLocking && !UseOptoBiasInlining) {
3557       __ biased_locking_enter(box, oop, disp_hdr, tmp, true, cont);
3558     }
3559 
3560     // Check for existing monitor
3561     __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor);
3562 
3563     // Set tmp to be (markOop of object | UNLOCK_VALUE).
3564     __ orr(tmp, disp_hdr, markOopDesc::unlocked_value);
3565 
3566     // Initialize the box. (Must happen before we update the object mark!)
3567     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3568 
3569     // Compare object markOop with an unlocked value (tmp) and if
3570     // equal exchange the stack address of our box with object markOop.
3571     // On failure disp_hdr contains the possibly locked markOop.
3572     __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
3573                /*release*/ true, /*weak*/ false, disp_hdr);
3574     __ br(Assembler::EQ, cont);
3575 
3576     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3577 
3578     // If the compare-and-exchange succeeded, then we found an unlocked
3579     // object, will have now locked it will continue at label cont
3580 
3581     __ bind(cas_failed);
3582     // We did not see an unlocked object so try the fast recursive case.
3583 
3584     // Check if the owner is self by comparing the value in the
3585     // markOop of object (disp_hdr) with the stack pointer.
3586     __ mov(rscratch1, sp);
3587     __ sub(disp_hdr, disp_hdr, rscratch1);
3588     __ mov(tmp, (address) (~(os::vm_page_size()-1) | (uintptr_t)markOopDesc::lock_mask_in_place));
3589     // If condition is true we are cont and hence we can store 0 as the
3590     // displaced header in the box, which indicates that it is a recursive lock.
3591     __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
3592     __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3593 
3594     __ b(cont);
3595 
3596     // Handle existing monitor.
3597     __ bind(object_has_monitor);
3598 
3599     // The object's monitor m is unlocked iff m->owner == NULL,
3600     // otherwise m->owner may contain a thread or a stack address.
3601     //
3602     // Try to CAS m->owner from NULL to current thread.
3603     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markOopDesc::monitor_value));
3604     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
3605                /*release*/ true, /*weak*/ false, noreg); // Sets flags for result
3606 
3607     // Store a non-null value into the box to avoid looking like a re-entrant
3608     // lock. The fast-path monitor unlock code checks for
3609     // markOopDesc::monitor_value so use markOopDesc::unused_mark which has the
3610     // relevant bit set, and also matches ObjectSynchronizer::slow_enter.
3611     __ mov(tmp, (address)markOopDesc::unused_mark());
3612     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3613 
3614     __ bind(cont);
3615     // flag == EQ indicates success
3616     // flag == NE indicates failure
3617   %}
3618 
3619   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3620     MacroAssembler _masm(&cbuf);
3621     Register oop = as_Register($object$$reg);
3622     Register box = as_Register($box$$reg);
3623     Register disp_hdr = as_Register($tmp$$reg);
3624     Register tmp = as_Register($tmp2$$reg);
3625     Label cont;
3626     Label object_has_monitor;
3627 
3628     assert_different_registers(oop, box, tmp, disp_hdr);
3629 
3630     if (UseBiasedLocking && !UseOptoBiasInlining) {
3631       __ biased_locking_exit(oop, tmp, cont);
3632     }
3633 
3634     // Find the lock address and load the displaced header from the stack.
3635     __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3636 
3637     // If the displaced header is 0, we have a recursive unlock.
3638     __ cmp(disp_hdr, zr);
3639     __ br(Assembler::EQ, cont);
3640 
3641     // Handle existing monitor.
3642     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3643     __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor);
3644 
3645     // Check if it is still a light weight lock, this is is true if we
3646     // see the stack address of the basicLock in the markOop of the
3647     // object.
3648 
3649     __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
3650                /*release*/ true, /*weak*/ false, tmp);
3651     __ b(cont);
3652 
3653     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3654 
3655     // Handle existing monitor.
3656     __ bind(object_has_monitor);
3657     __ add(tmp, tmp, -markOopDesc::monitor_value); // monitor
3658     __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3659     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3660     __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner.
3661     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions
3662     __ cmp(rscratch1, zr); // Sets flags for result
3663     __ br(Assembler::NE, cont);
3664 
3665     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
3666     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
3667     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3668     __ cmp(rscratch1, zr); // Sets flags for result
3669     __ cbnz(rscratch1, cont);
3670     // need a release store here
3671     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3672     __ stlr(zr, tmp); // set unowned
3673 
3674     __ bind(cont);
3675     // flag == EQ indicates success
3676     // flag == NE indicates failure
3677   %}
3678 
3679 %}
3680 
3681 //----------FRAME--------------------------------------------------------------
3682 // Definition of frame structure and management information.
3683 //
3684 //  S T A C K   L A Y O U T    Allocators stack-slot number
3685 //                             |   (to get allocators register number
3686 //  G  Owned by    |        |  v    add OptoReg::stack0())
3687 //  r   CALLER     |        |
3688 //  o     |        +--------+      pad to even-align allocators stack-slot
3689 //  w     V        |  pad0  |        numbers; owned by CALLER
3690 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
3691 //  h     ^        |   in   |  5
3692 //        |        |  args  |  4   Holes in incoming args owned by SELF
3693 //  |     |        |        |  3
3694 //  |     |        +--------+
3695 //  V     |        | old out|      Empty on Intel, window on Sparc
3696 //        |    old |preserve|      Must be even aligned.
3697 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
3698 //        |        |   in   |  3   area for Intel ret address
3699 //     Owned by    |preserve|      Empty on Sparc.
3700 //       SELF      +--------+
3701 //        |        |  pad2  |  2   pad to align old SP
3702 //        |        +--------+  1
3703 //        |        | locks  |  0
3704 //        |        +--------+----> OptoReg::stack0(), even aligned
3705 //        |        |  pad1  | 11   pad to align new SP
3706 //        |        +--------+
3707 //        |        |        | 10
3708 //        |        | spills |  9   spills
3709 //        V        |        |  8   (pad0 slot for callee)
3710 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
3711 //        ^        |  out   |  7
3712 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
3713 //     Owned by    +--------+
3714 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
3715 //        |    new |preserve|      Must be even-aligned.
3716 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
3717 //        |        |        |
3718 //
3719 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
3720 //         known from SELF's arguments and the Java calling convention.
3721 //         Region 6-7 is determined per call site.
3722 // Note 2: If the calling convention leaves holes in the incoming argument
3723 //         area, those holes are owned by SELF.  Holes in the outgoing area
3724 //         are owned by the CALLEE.  Holes should not be nessecary in the
3725 //         incoming area, as the Java calling convention is completely under
3726 //         the control of the AD file.  Doubles can be sorted and packed to
3727 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
3728 //         varargs C calling conventions.
3729 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
3730 //         even aligned with pad0 as needed.
3731 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
3732 //           (the latter is true on Intel but is it false on AArch64?)
3733 //         region 6-11 is even aligned; it may be padded out more so that
3734 //         the region from SP to FP meets the minimum stack alignment.
3735 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3736 //         alignment.  Region 11, pad1, may be dynamically extended so that
3737 //         SP meets the minimum alignment.
3738 
3739 frame %{
3740   // What direction does stack grow in (assumed to be same for C & Java)
3741   stack_direction(TOWARDS_LOW);
3742 
3743   // These three registers define part of the calling convention
3744   // between compiled code and the interpreter.
3745 
3746   // Inline Cache Register or methodOop for I2C.
3747   inline_cache_reg(R12);
3748 
3749   // Method Oop Register when calling interpreter.
3750   interpreter_method_oop_reg(R12);
3751 
3752   // Number of stack slots consumed by locking an object
3753   sync_stack_slots(2);
3754 
3755   // Compiled code's Frame Pointer
3756   frame_pointer(R31);
3757 
3758   // Interpreter stores its frame pointer in a register which is
3759   // stored to the stack by I2CAdaptors.
3760   // I2CAdaptors convert from interpreted java to compiled java.
3761   interpreter_frame_pointer(R29);
3762 
3763   // Stack alignment requirement
3764   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3765 
3766   // Number of stack slots between incoming argument block and the start of
3767   // a new frame.  The PROLOG must add this many slots to the stack.  The
3768   // EPILOG must remove this many slots. aarch64 needs two slots for
3769   // return address and fp.
3770   // TODO think this is correct but check
3771   in_preserve_stack_slots(4);
3772 
3773   // Number of outgoing stack slots killed above the out_preserve_stack_slots
3774   // for calls to C.  Supports the var-args backing area for register parms.
3775   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3776 
3777   // The after-PROLOG location of the return address.  Location of
3778   // return address specifies a type (REG or STACK) and a number
3779   // representing the register number (i.e. - use a register name) or
3780   // stack slot.
3781   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3782   // Otherwise, it is above the locks and verification slot and alignment word
3783   // TODO this may well be correct but need to check why that - 2 is there
3784   // ppc port uses 0 but we definitely need to allow for fixed_slots
3785   // which folds in the space used for monitors
3786   return_addr(STACK - 2 +
3787               align_up((Compile::current()->in_preserve_stack_slots() +
3788                         Compile::current()->fixed_slots()),
3789                        stack_alignment_in_slots()));
3790 
3791   // Body of function which returns an integer array locating
3792   // arguments either in registers or in stack slots.  Passed an array
3793   // of ideal registers called "sig" and a "length" count.  Stack-slot
3794   // offsets are based on outgoing arguments, i.e. a CALLER setting up
3795   // arguments for a CALLEE.  Incoming stack arguments are
3796   // automatically biased by the preserve_stack_slots field above.
3797 
3798   calling_convention
3799   %{
3800     // No difference between ingoing/outgoing just pass false
3801     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
3802   %}
3803 
3804   c_calling_convention
3805   %{
3806     // This is obviously always outgoing
3807     (void) SharedRuntime::c_calling_convention(sig_bt, regs, NULL, length);
3808   %}
3809 
3810   // Location of compiled Java return values.  Same as C for now.
3811   return_value
3812   %{
3813     // TODO do we allow ideal_reg == Op_RegN???
3814     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3815            "only return normal values");
3816 
3817     static const int lo[Op_RegL + 1] = { // enum name
3818       0,                                 // Op_Node
3819       0,                                 // Op_Set
3820       R0_num,                            // Op_RegN
3821       R0_num,                            // Op_RegI
3822       R0_num,                            // Op_RegP
3823       V0_num,                            // Op_RegF
3824       V0_num,                            // Op_RegD
3825       R0_num                             // Op_RegL
3826     };
3827 
3828     static const int hi[Op_RegL + 1] = { // enum name
3829       0,                                 // Op_Node
3830       0,                                 // Op_Set
3831       OptoReg::Bad,                       // Op_RegN
3832       OptoReg::Bad,                      // Op_RegI
3833       R0_H_num,                          // Op_RegP
3834       OptoReg::Bad,                      // Op_RegF
3835       V0_H_num,                          // Op_RegD
3836       R0_H_num                           // Op_RegL
3837     };
3838 
3839     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3840   %}
3841 %}
3842 
3843 //----------ATTRIBUTES---------------------------------------------------------
3844 //----------Operand Attributes-------------------------------------------------
3845 op_attrib op_cost(1);        // Required cost attribute
3846 
3847 //----------Instruction Attributes---------------------------------------------
3848 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3849 ins_attrib ins_size(32);        // Required size attribute (in bits)
3850 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3851                                 // a non-matching short branch variant
3852                                 // of some long branch?
3853 ins_attrib ins_alignment(4);    // Required alignment attribute (must
3854                                 // be a power of 2) specifies the
3855                                 // alignment that some part of the
3856                                 // instruction (not necessarily the
3857                                 // start) requires.  If > 1, a
3858                                 // compute_padding() function must be
3859                                 // provided for the instruction
3860 
3861 //----------OPERANDS-----------------------------------------------------------
3862 // Operand definitions must precede instruction definitions for correct parsing
3863 // in the ADLC because operands constitute user defined types which are used in
3864 // instruction definitions.
3865 
3866 //----------Simple Operands----------------------------------------------------
3867 
3868 // Integer operands 32 bit
3869 // 32 bit immediate
3870 operand immI()
3871 %{
3872   match(ConI);
3873 
3874   op_cost(0);
3875   format %{ %}
3876   interface(CONST_INTER);
3877 %}
3878 
3879 // 32 bit zero
3880 operand immI0()
3881 %{
3882   predicate(n->get_int() == 0);
3883   match(ConI);
3884 
3885   op_cost(0);
3886   format %{ %}
3887   interface(CONST_INTER);
3888 %}
3889 
3890 // 32 bit unit increment
3891 operand immI_1()
3892 %{
3893   predicate(n->get_int() == 1);
3894   match(ConI);
3895 
3896   op_cost(0);
3897   format %{ %}
3898   interface(CONST_INTER);
3899 %}
3900 
3901 // 32 bit unit decrement
3902 operand immI_M1()
3903 %{
3904   predicate(n->get_int() == -1);
3905   match(ConI);
3906 
3907   op_cost(0);
3908   format %{ %}
3909   interface(CONST_INTER);
3910 %}
3911 
3912 // Shift values for add/sub extension shift
3913 operand immIExt()
3914 %{
3915   predicate(0 <= n->get_int() && (n->get_int() <= 4));
3916   match(ConI);
3917 
3918   op_cost(0);
3919   format %{ %}
3920   interface(CONST_INTER);
3921 %}
3922 
3923 operand immI_le_4()
3924 %{
3925   predicate(n->get_int() <= 4);
3926   match(ConI);
3927 
3928   op_cost(0);
3929   format %{ %}
3930   interface(CONST_INTER);
3931 %}
3932 
3933 operand immI_31()
3934 %{
3935   predicate(n->get_int() == 31);
3936   match(ConI);
3937 
3938   op_cost(0);
3939   format %{ %}
3940   interface(CONST_INTER);
3941 %}
3942 
3943 operand immI_8()
3944 %{
3945   predicate(n->get_int() == 8);
3946   match(ConI);
3947 
3948   op_cost(0);
3949   format %{ %}
3950   interface(CONST_INTER);
3951 %}
3952 
3953 operand immI_16()
3954 %{
3955   predicate(n->get_int() == 16);
3956   match(ConI);
3957 
3958   op_cost(0);
3959   format %{ %}
3960   interface(CONST_INTER);
3961 %}
3962 
3963 operand immI_24()
3964 %{
3965   predicate(n->get_int() == 24);
3966   match(ConI);
3967 
3968   op_cost(0);
3969   format %{ %}
3970   interface(CONST_INTER);
3971 %}
3972 
3973 operand immI_32()
3974 %{
3975   predicate(n->get_int() == 32);
3976   match(ConI);
3977 
3978   op_cost(0);
3979   format %{ %}
3980   interface(CONST_INTER);
3981 %}
3982 
3983 operand immI_48()
3984 %{
3985   predicate(n->get_int() == 48);
3986   match(ConI);
3987 
3988   op_cost(0);
3989   format %{ %}
3990   interface(CONST_INTER);
3991 %}
3992 
3993 operand immI_56()
3994 %{
3995   predicate(n->get_int() == 56);
3996   match(ConI);
3997 
3998   op_cost(0);
3999   format %{ %}
4000   interface(CONST_INTER);
4001 %}
4002 
4003 operand immI_63()
4004 %{
4005   predicate(n->get_int() == 63);
4006   match(ConI);
4007 
4008   op_cost(0);
4009   format %{ %}
4010   interface(CONST_INTER);
4011 %}
4012 
4013 operand immI_64()
4014 %{
4015   predicate(n->get_int() == 64);
4016   match(ConI);
4017 
4018   op_cost(0);
4019   format %{ %}
4020   interface(CONST_INTER);
4021 %}
4022 
4023 operand immI_255()
4024 %{
4025   predicate(n->get_int() == 255);
4026   match(ConI);
4027 
4028   op_cost(0);
4029   format %{ %}
4030   interface(CONST_INTER);
4031 %}
4032 
4033 operand immI_65535()
4034 %{
4035   predicate(n->get_int() == 65535);
4036   match(ConI);
4037 
4038   op_cost(0);
4039   format %{ %}
4040   interface(CONST_INTER);
4041 %}
4042 
4043 operand immL_255()
4044 %{
4045   predicate(n->get_long() == 255L);
4046   match(ConL);
4047 
4048   op_cost(0);
4049   format %{ %}
4050   interface(CONST_INTER);
4051 %}
4052 
4053 operand immL_65535()
4054 %{
4055   predicate(n->get_long() == 65535L);
4056   match(ConL);
4057 
4058   op_cost(0);
4059   format %{ %}
4060   interface(CONST_INTER);
4061 %}
4062 
4063 operand immL_4294967295()
4064 %{
4065   predicate(n->get_long() == 4294967295L);
4066   match(ConL);
4067 
4068   op_cost(0);
4069   format %{ %}
4070   interface(CONST_INTER);
4071 %}
4072 
4073 operand immL_bitmask()
4074 %{
4075   predicate((n->get_long() != 0)
4076             && ((n->get_long() & 0xc000000000000000l) == 0)
4077             && is_power_of_2(n->get_long() + 1));
4078   match(ConL);
4079 
4080   op_cost(0);
4081   format %{ %}
4082   interface(CONST_INTER);
4083 %}
4084 
4085 operand immI_bitmask()
4086 %{
4087   predicate((n->get_int() != 0)
4088             && ((n->get_int() & 0xc0000000) == 0)
4089             && is_power_of_2(n->get_int() + 1));
4090   match(ConI);
4091 
4092   op_cost(0);
4093   format %{ %}
4094   interface(CONST_INTER);
4095 %}
4096 
4097 // Scale values for scaled offset addressing modes (up to long but not quad)
4098 operand immIScale()
4099 %{
4100   predicate(0 <= n->get_int() && (n->get_int() <= 3));
4101   match(ConI);
4102 
4103   op_cost(0);
4104   format %{ %}
4105   interface(CONST_INTER);
4106 %}
4107 
4108 // 26 bit signed offset -- for pc-relative branches
4109 operand immI26()
4110 %{
4111   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
4112   match(ConI);
4113 
4114   op_cost(0);
4115   format %{ %}
4116   interface(CONST_INTER);
4117 %}
4118 
4119 // 19 bit signed offset -- for pc-relative loads
4120 operand immI19()
4121 %{
4122   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
4123   match(ConI);
4124 
4125   op_cost(0);
4126   format %{ %}
4127   interface(CONST_INTER);
4128 %}
4129 
4130 // 12 bit unsigned offset -- for base plus immediate loads
4131 operand immIU12()
4132 %{
4133   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
4134   match(ConI);
4135 
4136   op_cost(0);
4137   format %{ %}
4138   interface(CONST_INTER);
4139 %}
4140 
4141 operand immLU12()
4142 %{
4143   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4144   match(ConL);
4145 
4146   op_cost(0);
4147   format %{ %}
4148   interface(CONST_INTER);
4149 %}
4150 
4151 // Offset for scaled or unscaled immediate loads and stores
4152 operand immIOffset()
4153 %{
4154   predicate(Address::offset_ok_for_immed(n->get_int()));
4155   match(ConI);
4156 
4157   op_cost(0);
4158   format %{ %}
4159   interface(CONST_INTER);
4160 %}
4161 
4162 operand immIOffset4()
4163 %{
4164   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4165   match(ConI);
4166 
4167   op_cost(0);
4168   format %{ %}
4169   interface(CONST_INTER);
4170 %}
4171 
4172 operand immIOffset8()
4173 %{
4174   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4175   match(ConI);
4176 
4177   op_cost(0);
4178   format %{ %}
4179   interface(CONST_INTER);
4180 %}
4181 
4182 operand immIOffset16()
4183 %{
4184   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4185   match(ConI);
4186 
4187   op_cost(0);
4188   format %{ %}
4189   interface(CONST_INTER);
4190 %}
4191 
4192 operand immLoffset()
4193 %{
4194   predicate(Address::offset_ok_for_immed(n->get_long()));
4195   match(ConL);
4196 
4197   op_cost(0);
4198   format %{ %}
4199   interface(CONST_INTER);
4200 %}
4201 
4202 operand immLoffset4()
4203 %{
4204   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4205   match(ConL);
4206 
4207   op_cost(0);
4208   format %{ %}
4209   interface(CONST_INTER);
4210 %}
4211 
4212 operand immLoffset8()
4213 %{
4214   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4215   match(ConL);
4216 
4217   op_cost(0);
4218   format %{ %}
4219   interface(CONST_INTER);
4220 %}
4221 
4222 operand immLoffset16()
4223 %{
4224   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4225   match(ConL);
4226 
4227   op_cost(0);
4228   format %{ %}
4229   interface(CONST_INTER);
4230 %}
4231 
4232 // 32 bit integer valid for add sub immediate
4233 operand immIAddSub()
4234 %{
4235   predicate(Assembler::operand_valid_for_add_sub_immediate((long)n->get_int()));
4236   match(ConI);
4237   op_cost(0);
4238   format %{ %}
4239   interface(CONST_INTER);
4240 %}
4241 
4242 // 32 bit unsigned integer valid for logical immediate
4243 // TODO -- check this is right when e.g the mask is 0x80000000
4244 operand immILog()
4245 %{
4246   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (unsigned long)n->get_int()));
4247   match(ConI);
4248 
4249   op_cost(0);
4250   format %{ %}
4251   interface(CONST_INTER);
4252 %}
4253 
4254 // Integer operands 64 bit
4255 // 64 bit immediate
4256 operand immL()
4257 %{
4258   match(ConL);
4259 
4260   op_cost(0);
4261   format %{ %}
4262   interface(CONST_INTER);
4263 %}
4264 
4265 // 64 bit zero
4266 operand immL0()
4267 %{
4268   predicate(n->get_long() == 0);
4269   match(ConL);
4270 
4271   op_cost(0);
4272   format %{ %}
4273   interface(CONST_INTER);
4274 %}
4275 
4276 // 64 bit unit increment
4277 operand immL_1()
4278 %{
4279   predicate(n->get_long() == 1);
4280   match(ConL);
4281 
4282   op_cost(0);
4283   format %{ %}
4284   interface(CONST_INTER);
4285 %}
4286 
4287 // 64 bit unit decrement
4288 operand immL_M1()
4289 %{
4290   predicate(n->get_long() == -1);
4291   match(ConL);
4292 
4293   op_cost(0);
4294   format %{ %}
4295   interface(CONST_INTER);
4296 %}
4297 
4298 // 32 bit offset of pc in thread anchor
4299 
4300 operand immL_pc_off()
4301 %{
4302   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4303                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4304   match(ConL);
4305 
4306   op_cost(0);
4307   format %{ %}
4308   interface(CONST_INTER);
4309 %}
4310 
4311 // 64 bit integer valid for add sub immediate
4312 operand immLAddSub()
4313 %{
4314   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4315   match(ConL);
4316   op_cost(0);
4317   format %{ %}
4318   interface(CONST_INTER);
4319 %}
4320 
4321 // 64 bit integer valid for logical immediate
4322 operand immLLog()
4323 %{
4324   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (unsigned long)n->get_long()));
4325   match(ConL);
4326   op_cost(0);
4327   format %{ %}
4328   interface(CONST_INTER);
4329 %}
4330 
4331 // Long Immediate: low 32-bit mask
4332 operand immL_32bits()
4333 %{
4334   predicate(n->get_long() == 0xFFFFFFFFL);
4335   match(ConL);
4336   op_cost(0);
4337   format %{ %}
4338   interface(CONST_INTER);
4339 %}
4340 
4341 // Pointer operands
4342 // Pointer Immediate
4343 operand immP()
4344 %{
4345   match(ConP);
4346 
4347   op_cost(0);
4348   format %{ %}
4349   interface(CONST_INTER);
4350 %}
4351 
4352 // NULL Pointer Immediate
4353 operand immP0()
4354 %{
4355   predicate(n->get_ptr() == 0);
4356   match(ConP);
4357 
4358   op_cost(0);
4359   format %{ %}
4360   interface(CONST_INTER);
4361 %}
4362 
4363 // Pointer Immediate One
4364 // this is used in object initialization (initial object header)
4365 operand immP_1()
4366 %{
4367   predicate(n->get_ptr() == 1);
4368   match(ConP);
4369 
4370   op_cost(0);
4371   format %{ %}
4372   interface(CONST_INTER);
4373 %}
4374 
4375 // Polling Page Pointer Immediate
4376 operand immPollPage()
4377 %{
4378   predicate((address)n->get_ptr() == os::get_polling_page());
4379   match(ConP);
4380 
4381   op_cost(0);
4382   format %{ %}
4383   interface(CONST_INTER);
4384 %}
4385 
4386 // Card Table Byte Map Base
4387 operand immByteMapBase()
4388 %{
4389   // Get base of card map
4390   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4391             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4392   match(ConP);
4393 
4394   op_cost(0);
4395   format %{ %}
4396   interface(CONST_INTER);
4397 %}
4398 
4399 // Pointer Immediate Minus One
4400 // this is used when we want to write the current PC to the thread anchor
4401 operand immP_M1()
4402 %{
4403   predicate(n->get_ptr() == -1);
4404   match(ConP);
4405 
4406   op_cost(0);
4407   format %{ %}
4408   interface(CONST_INTER);
4409 %}
4410 
4411 // Pointer Immediate Minus Two
4412 // this is used when we want to write the current PC to the thread anchor
4413 operand immP_M2()
4414 %{
4415   predicate(n->get_ptr() == -2);
4416   match(ConP);
4417 
4418   op_cost(0);
4419   format %{ %}
4420   interface(CONST_INTER);
4421 %}
4422 
4423 // Float and Double operands
4424 // Double Immediate
4425 operand immD()
4426 %{
4427   match(ConD);
4428   op_cost(0);
4429   format %{ %}
4430   interface(CONST_INTER);
4431 %}
4432 
4433 // Double Immediate: +0.0d
4434 operand immD0()
4435 %{
4436   predicate(jlong_cast(n->getd()) == 0);
4437   match(ConD);
4438 
4439   op_cost(0);
4440   format %{ %}
4441   interface(CONST_INTER);
4442 %}
4443 
4444 // constant 'double +0.0'.
4445 operand immDPacked()
4446 %{
4447   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4448   match(ConD);
4449   op_cost(0);
4450   format %{ %}
4451   interface(CONST_INTER);
4452 %}
4453 
4454 // Float Immediate
4455 operand immF()
4456 %{
4457   match(ConF);
4458   op_cost(0);
4459   format %{ %}
4460   interface(CONST_INTER);
4461 %}
4462 
4463 // Float Immediate: +0.0f.
4464 operand immF0()
4465 %{
4466   predicate(jint_cast(n->getf()) == 0);
4467   match(ConF);
4468 
4469   op_cost(0);
4470   format %{ %}
4471   interface(CONST_INTER);
4472 %}
4473 
4474 //
4475 operand immFPacked()
4476 %{
4477   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4478   match(ConF);
4479   op_cost(0);
4480   format %{ %}
4481   interface(CONST_INTER);
4482 %}
4483 
4484 // Narrow pointer operands
4485 // Narrow Pointer Immediate
4486 operand immN()
4487 %{
4488   match(ConN);
4489 
4490   op_cost(0);
4491   format %{ %}
4492   interface(CONST_INTER);
4493 %}
4494 
4495 // Narrow NULL Pointer Immediate
4496 operand immN0()
4497 %{
4498   predicate(n->get_narrowcon() == 0);
4499   match(ConN);
4500 
4501   op_cost(0);
4502   format %{ %}
4503   interface(CONST_INTER);
4504 %}
4505 
4506 operand immNKlass()
4507 %{
4508   match(ConNKlass);
4509 
4510   op_cost(0);
4511   format %{ %}
4512   interface(CONST_INTER);
4513 %}
4514 
4515 // Integer 32 bit Register Operands
4516 // Integer 32 bitRegister (excludes SP)
4517 operand iRegI()
4518 %{
4519   constraint(ALLOC_IN_RC(any_reg32));
4520   match(RegI);
4521   match(iRegINoSp);
4522   op_cost(0);
4523   format %{ %}
4524   interface(REG_INTER);
4525 %}
4526 
4527 // Integer 32 bit Register not Special
4528 operand iRegINoSp()
4529 %{
4530   constraint(ALLOC_IN_RC(no_special_reg32));
4531   match(RegI);
4532   op_cost(0);
4533   format %{ %}
4534   interface(REG_INTER);
4535 %}
4536 
4537 // Integer 64 bit Register Operands
4538 // Integer 64 bit Register (includes SP)
4539 operand iRegL()
4540 %{
4541   constraint(ALLOC_IN_RC(any_reg));
4542   match(RegL);
4543   match(iRegLNoSp);
4544   op_cost(0);
4545   format %{ %}
4546   interface(REG_INTER);
4547 %}
4548 
4549 // Integer 64 bit Register not Special
4550 operand iRegLNoSp()
4551 %{
4552   constraint(ALLOC_IN_RC(no_special_reg));
4553   match(RegL);
4554   match(iRegL_R0);
4555   format %{ %}
4556   interface(REG_INTER);
4557 %}
4558 
4559 // Pointer Register Operands
4560 // Pointer Register
4561 operand iRegP()
4562 %{
4563   constraint(ALLOC_IN_RC(ptr_reg));
4564   match(RegP);
4565   match(iRegPNoSp);
4566   match(iRegP_R0);
4567   //match(iRegP_R2);
4568   //match(iRegP_R4);
4569   //match(iRegP_R5);
4570   match(thread_RegP);
4571   op_cost(0);
4572   format %{ %}
4573   interface(REG_INTER);
4574 %}
4575 
4576 // Pointer 64 bit Register not Special
4577 operand iRegPNoSp()
4578 %{
4579   constraint(ALLOC_IN_RC(no_special_ptr_reg));
4580   match(RegP);
4581   // match(iRegP);
4582   // match(iRegP_R0);
4583   // match(iRegP_R2);
4584   // match(iRegP_R4);
4585   // match(iRegP_R5);
4586   // match(thread_RegP);
4587   op_cost(0);
4588   format %{ %}
4589   interface(REG_INTER);
4590 %}
4591 
4592 // Pointer 64 bit Register R0 only
4593 operand iRegP_R0()
4594 %{
4595   constraint(ALLOC_IN_RC(r0_reg));
4596   match(RegP);
4597   // match(iRegP);
4598   match(iRegPNoSp);
4599   op_cost(0);
4600   format %{ %}
4601   interface(REG_INTER);
4602 %}
4603 
4604 // Pointer 64 bit Register R1 only
4605 operand iRegP_R1()
4606 %{
4607   constraint(ALLOC_IN_RC(r1_reg));
4608   match(RegP);
4609   // match(iRegP);
4610   match(iRegPNoSp);
4611   op_cost(0);
4612   format %{ %}
4613   interface(REG_INTER);
4614 %}
4615 
4616 // Pointer 64 bit Register R2 only
4617 operand iRegP_R2()
4618 %{
4619   constraint(ALLOC_IN_RC(r2_reg));
4620   match(RegP);
4621   // match(iRegP);
4622   match(iRegPNoSp);
4623   op_cost(0);
4624   format %{ %}
4625   interface(REG_INTER);
4626 %}
4627 
4628 // Pointer 64 bit Register R3 only
4629 operand iRegP_R3()
4630 %{
4631   constraint(ALLOC_IN_RC(r3_reg));
4632   match(RegP);
4633   // match(iRegP);
4634   match(iRegPNoSp);
4635   op_cost(0);
4636   format %{ %}
4637   interface(REG_INTER);
4638 %}
4639 
4640 // Pointer 64 bit Register R4 only
4641 operand iRegP_R4()
4642 %{
4643   constraint(ALLOC_IN_RC(r4_reg));
4644   match(RegP);
4645   // match(iRegP);
4646   match(iRegPNoSp);
4647   op_cost(0);
4648   format %{ %}
4649   interface(REG_INTER);
4650 %}
4651 
4652 // Pointer 64 bit Register R5 only
4653 operand iRegP_R5()
4654 %{
4655   constraint(ALLOC_IN_RC(r5_reg));
4656   match(RegP);
4657   // match(iRegP);
4658   match(iRegPNoSp);
4659   op_cost(0);
4660   format %{ %}
4661   interface(REG_INTER);
4662 %}
4663 
4664 // Pointer 64 bit Register R10 only
4665 operand iRegP_R10()
4666 %{
4667   constraint(ALLOC_IN_RC(r10_reg));
4668   match(RegP);
4669   // match(iRegP);
4670   match(iRegPNoSp);
4671   op_cost(0);
4672   format %{ %}
4673   interface(REG_INTER);
4674 %}
4675 
4676 // Long 64 bit Register R0 only
4677 operand iRegL_R0()
4678 %{
4679   constraint(ALLOC_IN_RC(r0_reg));
4680   match(RegL);
4681   match(iRegLNoSp);
4682   op_cost(0);
4683   format %{ %}
4684   interface(REG_INTER);
4685 %}
4686 
4687 // Long 64 bit Register R2 only
4688 operand iRegL_R2()
4689 %{
4690   constraint(ALLOC_IN_RC(r2_reg));
4691   match(RegL);
4692   match(iRegLNoSp);
4693   op_cost(0);
4694   format %{ %}
4695   interface(REG_INTER);
4696 %}
4697 
4698 // Long 64 bit Register R3 only
4699 operand iRegL_R3()
4700 %{
4701   constraint(ALLOC_IN_RC(r3_reg));
4702   match(RegL);
4703   match(iRegLNoSp);
4704   op_cost(0);
4705   format %{ %}
4706   interface(REG_INTER);
4707 %}
4708 
4709 // Long 64 bit Register R11 only
4710 operand iRegL_R11()
4711 %{
4712   constraint(ALLOC_IN_RC(r11_reg));
4713   match(RegL);
4714   match(iRegLNoSp);
4715   op_cost(0);
4716   format %{ %}
4717   interface(REG_INTER);
4718 %}
4719 
4720 // Pointer 64 bit Register FP only
4721 operand iRegP_FP()
4722 %{
4723   constraint(ALLOC_IN_RC(fp_reg));
4724   match(RegP);
4725   // match(iRegP);
4726   op_cost(0);
4727   format %{ %}
4728   interface(REG_INTER);
4729 %}
4730 
4731 // Register R0 only
4732 operand iRegI_R0()
4733 %{
4734   constraint(ALLOC_IN_RC(int_r0_reg));
4735   match(RegI);
4736   match(iRegINoSp);
4737   op_cost(0);
4738   format %{ %}
4739   interface(REG_INTER);
4740 %}
4741 
4742 // Register R2 only
4743 operand iRegI_R2()
4744 %{
4745   constraint(ALLOC_IN_RC(int_r2_reg));
4746   match(RegI);
4747   match(iRegINoSp);
4748   op_cost(0);
4749   format %{ %}
4750   interface(REG_INTER);
4751 %}
4752 
4753 // Register R3 only
4754 operand iRegI_R3()
4755 %{
4756   constraint(ALLOC_IN_RC(int_r3_reg));
4757   match(RegI);
4758   match(iRegINoSp);
4759   op_cost(0);
4760   format %{ %}
4761   interface(REG_INTER);
4762 %}
4763 
4764 
4765 // Register R4 only
4766 operand iRegI_R4()
4767 %{
4768   constraint(ALLOC_IN_RC(int_r4_reg));
4769   match(RegI);
4770   match(iRegINoSp);
4771   op_cost(0);
4772   format %{ %}
4773   interface(REG_INTER);
4774 %}
4775 
4776 
4777 // Pointer Register Operands
4778 // Narrow Pointer Register
4779 operand iRegN()
4780 %{
4781   constraint(ALLOC_IN_RC(any_reg32));
4782   match(RegN);
4783   match(iRegNNoSp);
4784   op_cost(0);
4785   format %{ %}
4786   interface(REG_INTER);
4787 %}
4788 
4789 operand iRegN_R0()
4790 %{
4791   constraint(ALLOC_IN_RC(r0_reg));
4792   match(iRegN);
4793   op_cost(0);
4794   format %{ %}
4795   interface(REG_INTER);
4796 %}
4797 
4798 operand iRegN_R2()
4799 %{
4800   constraint(ALLOC_IN_RC(r2_reg));
4801   match(iRegN);
4802   op_cost(0);
4803   format %{ %}
4804   interface(REG_INTER);
4805 %}
4806 
4807 operand iRegN_R3()
4808 %{
4809   constraint(ALLOC_IN_RC(r3_reg));
4810   match(iRegN);
4811   op_cost(0);
4812   format %{ %}
4813   interface(REG_INTER);
4814 %}
4815 
4816 // Integer 64 bit Register not Special
4817 operand iRegNNoSp()
4818 %{
4819   constraint(ALLOC_IN_RC(no_special_reg32));
4820   match(RegN);
4821   op_cost(0);
4822   format %{ %}
4823   interface(REG_INTER);
4824 %}
4825 
4826 // heap base register -- used for encoding immN0
4827 
4828 operand iRegIHeapbase()
4829 %{
4830   constraint(ALLOC_IN_RC(heapbase_reg));
4831   match(RegI);
4832   op_cost(0);
4833   format %{ %}
4834   interface(REG_INTER);
4835 %}
4836 
4837 // Float Register
4838 // Float register operands
4839 operand vRegF()
4840 %{
4841   constraint(ALLOC_IN_RC(float_reg));
4842   match(RegF);
4843 
4844   op_cost(0);
4845   format %{ %}
4846   interface(REG_INTER);
4847 %}
4848 
4849 // Double Register
4850 // Double register operands
4851 operand vRegD()
4852 %{
4853   constraint(ALLOC_IN_RC(double_reg));
4854   match(RegD);
4855 
4856   op_cost(0);
4857   format %{ %}
4858   interface(REG_INTER);
4859 %}
4860 
4861 operand vecD()
4862 %{
4863   constraint(ALLOC_IN_RC(vectord_reg));
4864   match(VecD);
4865 
4866   op_cost(0);
4867   format %{ %}
4868   interface(REG_INTER);
4869 %}
4870 
4871 operand vecX()
4872 %{
4873   constraint(ALLOC_IN_RC(vectorx_reg));
4874   match(VecX);
4875 
4876   op_cost(0);
4877   format %{ %}
4878   interface(REG_INTER);
4879 %}
4880 
4881 operand vRegD_V0()
4882 %{
4883   constraint(ALLOC_IN_RC(v0_reg));
4884   match(RegD);
4885   op_cost(0);
4886   format %{ %}
4887   interface(REG_INTER);
4888 %}
4889 
4890 operand vRegD_V1()
4891 %{
4892   constraint(ALLOC_IN_RC(v1_reg));
4893   match(RegD);
4894   op_cost(0);
4895   format %{ %}
4896   interface(REG_INTER);
4897 %}
4898 
4899 operand vRegD_V2()
4900 %{
4901   constraint(ALLOC_IN_RC(v2_reg));
4902   match(RegD);
4903   op_cost(0);
4904   format %{ %}
4905   interface(REG_INTER);
4906 %}
4907 
4908 operand vRegD_V3()
4909 %{
4910   constraint(ALLOC_IN_RC(v3_reg));
4911   match(RegD);
4912   op_cost(0);
4913   format %{ %}
4914   interface(REG_INTER);
4915 %}
4916 
4917 operand vRegD_V4()
4918 %{
4919   constraint(ALLOC_IN_RC(v4_reg));
4920   match(RegD);
4921   op_cost(0);
4922   format %{ %}
4923   interface(REG_INTER);
4924 %}
4925 
4926 operand vRegD_V5()
4927 %{
4928   constraint(ALLOC_IN_RC(v5_reg));
4929   match(RegD);
4930   op_cost(0);
4931   format %{ %}
4932   interface(REG_INTER);
4933 %}
4934 
4935 operand vRegD_V6()
4936 %{
4937   constraint(ALLOC_IN_RC(v6_reg));
4938   match(RegD);
4939   op_cost(0);
4940   format %{ %}
4941   interface(REG_INTER);
4942 %}
4943 
4944 operand vRegD_V7()
4945 %{
4946   constraint(ALLOC_IN_RC(v7_reg));
4947   match(RegD);
4948   op_cost(0);
4949   format %{ %}
4950   interface(REG_INTER);
4951 %}
4952 
4953 operand vRegD_V8()
4954 %{
4955   constraint(ALLOC_IN_RC(v8_reg));
4956   match(RegD);
4957   op_cost(0);
4958   format %{ %}
4959   interface(REG_INTER);
4960 %}
4961 
4962 operand vRegD_V9()
4963 %{
4964   constraint(ALLOC_IN_RC(v9_reg));
4965   match(RegD);
4966   op_cost(0);
4967   format %{ %}
4968   interface(REG_INTER);
4969 %}
4970 
4971 operand vRegD_V10()
4972 %{
4973   constraint(ALLOC_IN_RC(v10_reg));
4974   match(RegD);
4975   op_cost(0);
4976   format %{ %}
4977   interface(REG_INTER);
4978 %}
4979 
4980 operand vRegD_V11()
4981 %{
4982   constraint(ALLOC_IN_RC(v11_reg));
4983   match(RegD);
4984   op_cost(0);
4985   format %{ %}
4986   interface(REG_INTER);
4987 %}
4988 
4989 operand vRegD_V12()
4990 %{
4991   constraint(ALLOC_IN_RC(v12_reg));
4992   match(RegD);
4993   op_cost(0);
4994   format %{ %}
4995   interface(REG_INTER);
4996 %}
4997 
4998 operand vRegD_V13()
4999 %{
5000   constraint(ALLOC_IN_RC(v13_reg));
5001   match(RegD);
5002   op_cost(0);
5003   format %{ %}
5004   interface(REG_INTER);
5005 %}
5006 
5007 operand vRegD_V14()
5008 %{
5009   constraint(ALLOC_IN_RC(v14_reg));
5010   match(RegD);
5011   op_cost(0);
5012   format %{ %}
5013   interface(REG_INTER);
5014 %}
5015 
5016 operand vRegD_V15()
5017 %{
5018   constraint(ALLOC_IN_RC(v15_reg));
5019   match(RegD);
5020   op_cost(0);
5021   format %{ %}
5022   interface(REG_INTER);
5023 %}
5024 
5025 operand vRegD_V16()
5026 %{
5027   constraint(ALLOC_IN_RC(v16_reg));
5028   match(RegD);
5029   op_cost(0);
5030   format %{ %}
5031   interface(REG_INTER);
5032 %}
5033 
5034 operand vRegD_V17()
5035 %{
5036   constraint(ALLOC_IN_RC(v17_reg));
5037   match(RegD);
5038   op_cost(0);
5039   format %{ %}
5040   interface(REG_INTER);
5041 %}
5042 
5043 operand vRegD_V18()
5044 %{
5045   constraint(ALLOC_IN_RC(v18_reg));
5046   match(RegD);
5047   op_cost(0);
5048   format %{ %}
5049   interface(REG_INTER);
5050 %}
5051 
5052 operand vRegD_V19()
5053 %{
5054   constraint(ALLOC_IN_RC(v19_reg));
5055   match(RegD);
5056   op_cost(0);
5057   format %{ %}
5058   interface(REG_INTER);
5059 %}
5060 
5061 operand vRegD_V20()
5062 %{
5063   constraint(ALLOC_IN_RC(v20_reg));
5064   match(RegD);
5065   op_cost(0);
5066   format %{ %}
5067   interface(REG_INTER);
5068 %}
5069 
5070 operand vRegD_V21()
5071 %{
5072   constraint(ALLOC_IN_RC(v21_reg));
5073   match(RegD);
5074   op_cost(0);
5075   format %{ %}
5076   interface(REG_INTER);
5077 %}
5078 
5079 operand vRegD_V22()
5080 %{
5081   constraint(ALLOC_IN_RC(v22_reg));
5082   match(RegD);
5083   op_cost(0);
5084   format %{ %}
5085   interface(REG_INTER);
5086 %}
5087 
5088 operand vRegD_V23()
5089 %{
5090   constraint(ALLOC_IN_RC(v23_reg));
5091   match(RegD);
5092   op_cost(0);
5093   format %{ %}
5094   interface(REG_INTER);
5095 %}
5096 
5097 operand vRegD_V24()
5098 %{
5099   constraint(ALLOC_IN_RC(v24_reg));
5100   match(RegD);
5101   op_cost(0);
5102   format %{ %}
5103   interface(REG_INTER);
5104 %}
5105 
5106 operand vRegD_V25()
5107 %{
5108   constraint(ALLOC_IN_RC(v25_reg));
5109   match(RegD);
5110   op_cost(0);
5111   format %{ %}
5112   interface(REG_INTER);
5113 %}
5114 
5115 operand vRegD_V26()
5116 %{
5117   constraint(ALLOC_IN_RC(v26_reg));
5118   match(RegD);
5119   op_cost(0);
5120   format %{ %}
5121   interface(REG_INTER);
5122 %}
5123 
5124 operand vRegD_V27()
5125 %{
5126   constraint(ALLOC_IN_RC(v27_reg));
5127   match(RegD);
5128   op_cost(0);
5129   format %{ %}
5130   interface(REG_INTER);
5131 %}
5132 
5133 operand vRegD_V28()
5134 %{
5135   constraint(ALLOC_IN_RC(v28_reg));
5136   match(RegD);
5137   op_cost(0);
5138   format %{ %}
5139   interface(REG_INTER);
5140 %}
5141 
5142 operand vRegD_V29()
5143 %{
5144   constraint(ALLOC_IN_RC(v29_reg));
5145   match(RegD);
5146   op_cost(0);
5147   format %{ %}
5148   interface(REG_INTER);
5149 %}
5150 
5151 operand vRegD_V30()
5152 %{
5153   constraint(ALLOC_IN_RC(v30_reg));
5154   match(RegD);
5155   op_cost(0);
5156   format %{ %}
5157   interface(REG_INTER);
5158 %}
5159 
5160 operand vRegD_V31()
5161 %{
5162   constraint(ALLOC_IN_RC(v31_reg));
5163   match(RegD);
5164   op_cost(0);
5165   format %{ %}
5166   interface(REG_INTER);
5167 %}
5168 
5169 // Flags register, used as output of signed compare instructions
5170 
5171 // note that on AArch64 we also use this register as the output for
5172 // for floating point compare instructions (CmpF CmpD). this ensures
5173 // that ordered inequality tests use GT, GE, LT or LE none of which
5174 // pass through cases where the result is unordered i.e. one or both
5175 // inputs to the compare is a NaN. this means that the ideal code can
5176 // replace e.g. a GT with an LE and not end up capturing the NaN case
5177 // (where the comparison should always fail). EQ and NE tests are
5178 // always generated in ideal code so that unordered folds into the NE
5179 // case, matching the behaviour of AArch64 NE.
5180 //
5181 // This differs from x86 where the outputs of FP compares use a
5182 // special FP flags registers and where compares based on this
5183 // register are distinguished into ordered inequalities (cmpOpUCF) and
5184 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5185 // to explicitly handle the unordered case in branches. x86 also has
5186 // to include extra CMoveX rules to accept a cmpOpUCF input.
5187 
5188 operand rFlagsReg()
5189 %{
5190   constraint(ALLOC_IN_RC(int_flags));
5191   match(RegFlags);
5192 
5193   op_cost(0);
5194   format %{ "RFLAGS" %}
5195   interface(REG_INTER);
5196 %}
5197 
5198 // Flags register, used as output of unsigned compare instructions
5199 operand rFlagsRegU()
5200 %{
5201   constraint(ALLOC_IN_RC(int_flags));
5202   match(RegFlags);
5203 
5204   op_cost(0);
5205   format %{ "RFLAGSU" %}
5206   interface(REG_INTER);
5207 %}
5208 
5209 // Special Registers
5210 
5211 // Method Register
5212 operand inline_cache_RegP(iRegP reg)
5213 %{
5214   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5215   match(reg);
5216   match(iRegPNoSp);
5217   op_cost(0);
5218   format %{ %}
5219   interface(REG_INTER);
5220 %}
5221 
5222 operand interpreter_method_oop_RegP(iRegP reg)
5223 %{
5224   constraint(ALLOC_IN_RC(method_reg)); // interpreter_method_oop_reg
5225   match(reg);
5226   match(iRegPNoSp);
5227   op_cost(0);
5228   format %{ %}
5229   interface(REG_INTER);
5230 %}
5231 
5232 // Thread Register
5233 operand thread_RegP(iRegP reg)
5234 %{
5235   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5236   match(reg);
5237   op_cost(0);
5238   format %{ %}
5239   interface(REG_INTER);
5240 %}
5241 
5242 operand lr_RegP(iRegP reg)
5243 %{
5244   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
5245   match(reg);
5246   op_cost(0);
5247   format %{ %}
5248   interface(REG_INTER);
5249 %}
5250 
5251 //----------Memory Operands----------------------------------------------------
5252 
5253 operand indirect(iRegP reg)
5254 %{
5255   constraint(ALLOC_IN_RC(ptr_reg));
5256   match(reg);
5257   op_cost(0);
5258   format %{ "[$reg]" %}
5259   interface(MEMORY_INTER) %{
5260     base($reg);
5261     index(0xffffffff);
5262     scale(0x0);
5263     disp(0x0);
5264   %}
5265 %}
5266 
5267 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5268 %{
5269   constraint(ALLOC_IN_RC(ptr_reg));
5270   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5271   match(AddP reg (LShiftL (ConvI2L ireg) scale));
5272   op_cost(0);
5273   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5274   interface(MEMORY_INTER) %{
5275     base($reg);
5276     index($ireg);
5277     scale($scale);
5278     disp(0x0);
5279   %}
5280 %}
5281 
5282 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5283 %{
5284   constraint(ALLOC_IN_RC(ptr_reg));
5285   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5286   match(AddP reg (LShiftL lreg scale));
5287   op_cost(0);
5288   format %{ "$reg, $lreg lsl($scale)" %}
5289   interface(MEMORY_INTER) %{
5290     base($reg);
5291     index($lreg);
5292     scale($scale);
5293     disp(0x0);
5294   %}
5295 %}
5296 
5297 operand indIndexI2L(iRegP reg, iRegI ireg)
5298 %{
5299   constraint(ALLOC_IN_RC(ptr_reg));
5300   match(AddP reg (ConvI2L ireg));
5301   op_cost(0);
5302   format %{ "$reg, $ireg, 0, I2L" %}
5303   interface(MEMORY_INTER) %{
5304     base($reg);
5305     index($ireg);
5306     scale(0x0);
5307     disp(0x0);
5308   %}
5309 %}
5310 
5311 operand indIndex(iRegP reg, iRegL lreg)
5312 %{
5313   constraint(ALLOC_IN_RC(ptr_reg));
5314   match(AddP reg lreg);
5315   op_cost(0);
5316   format %{ "$reg, $lreg" %}
5317   interface(MEMORY_INTER) %{
5318     base($reg);
5319     index($lreg);
5320     scale(0x0);
5321     disp(0x0);
5322   %}
5323 %}
5324 
5325 operand indOffI(iRegP reg, immIOffset off)
5326 %{
5327   constraint(ALLOC_IN_RC(ptr_reg));
5328   match(AddP reg off);
5329   op_cost(0);
5330   format %{ "[$reg, $off]" %}
5331   interface(MEMORY_INTER) %{
5332     base($reg);
5333     index(0xffffffff);
5334     scale(0x0);
5335     disp($off);
5336   %}
5337 %}
5338 
5339 operand indOffI4(iRegP reg, immIOffset4 off)
5340 %{
5341   constraint(ALLOC_IN_RC(ptr_reg));
5342   match(AddP reg off);
5343   op_cost(0);
5344   format %{ "[$reg, $off]" %}
5345   interface(MEMORY_INTER) %{
5346     base($reg);
5347     index(0xffffffff);
5348     scale(0x0);
5349     disp($off);
5350   %}
5351 %}
5352 
5353 operand indOffI8(iRegP reg, immIOffset8 off)
5354 %{
5355   constraint(ALLOC_IN_RC(ptr_reg));
5356   match(AddP reg off);
5357   op_cost(0);
5358   format %{ "[$reg, $off]" %}
5359   interface(MEMORY_INTER) %{
5360     base($reg);
5361     index(0xffffffff);
5362     scale(0x0);
5363     disp($off);
5364   %}
5365 %}
5366 
5367 operand indOffI16(iRegP reg, immIOffset16 off)
5368 %{
5369   constraint(ALLOC_IN_RC(ptr_reg));
5370   match(AddP reg off);
5371   op_cost(0);
5372   format %{ "[$reg, $off]" %}
5373   interface(MEMORY_INTER) %{
5374     base($reg);
5375     index(0xffffffff);
5376     scale(0x0);
5377     disp($off);
5378   %}
5379 %}
5380 
5381 operand indOffL(iRegP reg, immLoffset off)
5382 %{
5383   constraint(ALLOC_IN_RC(ptr_reg));
5384   match(AddP reg off);
5385   op_cost(0);
5386   format %{ "[$reg, $off]" %}
5387   interface(MEMORY_INTER) %{
5388     base($reg);
5389     index(0xffffffff);
5390     scale(0x0);
5391     disp($off);
5392   %}
5393 %}
5394 
5395 operand indOffL4(iRegP reg, immLoffset4 off)
5396 %{
5397   constraint(ALLOC_IN_RC(ptr_reg));
5398   match(AddP reg off);
5399   op_cost(0);
5400   format %{ "[$reg, $off]" %}
5401   interface(MEMORY_INTER) %{
5402     base($reg);
5403     index(0xffffffff);
5404     scale(0x0);
5405     disp($off);
5406   %}
5407 %}
5408 
5409 operand indOffL8(iRegP reg, immLoffset8 off)
5410 %{
5411   constraint(ALLOC_IN_RC(ptr_reg));
5412   match(AddP reg off);
5413   op_cost(0);
5414   format %{ "[$reg, $off]" %}
5415   interface(MEMORY_INTER) %{
5416     base($reg);
5417     index(0xffffffff);
5418     scale(0x0);
5419     disp($off);
5420   %}
5421 %}
5422 
5423 operand indOffL16(iRegP reg, immLoffset16 off)
5424 %{
5425   constraint(ALLOC_IN_RC(ptr_reg));
5426   match(AddP reg off);
5427   op_cost(0);
5428   format %{ "[$reg, $off]" %}
5429   interface(MEMORY_INTER) %{
5430     base($reg);
5431     index(0xffffffff);
5432     scale(0x0);
5433     disp($off);
5434   %}
5435 %}
5436 
5437 operand indirectN(iRegN reg)
5438 %{
5439   predicate(CompressedOops::shift() == 0);
5440   constraint(ALLOC_IN_RC(ptr_reg));
5441   match(DecodeN reg);
5442   op_cost(0);
5443   format %{ "[$reg]\t# narrow" %}
5444   interface(MEMORY_INTER) %{
5445     base($reg);
5446     index(0xffffffff);
5447     scale(0x0);
5448     disp(0x0);
5449   %}
5450 %}
5451 
5452 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5453 %{
5454   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5455   constraint(ALLOC_IN_RC(ptr_reg));
5456   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5457   op_cost(0);
5458   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5459   interface(MEMORY_INTER) %{
5460     base($reg);
5461     index($ireg);
5462     scale($scale);
5463     disp(0x0);
5464   %}
5465 %}
5466 
5467 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5468 %{
5469   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5470   constraint(ALLOC_IN_RC(ptr_reg));
5471   match(AddP (DecodeN reg) (LShiftL lreg scale));
5472   op_cost(0);
5473   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5474   interface(MEMORY_INTER) %{
5475     base($reg);
5476     index($lreg);
5477     scale($scale);
5478     disp(0x0);
5479   %}
5480 %}
5481 
5482 operand indIndexI2LN(iRegN reg, iRegI ireg)
5483 %{
5484   predicate(CompressedOops::shift() == 0);
5485   constraint(ALLOC_IN_RC(ptr_reg));
5486   match(AddP (DecodeN reg) (ConvI2L ireg));
5487   op_cost(0);
5488   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5489   interface(MEMORY_INTER) %{
5490     base($reg);
5491     index($ireg);
5492     scale(0x0);
5493     disp(0x0);
5494   %}
5495 %}
5496 
5497 operand indIndexN(iRegN reg, iRegL lreg)
5498 %{
5499   predicate(CompressedOops::shift() == 0);
5500   constraint(ALLOC_IN_RC(ptr_reg));
5501   match(AddP (DecodeN reg) lreg);
5502   op_cost(0);
5503   format %{ "$reg, $lreg\t# narrow" %}
5504   interface(MEMORY_INTER) %{
5505     base($reg);
5506     index($lreg);
5507     scale(0x0);
5508     disp(0x0);
5509   %}
5510 %}
5511 
5512 operand indOffIN(iRegN reg, immIOffset off)
5513 %{
5514   predicate(CompressedOops::shift() == 0);
5515   constraint(ALLOC_IN_RC(ptr_reg));
5516   match(AddP (DecodeN reg) off);
5517   op_cost(0);
5518   format %{ "[$reg, $off]\t# narrow" %}
5519   interface(MEMORY_INTER) %{
5520     base($reg);
5521     index(0xffffffff);
5522     scale(0x0);
5523     disp($off);
5524   %}
5525 %}
5526 
5527 operand indOffLN(iRegN reg, immLoffset off)
5528 %{
5529   predicate(CompressedOops::shift() == 0);
5530   constraint(ALLOC_IN_RC(ptr_reg));
5531   match(AddP (DecodeN reg) off);
5532   op_cost(0);
5533   format %{ "[$reg, $off]\t# narrow" %}
5534   interface(MEMORY_INTER) %{
5535     base($reg);
5536     index(0xffffffff);
5537     scale(0x0);
5538     disp($off);
5539   %}
5540 %}
5541 
5542 
5543 
5544 // AArch64 opto stubs need to write to the pc slot in the thread anchor
5545 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
5546 %{
5547   constraint(ALLOC_IN_RC(ptr_reg));
5548   match(AddP reg off);
5549   op_cost(0);
5550   format %{ "[$reg, $off]" %}
5551   interface(MEMORY_INTER) %{
5552     base($reg);
5553     index(0xffffffff);
5554     scale(0x0);
5555     disp($off);
5556   %}
5557 %}
5558 
5559 //----------Special Memory Operands--------------------------------------------
5560 // Stack Slot Operand - This operand is used for loading and storing temporary
5561 //                      values on the stack where a match requires a value to
5562 //                      flow through memory.
5563 operand stackSlotP(sRegP reg)
5564 %{
5565   constraint(ALLOC_IN_RC(stack_slots));
5566   op_cost(100);
5567   // No match rule because this operand is only generated in matching
5568   // match(RegP);
5569   format %{ "[$reg]" %}
5570   interface(MEMORY_INTER) %{
5571     base(0x1e);  // RSP
5572     index(0x0);  // No Index
5573     scale(0x0);  // No Scale
5574     disp($reg);  // Stack Offset
5575   %}
5576 %}
5577 
5578 operand stackSlotI(sRegI reg)
5579 %{
5580   constraint(ALLOC_IN_RC(stack_slots));
5581   // No match rule because this operand is only generated in matching
5582   // match(RegI);
5583   format %{ "[$reg]" %}
5584   interface(MEMORY_INTER) %{
5585     base(0x1e);  // RSP
5586     index(0x0);  // No Index
5587     scale(0x0);  // No Scale
5588     disp($reg);  // Stack Offset
5589   %}
5590 %}
5591 
5592 operand stackSlotF(sRegF reg)
5593 %{
5594   constraint(ALLOC_IN_RC(stack_slots));
5595   // No match rule because this operand is only generated in matching
5596   // match(RegF);
5597   format %{ "[$reg]" %}
5598   interface(MEMORY_INTER) %{
5599     base(0x1e);  // RSP
5600     index(0x0);  // No Index
5601     scale(0x0);  // No Scale
5602     disp($reg);  // Stack Offset
5603   %}
5604 %}
5605 
5606 operand stackSlotD(sRegD reg)
5607 %{
5608   constraint(ALLOC_IN_RC(stack_slots));
5609   // No match rule because this operand is only generated in matching
5610   // match(RegD);
5611   format %{ "[$reg]" %}
5612   interface(MEMORY_INTER) %{
5613     base(0x1e);  // RSP
5614     index(0x0);  // No Index
5615     scale(0x0);  // No Scale
5616     disp($reg);  // Stack Offset
5617   %}
5618 %}
5619 
5620 operand stackSlotL(sRegL reg)
5621 %{
5622   constraint(ALLOC_IN_RC(stack_slots));
5623   // No match rule because this operand is only generated in matching
5624   // match(RegL);
5625   format %{ "[$reg]" %}
5626   interface(MEMORY_INTER) %{
5627     base(0x1e);  // RSP
5628     index(0x0);  // No Index
5629     scale(0x0);  // No Scale
5630     disp($reg);  // Stack Offset
5631   %}
5632 %}
5633 
5634 // Operands for expressing Control Flow
5635 // NOTE: Label is a predefined operand which should not be redefined in
5636 //       the AD file. It is generically handled within the ADLC.
5637 
5638 //----------Conditional Branch Operands----------------------------------------
5639 // Comparison Op  - This is the operation of the comparison, and is limited to
5640 //                  the following set of codes:
5641 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5642 //
5643 // Other attributes of the comparison, such as unsignedness, are specified
5644 // by the comparison instruction that sets a condition code flags register.
5645 // That result is represented by a flags operand whose subtype is appropriate
5646 // to the unsignedness (etc.) of the comparison.
5647 //
5648 // Later, the instruction which matches both the Comparison Op (a Bool) and
5649 // the flags (produced by the Cmp) specifies the coding of the comparison op
5650 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5651 
5652 // used for signed integral comparisons and fp comparisons
5653 
5654 operand cmpOp()
5655 %{
5656   match(Bool);
5657 
5658   format %{ "" %}
5659   interface(COND_INTER) %{
5660     equal(0x0, "eq");
5661     not_equal(0x1, "ne");
5662     less(0xb, "lt");
5663     greater_equal(0xa, "ge");
5664     less_equal(0xd, "le");
5665     greater(0xc, "gt");
5666     overflow(0x6, "vs");
5667     no_overflow(0x7, "vc");
5668   %}
5669 %}
5670 
5671 // used for unsigned integral comparisons
5672 
5673 operand cmpOpU()
5674 %{
5675   match(Bool);
5676 
5677   format %{ "" %}
5678   interface(COND_INTER) %{
5679     equal(0x0, "eq");
5680     not_equal(0x1, "ne");
5681     less(0x3, "lo");
5682     greater_equal(0x2, "hs");
5683     less_equal(0x9, "ls");
5684     greater(0x8, "hi");
5685     overflow(0x6, "vs");
5686     no_overflow(0x7, "vc");
5687   %}
5688 %}
5689 
5690 // used for certain integral comparisons which can be
5691 // converted to cbxx or tbxx instructions
5692 
5693 operand cmpOpEqNe()
5694 %{
5695   match(Bool);
5696   match(CmpOp);
5697   op_cost(0);
5698   predicate(n->as_Bool()->_test._test == BoolTest::ne
5699             || n->as_Bool()->_test._test == BoolTest::eq);
5700 
5701   format %{ "" %}
5702   interface(COND_INTER) %{
5703     equal(0x0, "eq");
5704     not_equal(0x1, "ne");
5705     less(0xb, "lt");
5706     greater_equal(0xa, "ge");
5707     less_equal(0xd, "le");
5708     greater(0xc, "gt");
5709     overflow(0x6, "vs");
5710     no_overflow(0x7, "vc");
5711   %}
5712 %}
5713 
5714 // used for certain integral comparisons which can be
5715 // converted to cbxx or tbxx instructions
5716 
5717 operand cmpOpLtGe()
5718 %{
5719   match(Bool);
5720   match(CmpOp);
5721   op_cost(0);
5722 
5723   predicate(n->as_Bool()->_test._test == BoolTest::lt
5724             || n->as_Bool()->_test._test == BoolTest::ge);
5725 
5726   format %{ "" %}
5727   interface(COND_INTER) %{
5728     equal(0x0, "eq");
5729     not_equal(0x1, "ne");
5730     less(0xb, "lt");
5731     greater_equal(0xa, "ge");
5732     less_equal(0xd, "le");
5733     greater(0xc, "gt");
5734     overflow(0x6, "vs");
5735     no_overflow(0x7, "vc");
5736   %}
5737 %}
5738 
5739 // used for certain unsigned integral comparisons which can be
5740 // converted to cbxx or tbxx instructions
5741 
5742 operand cmpOpUEqNeLtGe()
5743 %{
5744   match(Bool);
5745   match(CmpOp);
5746   op_cost(0);
5747 
5748   predicate(n->as_Bool()->_test._test == BoolTest::eq
5749             || n->as_Bool()->_test._test == BoolTest::ne
5750             || n->as_Bool()->_test._test == BoolTest::lt
5751             || n->as_Bool()->_test._test == BoolTest::ge);
5752 
5753   format %{ "" %}
5754   interface(COND_INTER) %{
5755     equal(0x0, "eq");
5756     not_equal(0x1, "ne");
5757     less(0xb, "lt");
5758     greater_equal(0xa, "ge");
5759     less_equal(0xd, "le");
5760     greater(0xc, "gt");
5761     overflow(0x6, "vs");
5762     no_overflow(0x7, "vc");
5763   %}
5764 %}
5765 
5766 // Special operand allowing long args to int ops to be truncated for free
5767 
5768 operand iRegL2I(iRegL reg) %{
5769 
5770   op_cost(0);
5771 
5772   match(ConvL2I reg);
5773 
5774   format %{ "l2i($reg)" %}
5775 
5776   interface(REG_INTER)
5777 %}
5778 
5779 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5780 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5781 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5782 
5783 //----------OPERAND CLASSES----------------------------------------------------
5784 // Operand Classes are groups of operands that are used as to simplify
5785 // instruction definitions by not requiring the AD writer to specify
5786 // separate instructions for every form of operand when the
5787 // instruction accepts multiple operand types with the same basic
5788 // encoding and format. The classic case of this is memory operands.
5789 
5790 // memory is used to define read/write location for load/store
5791 // instruction defs. we can turn a memory op into an Address
5792 
5793 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI, indOffL,
5794                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
5795 
5796 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5797 // operations. it allows the src to be either an iRegI or a (ConvL2I
5798 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5799 // can be elided because the 32-bit instruction will just employ the
5800 // lower 32 bits anyway.
5801 //
5802 // n.b. this does not elide all L2I conversions. if the truncated
5803 // value is consumed by more than one operation then the ConvL2I
5804 // cannot be bundled into the consuming nodes so an l2i gets planted
5805 // (actually a movw $dst $src) and the downstream instructions consume
5806 // the result of the l2i as an iRegI input. That's a shame since the
5807 // movw is actually redundant but its not too costly.
5808 
5809 opclass iRegIorL2I(iRegI, iRegL2I);
5810 
5811 //----------PIPELINE-----------------------------------------------------------
5812 // Rules which define the behavior of the target architectures pipeline.
5813 
5814 // For specific pipelines, eg A53, define the stages of that pipeline
5815 //pipe_desc(ISS, EX1, EX2, WR);
5816 #define ISS S0
5817 #define EX1 S1
5818 #define EX2 S2
5819 #define WR  S3
5820 
5821 // Integer ALU reg operation
5822 pipeline %{
5823 
5824 attributes %{
5825   // ARM instructions are of fixed length
5826   fixed_size_instructions;        // Fixed size instructions TODO does
5827   max_instructions_per_bundle = 2;   // A53 = 2, A57 = 4
5828   // ARM instructions come in 32-bit word units
5829   instruction_unit_size = 4;         // An instruction is 4 bytes long
5830   instruction_fetch_unit_size = 64;  // The processor fetches one line
5831   instruction_fetch_units = 1;       // of 64 bytes
5832 
5833   // List of nop instructions
5834   nops( MachNop );
5835 %}
5836 
5837 // We don't use an actual pipeline model so don't care about resources
5838 // or description. we do use pipeline classes to introduce fixed
5839 // latencies
5840 
5841 //----------RESOURCES----------------------------------------------------------
5842 // Resources are the functional units available to the machine
5843 
5844 resources( INS0, INS1, INS01 = INS0 | INS1,
5845            ALU0, ALU1, ALU = ALU0 | ALU1,
5846            MAC,
5847            DIV,
5848            BRANCH,
5849            LDST,
5850            NEON_FP);
5851 
5852 //----------PIPELINE DESCRIPTION-----------------------------------------------
5853 // Pipeline Description specifies the stages in the machine's pipeline
5854 
5855 // Define the pipeline as a generic 6 stage pipeline
5856 pipe_desc(S0, S1, S2, S3, S4, S5);
5857 
5858 //----------PIPELINE CLASSES---------------------------------------------------
5859 // Pipeline Classes describe the stages in which input and output are
5860 // referenced by the hardware pipeline.
5861 
5862 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5863 %{
5864   single_instruction;
5865   src1   : S1(read);
5866   src2   : S2(read);
5867   dst    : S5(write);
5868   INS01  : ISS;
5869   NEON_FP : S5;
5870 %}
5871 
5872 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5873 %{
5874   single_instruction;
5875   src1   : S1(read);
5876   src2   : S2(read);
5877   dst    : S5(write);
5878   INS01  : ISS;
5879   NEON_FP : S5;
5880 %}
5881 
5882 pipe_class fp_uop_s(vRegF dst, vRegF src)
5883 %{
5884   single_instruction;
5885   src    : S1(read);
5886   dst    : S5(write);
5887   INS01  : ISS;
5888   NEON_FP : S5;
5889 %}
5890 
5891 pipe_class fp_uop_d(vRegD dst, vRegD src)
5892 %{
5893   single_instruction;
5894   src    : S1(read);
5895   dst    : S5(write);
5896   INS01  : ISS;
5897   NEON_FP : S5;
5898 %}
5899 
5900 pipe_class fp_d2f(vRegF dst, vRegD src)
5901 %{
5902   single_instruction;
5903   src    : S1(read);
5904   dst    : S5(write);
5905   INS01  : ISS;
5906   NEON_FP : S5;
5907 %}
5908 
5909 pipe_class fp_f2d(vRegD dst, vRegF src)
5910 %{
5911   single_instruction;
5912   src    : S1(read);
5913   dst    : S5(write);
5914   INS01  : ISS;
5915   NEON_FP : S5;
5916 %}
5917 
5918 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5919 %{
5920   single_instruction;
5921   src    : S1(read);
5922   dst    : S5(write);
5923   INS01  : ISS;
5924   NEON_FP : S5;
5925 %}
5926 
5927 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5928 %{
5929   single_instruction;
5930   src    : S1(read);
5931   dst    : S5(write);
5932   INS01  : ISS;
5933   NEON_FP : S5;
5934 %}
5935 
5936 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5937 %{
5938   single_instruction;
5939   src    : S1(read);
5940   dst    : S5(write);
5941   INS01  : ISS;
5942   NEON_FP : S5;
5943 %}
5944 
5945 pipe_class fp_l2f(vRegF dst, iRegL src)
5946 %{
5947   single_instruction;
5948   src    : S1(read);
5949   dst    : S5(write);
5950   INS01  : ISS;
5951   NEON_FP : S5;
5952 %}
5953 
5954 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5955 %{
5956   single_instruction;
5957   src    : S1(read);
5958   dst    : S5(write);
5959   INS01  : ISS;
5960   NEON_FP : S5;
5961 %}
5962 
5963 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5964 %{
5965   single_instruction;
5966   src    : S1(read);
5967   dst    : S5(write);
5968   INS01  : ISS;
5969   NEON_FP : S5;
5970 %}
5971 
5972 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5973 %{
5974   single_instruction;
5975   src    : S1(read);
5976   dst    : S5(write);
5977   INS01  : ISS;
5978   NEON_FP : S5;
5979 %}
5980 
5981 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5982 %{
5983   single_instruction;
5984   src    : S1(read);
5985   dst    : S5(write);
5986   INS01  : ISS;
5987   NEON_FP : S5;
5988 %}
5989 
5990 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
5991 %{
5992   single_instruction;
5993   src1   : S1(read);
5994   src2   : S2(read);
5995   dst    : S5(write);
5996   INS0   : ISS;
5997   NEON_FP : S5;
5998 %}
5999 
6000 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6001 %{
6002   single_instruction;
6003   src1   : S1(read);
6004   src2   : S2(read);
6005   dst    : S5(write);
6006   INS0   : ISS;
6007   NEON_FP : S5;
6008 %}
6009 
6010 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6011 %{
6012   single_instruction;
6013   cr     : S1(read);
6014   src1   : S1(read);
6015   src2   : S1(read);
6016   dst    : S3(write);
6017   INS01  : ISS;
6018   NEON_FP : S3;
6019 %}
6020 
6021 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6022 %{
6023   single_instruction;
6024   cr     : S1(read);
6025   src1   : S1(read);
6026   src2   : S1(read);
6027   dst    : S3(write);
6028   INS01  : ISS;
6029   NEON_FP : S3;
6030 %}
6031 
6032 pipe_class fp_imm_s(vRegF dst)
6033 %{
6034   single_instruction;
6035   dst    : S3(write);
6036   INS01  : ISS;
6037   NEON_FP : S3;
6038 %}
6039 
6040 pipe_class fp_imm_d(vRegD dst)
6041 %{
6042   single_instruction;
6043   dst    : S3(write);
6044   INS01  : ISS;
6045   NEON_FP : S3;
6046 %}
6047 
6048 pipe_class fp_load_constant_s(vRegF dst)
6049 %{
6050   single_instruction;
6051   dst    : S4(write);
6052   INS01  : ISS;
6053   NEON_FP : S4;
6054 %}
6055 
6056 pipe_class fp_load_constant_d(vRegD dst)
6057 %{
6058   single_instruction;
6059   dst    : S4(write);
6060   INS01  : ISS;
6061   NEON_FP : S4;
6062 %}
6063 
6064 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
6065 %{
6066   single_instruction;
6067   dst    : S5(write);
6068   src1   : S1(read);
6069   src2   : S1(read);
6070   INS01  : ISS;
6071   NEON_FP : S5;
6072 %}
6073 
6074 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
6075 %{
6076   single_instruction;
6077   dst    : S5(write);
6078   src1   : S1(read);
6079   src2   : S1(read);
6080   INS0   : ISS;
6081   NEON_FP : S5;
6082 %}
6083 
6084 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
6085 %{
6086   single_instruction;
6087   dst    : S5(write);
6088   src1   : S1(read);
6089   src2   : S1(read);
6090   dst    : S1(read);
6091   INS01  : ISS;
6092   NEON_FP : S5;
6093 %}
6094 
6095 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
6096 %{
6097   single_instruction;
6098   dst    : S5(write);
6099   src1   : S1(read);
6100   src2   : S1(read);
6101   dst    : S1(read);
6102   INS0   : ISS;
6103   NEON_FP : S5;
6104 %}
6105 
6106 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
6107 %{
6108   single_instruction;
6109   dst    : S4(write);
6110   src1   : S2(read);
6111   src2   : S2(read);
6112   INS01  : ISS;
6113   NEON_FP : S4;
6114 %}
6115 
6116 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
6117 %{
6118   single_instruction;
6119   dst    : S4(write);
6120   src1   : S2(read);
6121   src2   : S2(read);
6122   INS0   : ISS;
6123   NEON_FP : S4;
6124 %}
6125 
6126 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
6127 %{
6128   single_instruction;
6129   dst    : S3(write);
6130   src1   : S2(read);
6131   src2   : S2(read);
6132   INS01  : ISS;
6133   NEON_FP : S3;
6134 %}
6135 
6136 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
6137 %{
6138   single_instruction;
6139   dst    : S3(write);
6140   src1   : S2(read);
6141   src2   : S2(read);
6142   INS0   : ISS;
6143   NEON_FP : S3;
6144 %}
6145 
6146 pipe_class vshift64(vecD dst, vecD src, vecX shift)
6147 %{
6148   single_instruction;
6149   dst    : S3(write);
6150   src    : S1(read);
6151   shift  : S1(read);
6152   INS01  : ISS;
6153   NEON_FP : S3;
6154 %}
6155 
6156 pipe_class vshift128(vecX dst, vecX src, vecX shift)
6157 %{
6158   single_instruction;
6159   dst    : S3(write);
6160   src    : S1(read);
6161   shift  : S1(read);
6162   INS0   : ISS;
6163   NEON_FP : S3;
6164 %}
6165 
6166 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
6167 %{
6168   single_instruction;
6169   dst    : S3(write);
6170   src    : S1(read);
6171   INS01  : ISS;
6172   NEON_FP : S3;
6173 %}
6174 
6175 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
6176 %{
6177   single_instruction;
6178   dst    : S3(write);
6179   src    : S1(read);
6180   INS0   : ISS;
6181   NEON_FP : S3;
6182 %}
6183 
6184 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
6185 %{
6186   single_instruction;
6187   dst    : S5(write);
6188   src1   : S1(read);
6189   src2   : S1(read);
6190   INS01  : ISS;
6191   NEON_FP : S5;
6192 %}
6193 
6194 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
6195 %{
6196   single_instruction;
6197   dst    : S5(write);
6198   src1   : S1(read);
6199   src2   : S1(read);
6200   INS0   : ISS;
6201   NEON_FP : S5;
6202 %}
6203 
6204 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
6205 %{
6206   single_instruction;
6207   dst    : S5(write);
6208   src1   : S1(read);
6209   src2   : S1(read);
6210   INS0   : ISS;
6211   NEON_FP : S5;
6212 %}
6213 
6214 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
6215 %{
6216   single_instruction;
6217   dst    : S5(write);
6218   src1   : S1(read);
6219   src2   : S1(read);
6220   INS0   : ISS;
6221   NEON_FP : S5;
6222 %}
6223 
6224 pipe_class vsqrt_fp128(vecX dst, vecX src)
6225 %{
6226   single_instruction;
6227   dst    : S5(write);
6228   src    : S1(read);
6229   INS0   : ISS;
6230   NEON_FP : S5;
6231 %}
6232 
6233 pipe_class vunop_fp64(vecD dst, vecD src)
6234 %{
6235   single_instruction;
6236   dst    : S5(write);
6237   src    : S1(read);
6238   INS01  : ISS;
6239   NEON_FP : S5;
6240 %}
6241 
6242 pipe_class vunop_fp128(vecX dst, vecX src)
6243 %{
6244   single_instruction;
6245   dst    : S5(write);
6246   src    : S1(read);
6247   INS0   : ISS;
6248   NEON_FP : S5;
6249 %}
6250 
6251 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
6252 %{
6253   single_instruction;
6254   dst    : S3(write);
6255   src    : S1(read);
6256   INS01  : ISS;
6257   NEON_FP : S3;
6258 %}
6259 
6260 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
6261 %{
6262   single_instruction;
6263   dst    : S3(write);
6264   src    : S1(read);
6265   INS01  : ISS;
6266   NEON_FP : S3;
6267 %}
6268 
6269 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
6270 %{
6271   single_instruction;
6272   dst    : S3(write);
6273   src    : S1(read);
6274   INS01  : ISS;
6275   NEON_FP : S3;
6276 %}
6277 
6278 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
6279 %{
6280   single_instruction;
6281   dst    : S3(write);
6282   src    : S1(read);
6283   INS01  : ISS;
6284   NEON_FP : S3;
6285 %}
6286 
6287 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
6288 %{
6289   single_instruction;
6290   dst    : S3(write);
6291   src    : S1(read);
6292   INS01  : ISS;
6293   NEON_FP : S3;
6294 %}
6295 
6296 pipe_class vmovi_reg_imm64(vecD dst)
6297 %{
6298   single_instruction;
6299   dst    : S3(write);
6300   INS01  : ISS;
6301   NEON_FP : S3;
6302 %}
6303 
6304 pipe_class vmovi_reg_imm128(vecX dst)
6305 %{
6306   single_instruction;
6307   dst    : S3(write);
6308   INS0   : ISS;
6309   NEON_FP : S3;
6310 %}
6311 
6312 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
6313 %{
6314   single_instruction;
6315   dst    : S5(write);
6316   mem    : ISS(read);
6317   INS01  : ISS;
6318   NEON_FP : S3;
6319 %}
6320 
6321 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
6322 %{
6323   single_instruction;
6324   dst    : S5(write);
6325   mem    : ISS(read);
6326   INS01  : ISS;
6327   NEON_FP : S3;
6328 %}
6329 
6330 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
6331 %{
6332   single_instruction;
6333   mem    : ISS(read);
6334   src    : S2(read);
6335   INS01  : ISS;
6336   NEON_FP : S3;
6337 %}
6338 
6339 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
6340 %{
6341   single_instruction;
6342   mem    : ISS(read);
6343   src    : S2(read);
6344   INS01  : ISS;
6345   NEON_FP : S3;
6346 %}
6347 
6348 //------- Integer ALU operations --------------------------
6349 
6350 // Integer ALU reg-reg operation
6351 // Operands needed in EX1, result generated in EX2
6352 // Eg.  ADD     x0, x1, x2
6353 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6354 %{
6355   single_instruction;
6356   dst    : EX2(write);
6357   src1   : EX1(read);
6358   src2   : EX1(read);
6359   INS01  : ISS; // Dual issue as instruction 0 or 1
6360   ALU    : EX2;
6361 %}
6362 
6363 // Integer ALU reg-reg operation with constant shift
6364 // Shifted register must be available in LATE_ISS instead of EX1
6365 // Eg.  ADD     x0, x1, x2, LSL #2
6366 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6367 %{
6368   single_instruction;
6369   dst    : EX2(write);
6370   src1   : EX1(read);
6371   src2   : ISS(read);
6372   INS01  : ISS;
6373   ALU    : EX2;
6374 %}
6375 
6376 // Integer ALU reg operation with constant shift
6377 // Eg.  LSL     x0, x1, #shift
6378 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6379 %{
6380   single_instruction;
6381   dst    : EX2(write);
6382   src1   : ISS(read);
6383   INS01  : ISS;
6384   ALU    : EX2;
6385 %}
6386 
6387 // Integer ALU reg-reg operation with variable shift
6388 // Both operands must be available in LATE_ISS instead of EX1
6389 // Result is available in EX1 instead of EX2
6390 // Eg.  LSLV    x0, x1, x2
6391 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6392 %{
6393   single_instruction;
6394   dst    : EX1(write);
6395   src1   : ISS(read);
6396   src2   : ISS(read);
6397   INS01  : ISS;
6398   ALU    : EX1;
6399 %}
6400 
6401 // Integer ALU reg-reg operation with extract
6402 // As for _vshift above, but result generated in EX2
6403 // Eg.  EXTR    x0, x1, x2, #N
6404 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6405 %{
6406   single_instruction;
6407   dst    : EX2(write);
6408   src1   : ISS(read);
6409   src2   : ISS(read);
6410   INS1   : ISS; // Can only dual issue as Instruction 1
6411   ALU    : EX1;
6412 %}
6413 
6414 // Integer ALU reg operation
6415 // Eg.  NEG     x0, x1
6416 pipe_class ialu_reg(iRegI dst, iRegI src)
6417 %{
6418   single_instruction;
6419   dst    : EX2(write);
6420   src    : EX1(read);
6421   INS01  : ISS;
6422   ALU    : EX2;
6423 %}
6424 
6425 // Integer ALU reg mmediate operation
6426 // Eg.  ADD     x0, x1, #N
6427 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6428 %{
6429   single_instruction;
6430   dst    : EX2(write);
6431   src1   : EX1(read);
6432   INS01  : ISS;
6433   ALU    : EX2;
6434 %}
6435 
6436 // Integer ALU immediate operation (no source operands)
6437 // Eg.  MOV     x0, #N
6438 pipe_class ialu_imm(iRegI dst)
6439 %{
6440   single_instruction;
6441   dst    : EX1(write);
6442   INS01  : ISS;
6443   ALU    : EX1;
6444 %}
6445 
6446 //------- Compare operation -------------------------------
6447 
6448 // Compare reg-reg
6449 // Eg.  CMP     x0, x1
6450 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6451 %{
6452   single_instruction;
6453 //  fixed_latency(16);
6454   cr     : EX2(write);
6455   op1    : EX1(read);
6456   op2    : EX1(read);
6457   INS01  : ISS;
6458   ALU    : EX2;
6459 %}
6460 
6461 // Compare reg-reg
6462 // Eg.  CMP     x0, #N
6463 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6464 %{
6465   single_instruction;
6466 //  fixed_latency(16);
6467   cr     : EX2(write);
6468   op1    : EX1(read);
6469   INS01  : ISS;
6470   ALU    : EX2;
6471 %}
6472 
6473 //------- Conditional instructions ------------------------
6474 
6475 // Conditional no operands
6476 // Eg.  CSINC   x0, zr, zr, <cond>
6477 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6478 %{
6479   single_instruction;
6480   cr     : EX1(read);
6481   dst    : EX2(write);
6482   INS01  : ISS;
6483   ALU    : EX2;
6484 %}
6485 
6486 // Conditional 2 operand
6487 // EG.  CSEL    X0, X1, X2, <cond>
6488 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6489 %{
6490   single_instruction;
6491   cr     : EX1(read);
6492   src1   : EX1(read);
6493   src2   : EX1(read);
6494   dst    : EX2(write);
6495   INS01  : ISS;
6496   ALU    : EX2;
6497 %}
6498 
6499 // Conditional 2 operand
6500 // EG.  CSEL    X0, X1, X2, <cond>
6501 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6502 %{
6503   single_instruction;
6504   cr     : EX1(read);
6505   src    : EX1(read);
6506   dst    : EX2(write);
6507   INS01  : ISS;
6508   ALU    : EX2;
6509 %}
6510 
6511 //------- Multiply pipeline operations --------------------
6512 
6513 // Multiply reg-reg
6514 // Eg.  MUL     w0, w1, w2
6515 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6516 %{
6517   single_instruction;
6518   dst    : WR(write);
6519   src1   : ISS(read);
6520   src2   : ISS(read);
6521   INS01  : ISS;
6522   MAC    : WR;
6523 %}
6524 
6525 // Multiply accumulate
6526 // Eg.  MADD    w0, w1, w2, w3
6527 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6528 %{
6529   single_instruction;
6530   dst    : WR(write);
6531   src1   : ISS(read);
6532   src2   : ISS(read);
6533   src3   : ISS(read);
6534   INS01  : ISS;
6535   MAC    : WR;
6536 %}
6537 
6538 // Eg.  MUL     w0, w1, w2
6539 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6540 %{
6541   single_instruction;
6542   fixed_latency(3); // Maximum latency for 64 bit mul
6543   dst    : WR(write);
6544   src1   : ISS(read);
6545   src2   : ISS(read);
6546   INS01  : ISS;
6547   MAC    : WR;
6548 %}
6549 
6550 // Multiply accumulate
6551 // Eg.  MADD    w0, w1, w2, w3
6552 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6553 %{
6554   single_instruction;
6555   fixed_latency(3); // Maximum latency for 64 bit mul
6556   dst    : WR(write);
6557   src1   : ISS(read);
6558   src2   : ISS(read);
6559   src3   : ISS(read);
6560   INS01  : ISS;
6561   MAC    : WR;
6562 %}
6563 
6564 //------- Divide pipeline operations --------------------
6565 
6566 // Eg.  SDIV    w0, w1, w2
6567 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6568 %{
6569   single_instruction;
6570   fixed_latency(8); // Maximum latency for 32 bit divide
6571   dst    : WR(write);
6572   src1   : ISS(read);
6573   src2   : ISS(read);
6574   INS0   : ISS; // Can only dual issue as instruction 0
6575   DIV    : WR;
6576 %}
6577 
6578 // Eg.  SDIV    x0, x1, x2
6579 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6580 %{
6581   single_instruction;
6582   fixed_latency(16); // Maximum latency for 64 bit divide
6583   dst    : WR(write);
6584   src1   : ISS(read);
6585   src2   : ISS(read);
6586   INS0   : ISS; // Can only dual issue as instruction 0
6587   DIV    : WR;
6588 %}
6589 
6590 //------- Load pipeline operations ------------------------
6591 
6592 // Load - prefetch
6593 // Eg.  PFRM    <mem>
6594 pipe_class iload_prefetch(memory mem)
6595 %{
6596   single_instruction;
6597   mem    : ISS(read);
6598   INS01  : ISS;
6599   LDST   : WR;
6600 %}
6601 
6602 // Load - reg, mem
6603 // Eg.  LDR     x0, <mem>
6604 pipe_class iload_reg_mem(iRegI dst, memory mem)
6605 %{
6606   single_instruction;
6607   dst    : WR(write);
6608   mem    : ISS(read);
6609   INS01  : ISS;
6610   LDST   : WR;
6611 %}
6612 
6613 // Load - reg, reg
6614 // Eg.  LDR     x0, [sp, x1]
6615 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6616 %{
6617   single_instruction;
6618   dst    : WR(write);
6619   src    : ISS(read);
6620   INS01  : ISS;
6621   LDST   : WR;
6622 %}
6623 
6624 //------- Store pipeline operations -----------------------
6625 
6626 // Store - zr, mem
6627 // Eg.  STR     zr, <mem>
6628 pipe_class istore_mem(memory mem)
6629 %{
6630   single_instruction;
6631   mem    : ISS(read);
6632   INS01  : ISS;
6633   LDST   : WR;
6634 %}
6635 
6636 // Store - reg, mem
6637 // Eg.  STR     x0, <mem>
6638 pipe_class istore_reg_mem(iRegI src, memory mem)
6639 %{
6640   single_instruction;
6641   mem    : ISS(read);
6642   src    : EX2(read);
6643   INS01  : ISS;
6644   LDST   : WR;
6645 %}
6646 
6647 // Store - reg, reg
6648 // Eg. STR      x0, [sp, x1]
6649 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6650 %{
6651   single_instruction;
6652   dst    : ISS(read);
6653   src    : EX2(read);
6654   INS01  : ISS;
6655   LDST   : WR;
6656 %}
6657 
6658 //------- Store pipeline operations -----------------------
6659 
6660 // Branch
6661 pipe_class pipe_branch()
6662 %{
6663   single_instruction;
6664   INS01  : ISS;
6665   BRANCH : EX1;
6666 %}
6667 
6668 // Conditional branch
6669 pipe_class pipe_branch_cond(rFlagsReg cr)
6670 %{
6671   single_instruction;
6672   cr     : EX1(read);
6673   INS01  : ISS;
6674   BRANCH : EX1;
6675 %}
6676 
6677 // Compare & Branch
6678 // EG.  CBZ/CBNZ
6679 pipe_class pipe_cmp_branch(iRegI op1)
6680 %{
6681   single_instruction;
6682   op1    : EX1(read);
6683   INS01  : ISS;
6684   BRANCH : EX1;
6685 %}
6686 
6687 //------- Synchronisation operations ----------------------
6688 
6689 // Any operation requiring serialization.
6690 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
6691 pipe_class pipe_serial()
6692 %{
6693   single_instruction;
6694   force_serialization;
6695   fixed_latency(16);
6696   INS01  : ISS(2); // Cannot dual issue with any other instruction
6697   LDST   : WR;
6698 %}
6699 
6700 // Generic big/slow expanded idiom - also serialized
6701 pipe_class pipe_slow()
6702 %{
6703   instruction_count(10);
6704   multiple_bundles;
6705   force_serialization;
6706   fixed_latency(16);
6707   INS01  : ISS(2); // Cannot dual issue with any other instruction
6708   LDST   : WR;
6709 %}
6710 
6711 // Empty pipeline class
6712 pipe_class pipe_class_empty()
6713 %{
6714   single_instruction;
6715   fixed_latency(0);
6716 %}
6717 
6718 // Default pipeline class.
6719 pipe_class pipe_class_default()
6720 %{
6721   single_instruction;
6722   fixed_latency(2);
6723 %}
6724 
6725 // Pipeline class for compares.
6726 pipe_class pipe_class_compare()
6727 %{
6728   single_instruction;
6729   fixed_latency(16);
6730 %}
6731 
6732 // Pipeline class for memory operations.
6733 pipe_class pipe_class_memory()
6734 %{
6735   single_instruction;
6736   fixed_latency(16);
6737 %}
6738 
6739 // Pipeline class for call.
6740 pipe_class pipe_class_call()
6741 %{
6742   single_instruction;
6743   fixed_latency(100);
6744 %}
6745 
6746 // Define the class for the Nop node.
6747 define %{
6748    MachNop = pipe_class_empty;
6749 %}
6750 
6751 %}
6752 //----------INSTRUCTIONS-------------------------------------------------------
6753 //
6754 // match      -- States which machine-independent subtree may be replaced
6755 //               by this instruction.
6756 // ins_cost   -- The estimated cost of this instruction is used by instruction
6757 //               selection to identify a minimum cost tree of machine
6758 //               instructions that matches a tree of machine-independent
6759 //               instructions.
6760 // format     -- A string providing the disassembly for this instruction.
6761 //               The value of an instruction's operand may be inserted
6762 //               by referring to it with a '$' prefix.
6763 // opcode     -- Three instruction opcodes may be provided.  These are referred
6764 //               to within an encode class as $primary, $secondary, and $tertiary
6765 //               rrspectively.  The primary opcode is commonly used to
6766 //               indicate the type of machine instruction, while secondary
6767 //               and tertiary are often used for prefix options or addressing
6768 //               modes.
6769 // ins_encode -- A list of encode classes with parameters. The encode class
6770 //               name must have been defined in an 'enc_class' specification
6771 //               in the encode section of the architecture description.
6772 
6773 // ============================================================================
6774 // Memory (Load/Store) Instructions
6775 
6776 // Load Instructions
6777 
6778 // Load Byte (8 bit signed)
6779 instruct loadB(iRegINoSp dst, memory mem)
6780 %{
6781   match(Set dst (LoadB mem));
6782   predicate(!needs_acquiring_load(n));
6783 
6784   ins_cost(4 * INSN_COST);
6785   format %{ "ldrsbw  $dst, $mem\t# byte" %}
6786 
6787   ins_encode(aarch64_enc_ldrsbw(dst, mem));
6788 
6789   ins_pipe(iload_reg_mem);
6790 %}
6791 
6792 // Load Byte (8 bit signed) into long
6793 instruct loadB2L(iRegLNoSp dst, memory mem)
6794 %{
6795   match(Set dst (ConvI2L (LoadB mem)));
6796   predicate(!needs_acquiring_load(n->in(1)));
6797 
6798   ins_cost(4 * INSN_COST);
6799   format %{ "ldrsb  $dst, $mem\t# byte" %}
6800 
6801   ins_encode(aarch64_enc_ldrsb(dst, mem));
6802 
6803   ins_pipe(iload_reg_mem);
6804 %}
6805 
6806 // Load Byte (8 bit unsigned)
6807 instruct loadUB(iRegINoSp dst, memory mem)
6808 %{
6809   match(Set dst (LoadUB mem));
6810   predicate(!needs_acquiring_load(n));
6811 
6812   ins_cost(4 * INSN_COST);
6813   format %{ "ldrbw  $dst, $mem\t# byte" %}
6814 
6815   ins_encode(aarch64_enc_ldrb(dst, mem));
6816 
6817   ins_pipe(iload_reg_mem);
6818 %}
6819 
6820 // Load Byte (8 bit unsigned) into long
6821 instruct loadUB2L(iRegLNoSp dst, memory mem)
6822 %{
6823   match(Set dst (ConvI2L (LoadUB mem)));
6824   predicate(!needs_acquiring_load(n->in(1)));
6825 
6826   ins_cost(4 * INSN_COST);
6827   format %{ "ldrb  $dst, $mem\t# byte" %}
6828 
6829   ins_encode(aarch64_enc_ldrb(dst, mem));
6830 
6831   ins_pipe(iload_reg_mem);
6832 %}
6833 
6834 // Load Short (16 bit signed)
6835 instruct loadS(iRegINoSp dst, memory mem)
6836 %{
6837   match(Set dst (LoadS mem));
6838   predicate(!needs_acquiring_load(n));
6839 
6840   ins_cost(4 * INSN_COST);
6841   format %{ "ldrshw  $dst, $mem\t# short" %}
6842 
6843   ins_encode(aarch64_enc_ldrshw(dst, mem));
6844 
6845   ins_pipe(iload_reg_mem);
6846 %}
6847 
6848 // Load Short (16 bit signed) into long
6849 instruct loadS2L(iRegLNoSp dst, memory mem)
6850 %{
6851   match(Set dst (ConvI2L (LoadS mem)));
6852   predicate(!needs_acquiring_load(n->in(1)));
6853 
6854   ins_cost(4 * INSN_COST);
6855   format %{ "ldrsh  $dst, $mem\t# short" %}
6856 
6857   ins_encode(aarch64_enc_ldrsh(dst, mem));
6858 
6859   ins_pipe(iload_reg_mem);
6860 %}
6861 
6862 // Load Char (16 bit unsigned)
6863 instruct loadUS(iRegINoSp dst, memory mem)
6864 %{
6865   match(Set dst (LoadUS mem));
6866   predicate(!needs_acquiring_load(n));
6867 
6868   ins_cost(4 * INSN_COST);
6869   format %{ "ldrh  $dst, $mem\t# short" %}
6870 
6871   ins_encode(aarch64_enc_ldrh(dst, mem));
6872 
6873   ins_pipe(iload_reg_mem);
6874 %}
6875 
6876 // Load Short/Char (16 bit unsigned) into long
6877 instruct loadUS2L(iRegLNoSp dst, memory mem)
6878 %{
6879   match(Set dst (ConvI2L (LoadUS mem)));
6880   predicate(!needs_acquiring_load(n->in(1)));
6881 
6882   ins_cost(4 * INSN_COST);
6883   format %{ "ldrh  $dst, $mem\t# short" %}
6884 
6885   ins_encode(aarch64_enc_ldrh(dst, mem));
6886 
6887   ins_pipe(iload_reg_mem);
6888 %}
6889 
6890 // Load Integer (32 bit signed)
6891 instruct loadI(iRegINoSp dst, memory mem)
6892 %{
6893   match(Set dst (LoadI mem));
6894   predicate(!needs_acquiring_load(n));
6895 
6896   ins_cost(4 * INSN_COST);
6897   format %{ "ldrw  $dst, $mem\t# int" %}
6898 
6899   ins_encode(aarch64_enc_ldrw(dst, mem));
6900 
6901   ins_pipe(iload_reg_mem);
6902 %}
6903 
6904 // Load Integer (32 bit signed) into long
6905 instruct loadI2L(iRegLNoSp dst, memory mem)
6906 %{
6907   match(Set dst (ConvI2L (LoadI mem)));
6908   predicate(!needs_acquiring_load(n->in(1)));
6909 
6910   ins_cost(4 * INSN_COST);
6911   format %{ "ldrsw  $dst, $mem\t# int" %}
6912 
6913   ins_encode(aarch64_enc_ldrsw(dst, mem));
6914 
6915   ins_pipe(iload_reg_mem);
6916 %}
6917 
6918 // Load Integer (32 bit unsigned) into long
6919 instruct loadUI2L(iRegLNoSp dst, memory mem, immL_32bits mask)
6920 %{
6921   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6922   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6923 
6924   ins_cost(4 * INSN_COST);
6925   format %{ "ldrw  $dst, $mem\t# int" %}
6926 
6927   ins_encode(aarch64_enc_ldrw(dst, mem));
6928 
6929   ins_pipe(iload_reg_mem);
6930 %}
6931 
6932 // Load Long (64 bit signed)
6933 instruct loadL(iRegLNoSp dst, memory mem)
6934 %{
6935   match(Set dst (LoadL mem));
6936   predicate(!needs_acquiring_load(n));
6937 
6938   ins_cost(4 * INSN_COST);
6939   format %{ "ldr  $dst, $mem\t# int" %}
6940 
6941   ins_encode(aarch64_enc_ldr(dst, mem));
6942 
6943   ins_pipe(iload_reg_mem);
6944 %}
6945 
6946 // Load Range
6947 instruct loadRange(iRegINoSp dst, memory mem)
6948 %{
6949   match(Set dst (LoadRange mem));
6950 
6951   ins_cost(4 * INSN_COST);
6952   format %{ "ldrw  $dst, $mem\t# range" %}
6953 
6954   ins_encode(aarch64_enc_ldrw(dst, mem));
6955 
6956   ins_pipe(iload_reg_mem);
6957 %}
6958 
6959 // Load Pointer
6960 instruct loadP(iRegPNoSp dst, memory mem)
6961 %{
6962   match(Set dst (LoadP mem));
6963   predicate(!needs_acquiring_load(n));
6964 
6965   ins_cost(4 * INSN_COST);
6966   format %{ "ldr  $dst, $mem\t# ptr" %}
6967 
6968   ins_encode(aarch64_enc_ldr(dst, mem));
6969 
6970   ins_pipe(iload_reg_mem);
6971 %}
6972 
6973 // Load Compressed Pointer
6974 instruct loadN(iRegNNoSp dst, memory mem)
6975 %{
6976   match(Set dst (LoadN mem));
6977   predicate(!needs_acquiring_load(n));
6978 
6979   ins_cost(4 * INSN_COST);
6980   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
6981 
6982   ins_encode(aarch64_enc_ldrw(dst, mem));
6983 
6984   ins_pipe(iload_reg_mem);
6985 %}
6986 
6987 // Load Klass Pointer
6988 instruct loadKlass(iRegPNoSp dst, memory mem)
6989 %{
6990   match(Set dst (LoadKlass mem));
6991   predicate(!needs_acquiring_load(n));
6992 
6993   ins_cost(4 * INSN_COST);
6994   format %{ "ldr  $dst, $mem\t# class" %}
6995 
6996   ins_encode(aarch64_enc_ldr(dst, mem));
6997 
6998   ins_pipe(iload_reg_mem);
6999 %}
7000 
7001 // Load Narrow Klass Pointer
7002 instruct loadNKlass(iRegNNoSp dst, memory mem)
7003 %{
7004   match(Set dst (LoadNKlass mem));
7005   predicate(!needs_acquiring_load(n));
7006 
7007   ins_cost(4 * INSN_COST);
7008   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
7009 
7010   ins_encode(aarch64_enc_ldrw(dst, mem));
7011 
7012   ins_pipe(iload_reg_mem);
7013 %}
7014 
7015 // Load Float
7016 instruct loadF(vRegF dst, memory mem)
7017 %{
7018   match(Set dst (LoadF mem));
7019   predicate(!needs_acquiring_load(n));
7020 
7021   ins_cost(4 * INSN_COST);
7022   format %{ "ldrs  $dst, $mem\t# float" %}
7023 
7024   ins_encode( aarch64_enc_ldrs(dst, mem) );
7025 
7026   ins_pipe(pipe_class_memory);
7027 %}
7028 
7029 // Load Double
7030 instruct loadD(vRegD dst, memory mem)
7031 %{
7032   match(Set dst (LoadD mem));
7033   predicate(!needs_acquiring_load(n));
7034 
7035   ins_cost(4 * INSN_COST);
7036   format %{ "ldrd  $dst, $mem\t# double" %}
7037 
7038   ins_encode( aarch64_enc_ldrd(dst, mem) );
7039 
7040   ins_pipe(pipe_class_memory);
7041 %}
7042 
7043 
7044 // Load Int Constant
7045 instruct loadConI(iRegINoSp dst, immI src)
7046 %{
7047   match(Set dst src);
7048 
7049   ins_cost(INSN_COST);
7050   format %{ "mov $dst, $src\t# int" %}
7051 
7052   ins_encode( aarch64_enc_movw_imm(dst, src) );
7053 
7054   ins_pipe(ialu_imm);
7055 %}
7056 
7057 // Load Long Constant
7058 instruct loadConL(iRegLNoSp dst, immL src)
7059 %{
7060   match(Set dst src);
7061 
7062   ins_cost(INSN_COST);
7063   format %{ "mov $dst, $src\t# long" %}
7064 
7065   ins_encode( aarch64_enc_mov_imm(dst, src) );
7066 
7067   ins_pipe(ialu_imm);
7068 %}
7069 
7070 // Load Pointer Constant
7071 
7072 instruct loadConP(iRegPNoSp dst, immP con)
7073 %{
7074   match(Set dst con);
7075 
7076   ins_cost(INSN_COST * 4);
7077   format %{
7078     "mov  $dst, $con\t# ptr\n\t"
7079   %}
7080 
7081   ins_encode(aarch64_enc_mov_p(dst, con));
7082 
7083   ins_pipe(ialu_imm);
7084 %}
7085 
7086 // Load Null Pointer Constant
7087 
7088 instruct loadConP0(iRegPNoSp dst, immP0 con)
7089 %{
7090   match(Set dst con);
7091 
7092   ins_cost(INSN_COST);
7093   format %{ "mov  $dst, $con\t# NULL ptr" %}
7094 
7095   ins_encode(aarch64_enc_mov_p0(dst, con));
7096 
7097   ins_pipe(ialu_imm);
7098 %}
7099 
7100 // Load Pointer Constant One
7101 
7102 instruct loadConP1(iRegPNoSp dst, immP_1 con)
7103 %{
7104   match(Set dst con);
7105 
7106   ins_cost(INSN_COST);
7107   format %{ "mov  $dst, $con\t# NULL ptr" %}
7108 
7109   ins_encode(aarch64_enc_mov_p1(dst, con));
7110 
7111   ins_pipe(ialu_imm);
7112 %}
7113 
7114 // Load Poll Page Constant
7115 
7116 instruct loadConPollPage(iRegPNoSp dst, immPollPage con)
7117 %{
7118   match(Set dst con);
7119 
7120   ins_cost(INSN_COST);
7121   format %{ "adr  $dst, $con\t# Poll Page Ptr" %}
7122 
7123   ins_encode(aarch64_enc_mov_poll_page(dst, con));
7124 
7125   ins_pipe(ialu_imm);
7126 %}
7127 
7128 // Load Byte Map Base Constant
7129 
7130 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
7131 %{
7132   match(Set dst con);
7133 
7134   ins_cost(INSN_COST);
7135   format %{ "adr  $dst, $con\t# Byte Map Base" %}
7136 
7137   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
7138 
7139   ins_pipe(ialu_imm);
7140 %}
7141 
7142 // Load Narrow Pointer Constant
7143 
7144 instruct loadConN(iRegNNoSp dst, immN con)
7145 %{
7146   match(Set dst con);
7147 
7148   ins_cost(INSN_COST * 4);
7149   format %{ "mov  $dst, $con\t# compressed ptr" %}
7150 
7151   ins_encode(aarch64_enc_mov_n(dst, con));
7152 
7153   ins_pipe(ialu_imm);
7154 %}
7155 
7156 // Load Narrow Null Pointer Constant
7157 
7158 instruct loadConN0(iRegNNoSp dst, immN0 con)
7159 %{
7160   match(Set dst con);
7161 
7162   ins_cost(INSN_COST);
7163   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
7164 
7165   ins_encode(aarch64_enc_mov_n0(dst, con));
7166 
7167   ins_pipe(ialu_imm);
7168 %}
7169 
7170 // Load Narrow Klass Constant
7171 
7172 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7173 %{
7174   match(Set dst con);
7175 
7176   ins_cost(INSN_COST);
7177   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
7178 
7179   ins_encode(aarch64_enc_mov_nk(dst, con));
7180 
7181   ins_pipe(ialu_imm);
7182 %}
7183 
7184 // Load Packed Float Constant
7185 
7186 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7187   match(Set dst con);
7188   ins_cost(INSN_COST * 4);
7189   format %{ "fmovs  $dst, $con"%}
7190   ins_encode %{
7191     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7192   %}
7193 
7194   ins_pipe(fp_imm_s);
7195 %}
7196 
7197 // Load Float Constant
7198 
7199 instruct loadConF(vRegF dst, immF con) %{
7200   match(Set dst con);
7201 
7202   ins_cost(INSN_COST * 4);
7203 
7204   format %{
7205     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7206   %}
7207 
7208   ins_encode %{
7209     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7210   %}
7211 
7212   ins_pipe(fp_load_constant_s);
7213 %}
7214 
7215 // Load Packed Double Constant
7216 
7217 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7218   match(Set dst con);
7219   ins_cost(INSN_COST);
7220   format %{ "fmovd  $dst, $con"%}
7221   ins_encode %{
7222     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7223   %}
7224 
7225   ins_pipe(fp_imm_d);
7226 %}
7227 
7228 // Load Double Constant
7229 
7230 instruct loadConD(vRegD dst, immD con) %{
7231   match(Set dst con);
7232 
7233   ins_cost(INSN_COST * 5);
7234   format %{
7235     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7236   %}
7237 
7238   ins_encode %{
7239     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7240   %}
7241 
7242   ins_pipe(fp_load_constant_d);
7243 %}
7244 
7245 // Store Instructions
7246 
7247 // Store CMS card-mark Immediate
7248 instruct storeimmCM0(immI0 zero, memory mem)
7249 %{
7250   match(Set mem (StoreCM mem zero));
7251   predicate(unnecessary_storestore(n));
7252 
7253   ins_cost(INSN_COST);
7254   format %{ "storestore (elided)\n\t"
7255             "strb zr, $mem\t# byte" %}
7256 
7257   ins_encode(aarch64_enc_strb0(mem));
7258 
7259   ins_pipe(istore_mem);
7260 %}
7261 
7262 // Store CMS card-mark Immediate with intervening StoreStore
7263 // needed when using CMS with no conditional card marking
7264 instruct storeimmCM0_ordered(immI0 zero, memory mem)
7265 %{
7266   match(Set mem (StoreCM mem zero));
7267 
7268   ins_cost(INSN_COST * 2);
7269   format %{ "storestore\n\t"
7270             "dmb ishst"
7271             "\n\tstrb zr, $mem\t# byte" %}
7272 
7273   ins_encode(aarch64_enc_strb0_ordered(mem));
7274 
7275   ins_pipe(istore_mem);
7276 %}
7277 
7278 // Store Byte
7279 instruct storeB(iRegIorL2I src, memory mem)
7280 %{
7281   match(Set mem (StoreB mem src));
7282   predicate(!needs_releasing_store(n));
7283 
7284   ins_cost(INSN_COST);
7285   format %{ "strb  $src, $mem\t# byte" %}
7286 
7287   ins_encode(aarch64_enc_strb(src, mem));
7288 
7289   ins_pipe(istore_reg_mem);
7290 %}
7291 
7292 
7293 instruct storeimmB0(immI0 zero, memory mem)
7294 %{
7295   match(Set mem (StoreB mem zero));
7296   predicate(!needs_releasing_store(n));
7297 
7298   ins_cost(INSN_COST);
7299   format %{ "strb rscractch2, $mem\t# byte" %}
7300 
7301   ins_encode(aarch64_enc_strb0(mem));
7302 
7303   ins_pipe(istore_mem);
7304 %}
7305 
7306 // Store Char/Short
7307 instruct storeC(iRegIorL2I src, memory mem)
7308 %{
7309   match(Set mem (StoreC mem src));
7310   predicate(!needs_releasing_store(n));
7311 
7312   ins_cost(INSN_COST);
7313   format %{ "strh  $src, $mem\t# short" %}
7314 
7315   ins_encode(aarch64_enc_strh(src, mem));
7316 
7317   ins_pipe(istore_reg_mem);
7318 %}
7319 
7320 instruct storeimmC0(immI0 zero, memory mem)
7321 %{
7322   match(Set mem (StoreC mem zero));
7323   predicate(!needs_releasing_store(n));
7324 
7325   ins_cost(INSN_COST);
7326   format %{ "strh  zr, $mem\t# short" %}
7327 
7328   ins_encode(aarch64_enc_strh0(mem));
7329 
7330   ins_pipe(istore_mem);
7331 %}
7332 
7333 // Store Integer
7334 
7335 instruct storeI(iRegIorL2I src, memory mem)
7336 %{
7337   match(Set mem(StoreI mem src));
7338   predicate(!needs_releasing_store(n));
7339 
7340   ins_cost(INSN_COST);
7341   format %{ "strw  $src, $mem\t# int" %}
7342 
7343   ins_encode(aarch64_enc_strw(src, mem));
7344 
7345   ins_pipe(istore_reg_mem);
7346 %}
7347 
7348 instruct storeimmI0(immI0 zero, memory mem)
7349 %{
7350   match(Set mem(StoreI mem zero));
7351   predicate(!needs_releasing_store(n));
7352 
7353   ins_cost(INSN_COST);
7354   format %{ "strw  zr, $mem\t# int" %}
7355 
7356   ins_encode(aarch64_enc_strw0(mem));
7357 
7358   ins_pipe(istore_mem);
7359 %}
7360 
7361 // Store Long (64 bit signed)
7362 instruct storeL(iRegL src, memory mem)
7363 %{
7364   match(Set mem (StoreL mem src));
7365   predicate(!needs_releasing_store(n));
7366 
7367   ins_cost(INSN_COST);
7368   format %{ "str  $src, $mem\t# int" %}
7369 
7370   ins_encode(aarch64_enc_str(src, mem));
7371 
7372   ins_pipe(istore_reg_mem);
7373 %}
7374 
7375 // Store Long (64 bit signed)
7376 instruct storeimmL0(immL0 zero, memory mem)
7377 %{
7378   match(Set mem (StoreL mem zero));
7379   predicate(!needs_releasing_store(n));
7380 
7381   ins_cost(INSN_COST);
7382   format %{ "str  zr, $mem\t# int" %}
7383 
7384   ins_encode(aarch64_enc_str0(mem));
7385 
7386   ins_pipe(istore_mem);
7387 %}
7388 
7389 // Store Pointer
7390 instruct storeP(iRegP src, memory mem)
7391 %{
7392   match(Set mem (StoreP mem src));
7393   predicate(!needs_releasing_store(n));
7394 
7395   ins_cost(INSN_COST);
7396   format %{ "str  $src, $mem\t# ptr" %}
7397 
7398   ins_encode(aarch64_enc_str(src, mem));
7399 
7400   ins_pipe(istore_reg_mem);
7401 %}
7402 
7403 // Store Pointer
7404 instruct storeimmP0(immP0 zero, memory mem)
7405 %{
7406   match(Set mem (StoreP mem zero));
7407   predicate(!needs_releasing_store(n));
7408 
7409   ins_cost(INSN_COST);
7410   format %{ "str zr, $mem\t# ptr" %}
7411 
7412   ins_encode(aarch64_enc_str0(mem));
7413 
7414   ins_pipe(istore_mem);
7415 %}
7416 
7417 // Store Compressed Pointer
7418 instruct storeN(iRegN src, memory mem)
7419 %{
7420   match(Set mem (StoreN mem src));
7421   predicate(!needs_releasing_store(n));
7422 
7423   ins_cost(INSN_COST);
7424   format %{ "strw  $src, $mem\t# compressed ptr" %}
7425 
7426   ins_encode(aarch64_enc_strw(src, mem));
7427 
7428   ins_pipe(istore_reg_mem);
7429 %}
7430 
7431 instruct storeImmN0(iRegIHeapbase heapbase, immN0 zero, memory mem)
7432 %{
7433   match(Set mem (StoreN mem zero));
7434   predicate(CompressedOops::base() == NULL &&
7435             CompressedKlassPointers::base() == NULL &&
7436             (!needs_releasing_store(n)));
7437 
7438   ins_cost(INSN_COST);
7439   format %{ "strw  rheapbase, $mem\t# compressed ptr (rheapbase==0)" %}
7440 
7441   ins_encode(aarch64_enc_strw(heapbase, mem));
7442 
7443   ins_pipe(istore_reg_mem);
7444 %}
7445 
7446 // Store Float
7447 instruct storeF(vRegF src, memory mem)
7448 %{
7449   match(Set mem (StoreF mem src));
7450   predicate(!needs_releasing_store(n));
7451 
7452   ins_cost(INSN_COST);
7453   format %{ "strs  $src, $mem\t# float" %}
7454 
7455   ins_encode( aarch64_enc_strs(src, mem) );
7456 
7457   ins_pipe(pipe_class_memory);
7458 %}
7459 
7460 // TODO
7461 // implement storeImmF0 and storeFImmPacked
7462 
7463 // Store Double
7464 instruct storeD(vRegD src, memory mem)
7465 %{
7466   match(Set mem (StoreD mem src));
7467   predicate(!needs_releasing_store(n));
7468 
7469   ins_cost(INSN_COST);
7470   format %{ "strd  $src, $mem\t# double" %}
7471 
7472   ins_encode( aarch64_enc_strd(src, mem) );
7473 
7474   ins_pipe(pipe_class_memory);
7475 %}
7476 
7477 // Store Compressed Klass Pointer
7478 instruct storeNKlass(iRegN src, memory mem)
7479 %{
7480   predicate(!needs_releasing_store(n));
7481   match(Set mem (StoreNKlass mem src));
7482 
7483   ins_cost(INSN_COST);
7484   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
7485 
7486   ins_encode(aarch64_enc_strw(src, mem));
7487 
7488   ins_pipe(istore_reg_mem);
7489 %}
7490 
7491 // TODO
7492 // implement storeImmD0 and storeDImmPacked
7493 
7494 // prefetch instructions
7495 // Must be safe to execute with invalid address (cannot fault).
7496 
7497 instruct prefetchalloc( memory mem ) %{
7498   match(PrefetchAllocation mem);
7499 
7500   ins_cost(INSN_COST);
7501   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7502 
7503   ins_encode( aarch64_enc_prefetchw(mem) );
7504 
7505   ins_pipe(iload_prefetch);
7506 %}
7507 
7508 //  ---------------- volatile loads and stores ----------------
7509 
7510 // Load Byte (8 bit signed)
7511 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7512 %{
7513   match(Set dst (LoadB mem));
7514 
7515   ins_cost(VOLATILE_REF_COST);
7516   format %{ "ldarsb  $dst, $mem\t# byte" %}
7517 
7518   ins_encode(aarch64_enc_ldarsb(dst, mem));
7519 
7520   ins_pipe(pipe_serial);
7521 %}
7522 
7523 // Load Byte (8 bit signed) into long
7524 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7525 %{
7526   match(Set dst (ConvI2L (LoadB mem)));
7527 
7528   ins_cost(VOLATILE_REF_COST);
7529   format %{ "ldarsb  $dst, $mem\t# byte" %}
7530 
7531   ins_encode(aarch64_enc_ldarsb(dst, mem));
7532 
7533   ins_pipe(pipe_serial);
7534 %}
7535 
7536 // Load Byte (8 bit unsigned)
7537 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7538 %{
7539   match(Set dst (LoadUB mem));
7540 
7541   ins_cost(VOLATILE_REF_COST);
7542   format %{ "ldarb  $dst, $mem\t# byte" %}
7543 
7544   ins_encode(aarch64_enc_ldarb(dst, mem));
7545 
7546   ins_pipe(pipe_serial);
7547 %}
7548 
7549 // Load Byte (8 bit unsigned) into long
7550 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7551 %{
7552   match(Set dst (ConvI2L (LoadUB mem)));
7553 
7554   ins_cost(VOLATILE_REF_COST);
7555   format %{ "ldarb  $dst, $mem\t# byte" %}
7556 
7557   ins_encode(aarch64_enc_ldarb(dst, mem));
7558 
7559   ins_pipe(pipe_serial);
7560 %}
7561 
7562 // Load Short (16 bit signed)
7563 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7564 %{
7565   match(Set dst (LoadS mem));
7566 
7567   ins_cost(VOLATILE_REF_COST);
7568   format %{ "ldarshw  $dst, $mem\t# short" %}
7569 
7570   ins_encode(aarch64_enc_ldarshw(dst, mem));
7571 
7572   ins_pipe(pipe_serial);
7573 %}
7574 
7575 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7576 %{
7577   match(Set dst (LoadUS mem));
7578 
7579   ins_cost(VOLATILE_REF_COST);
7580   format %{ "ldarhw  $dst, $mem\t# short" %}
7581 
7582   ins_encode(aarch64_enc_ldarhw(dst, mem));
7583 
7584   ins_pipe(pipe_serial);
7585 %}
7586 
7587 // Load Short/Char (16 bit unsigned) into long
7588 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7589 %{
7590   match(Set dst (ConvI2L (LoadUS mem)));
7591 
7592   ins_cost(VOLATILE_REF_COST);
7593   format %{ "ldarh  $dst, $mem\t# short" %}
7594 
7595   ins_encode(aarch64_enc_ldarh(dst, mem));
7596 
7597   ins_pipe(pipe_serial);
7598 %}
7599 
7600 // Load Short/Char (16 bit signed) into long
7601 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7602 %{
7603   match(Set dst (ConvI2L (LoadS mem)));
7604 
7605   ins_cost(VOLATILE_REF_COST);
7606   format %{ "ldarh  $dst, $mem\t# short" %}
7607 
7608   ins_encode(aarch64_enc_ldarsh(dst, mem));
7609 
7610   ins_pipe(pipe_serial);
7611 %}
7612 
7613 // Load Integer (32 bit signed)
7614 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7615 %{
7616   match(Set dst (LoadI mem));
7617 
7618   ins_cost(VOLATILE_REF_COST);
7619   format %{ "ldarw  $dst, $mem\t# int" %}
7620 
7621   ins_encode(aarch64_enc_ldarw(dst, mem));
7622 
7623   ins_pipe(pipe_serial);
7624 %}
7625 
7626 // Load Integer (32 bit unsigned) into long
7627 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7628 %{
7629   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7630 
7631   ins_cost(VOLATILE_REF_COST);
7632   format %{ "ldarw  $dst, $mem\t# int" %}
7633 
7634   ins_encode(aarch64_enc_ldarw(dst, mem));
7635 
7636   ins_pipe(pipe_serial);
7637 %}
7638 
7639 // Load Long (64 bit signed)
7640 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7641 %{
7642   match(Set dst (LoadL mem));
7643 
7644   ins_cost(VOLATILE_REF_COST);
7645   format %{ "ldar  $dst, $mem\t# int" %}
7646 
7647   ins_encode(aarch64_enc_ldar(dst, mem));
7648 
7649   ins_pipe(pipe_serial);
7650 %}
7651 
7652 // Load Pointer
7653 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7654 %{
7655   match(Set dst (LoadP mem));
7656 
7657   ins_cost(VOLATILE_REF_COST);
7658   format %{ "ldar  $dst, $mem\t# ptr" %}
7659 
7660   ins_encode(aarch64_enc_ldar(dst, mem));
7661 
7662   ins_pipe(pipe_serial);
7663 %}
7664 
7665 // Load Compressed Pointer
7666 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7667 %{
7668   match(Set dst (LoadN mem));
7669 
7670   ins_cost(VOLATILE_REF_COST);
7671   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
7672 
7673   ins_encode(aarch64_enc_ldarw(dst, mem));
7674 
7675   ins_pipe(pipe_serial);
7676 %}
7677 
7678 // Load Float
7679 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7680 %{
7681   match(Set dst (LoadF mem));
7682 
7683   ins_cost(VOLATILE_REF_COST);
7684   format %{ "ldars  $dst, $mem\t# float" %}
7685 
7686   ins_encode( aarch64_enc_fldars(dst, mem) );
7687 
7688   ins_pipe(pipe_serial);
7689 %}
7690 
7691 // Load Double
7692 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7693 %{
7694   match(Set dst (LoadD mem));
7695 
7696   ins_cost(VOLATILE_REF_COST);
7697   format %{ "ldard  $dst, $mem\t# double" %}
7698 
7699   ins_encode( aarch64_enc_fldard(dst, mem) );
7700 
7701   ins_pipe(pipe_serial);
7702 %}
7703 
7704 // Store Byte
7705 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7706 %{
7707   match(Set mem (StoreB mem src));
7708 
7709   ins_cost(VOLATILE_REF_COST);
7710   format %{ "stlrb  $src, $mem\t# byte" %}
7711 
7712   ins_encode(aarch64_enc_stlrb(src, mem));
7713 
7714   ins_pipe(pipe_class_memory);
7715 %}
7716 
7717 // Store Char/Short
7718 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7719 %{
7720   match(Set mem (StoreC mem src));
7721 
7722   ins_cost(VOLATILE_REF_COST);
7723   format %{ "stlrh  $src, $mem\t# short" %}
7724 
7725   ins_encode(aarch64_enc_stlrh(src, mem));
7726 
7727   ins_pipe(pipe_class_memory);
7728 %}
7729 
7730 // Store Integer
7731 
7732 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7733 %{
7734   match(Set mem(StoreI mem src));
7735 
7736   ins_cost(VOLATILE_REF_COST);
7737   format %{ "stlrw  $src, $mem\t# int" %}
7738 
7739   ins_encode(aarch64_enc_stlrw(src, mem));
7740 
7741   ins_pipe(pipe_class_memory);
7742 %}
7743 
7744 // Store Long (64 bit signed)
7745 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7746 %{
7747   match(Set mem (StoreL mem src));
7748 
7749   ins_cost(VOLATILE_REF_COST);
7750   format %{ "stlr  $src, $mem\t# int" %}
7751 
7752   ins_encode(aarch64_enc_stlr(src, mem));
7753 
7754   ins_pipe(pipe_class_memory);
7755 %}
7756 
7757 // Store Pointer
7758 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7759 %{
7760   match(Set mem (StoreP mem src));
7761 
7762   ins_cost(VOLATILE_REF_COST);
7763   format %{ "stlr  $src, $mem\t# ptr" %}
7764 
7765   ins_encode(aarch64_enc_stlr(src, mem));
7766 
7767   ins_pipe(pipe_class_memory);
7768 %}
7769 
7770 // Store Compressed Pointer
7771 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7772 %{
7773   match(Set mem (StoreN mem src));
7774 
7775   ins_cost(VOLATILE_REF_COST);
7776   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
7777 
7778   ins_encode(aarch64_enc_stlrw(src, mem));
7779 
7780   ins_pipe(pipe_class_memory);
7781 %}
7782 
7783 // Store Float
7784 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7785 %{
7786   match(Set mem (StoreF mem src));
7787 
7788   ins_cost(VOLATILE_REF_COST);
7789   format %{ "stlrs  $src, $mem\t# float" %}
7790 
7791   ins_encode( aarch64_enc_fstlrs(src, mem) );
7792 
7793   ins_pipe(pipe_class_memory);
7794 %}
7795 
7796 // TODO
7797 // implement storeImmF0 and storeFImmPacked
7798 
7799 // Store Double
7800 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7801 %{
7802   match(Set mem (StoreD mem src));
7803 
7804   ins_cost(VOLATILE_REF_COST);
7805   format %{ "stlrd  $src, $mem\t# double" %}
7806 
7807   ins_encode( aarch64_enc_fstlrd(src, mem) );
7808 
7809   ins_pipe(pipe_class_memory);
7810 %}
7811 
7812 //  ---------------- end of volatile loads and stores ----------------
7813 
7814 // ============================================================================
7815 // BSWAP Instructions
7816 
7817 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7818   match(Set dst (ReverseBytesI src));
7819 
7820   ins_cost(INSN_COST);
7821   format %{ "revw  $dst, $src" %}
7822 
7823   ins_encode %{
7824     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7825   %}
7826 
7827   ins_pipe(ialu_reg);
7828 %}
7829 
7830 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7831   match(Set dst (ReverseBytesL src));
7832 
7833   ins_cost(INSN_COST);
7834   format %{ "rev  $dst, $src" %}
7835 
7836   ins_encode %{
7837     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7838   %}
7839 
7840   ins_pipe(ialu_reg);
7841 %}
7842 
7843 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7844   match(Set dst (ReverseBytesUS src));
7845 
7846   ins_cost(INSN_COST);
7847   format %{ "rev16w  $dst, $src" %}
7848 
7849   ins_encode %{
7850     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7851   %}
7852 
7853   ins_pipe(ialu_reg);
7854 %}
7855 
7856 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7857   match(Set dst (ReverseBytesS src));
7858 
7859   ins_cost(INSN_COST);
7860   format %{ "rev16w  $dst, $src\n\t"
7861             "sbfmw $dst, $dst, #0, #15" %}
7862 
7863   ins_encode %{
7864     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7865     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7866   %}
7867 
7868   ins_pipe(ialu_reg);
7869 %}
7870 
7871 // ============================================================================
7872 // Zero Count Instructions
7873 
7874 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7875   match(Set dst (CountLeadingZerosI src));
7876 
7877   ins_cost(INSN_COST);
7878   format %{ "clzw  $dst, $src" %}
7879   ins_encode %{
7880     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7881   %}
7882 
7883   ins_pipe(ialu_reg);
7884 %}
7885 
7886 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7887   match(Set dst (CountLeadingZerosL src));
7888 
7889   ins_cost(INSN_COST);
7890   format %{ "clz   $dst, $src" %}
7891   ins_encode %{
7892     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7893   %}
7894 
7895   ins_pipe(ialu_reg);
7896 %}
7897 
7898 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7899   match(Set dst (CountTrailingZerosI src));
7900 
7901   ins_cost(INSN_COST * 2);
7902   format %{ "rbitw  $dst, $src\n\t"
7903             "clzw   $dst, $dst" %}
7904   ins_encode %{
7905     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7906     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7907   %}
7908 
7909   ins_pipe(ialu_reg);
7910 %}
7911 
7912 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7913   match(Set dst (CountTrailingZerosL src));
7914 
7915   ins_cost(INSN_COST * 2);
7916   format %{ "rbit   $dst, $src\n\t"
7917             "clz    $dst, $dst" %}
7918   ins_encode %{
7919     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7920     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7921   %}
7922 
7923   ins_pipe(ialu_reg);
7924 %}
7925 
7926 //---------- Population Count Instructions -------------------------------------
7927 //
7928 
7929 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7930   predicate(UsePopCountInstruction);
7931   match(Set dst (PopCountI src));
7932   effect(TEMP tmp);
7933   ins_cost(INSN_COST * 13);
7934 
7935   format %{ "movw   $src, $src\n\t"
7936             "mov    $tmp, $src\t# vector (1D)\n\t"
7937             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
7938             "addv   $tmp, $tmp\t# vector (8B)\n\t"
7939             "mov    $dst, $tmp\t# vector (1D)" %}
7940   ins_encode %{
7941     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7942     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
7943     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7944     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7945     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7946   %}
7947 
7948   ins_pipe(pipe_class_default);
7949 %}
7950 
7951 instruct popCountI_mem(iRegINoSp dst, memory mem, vRegF tmp) %{
7952   predicate(UsePopCountInstruction);
7953   match(Set dst (PopCountI (LoadI mem)));
7954   effect(TEMP tmp);
7955   ins_cost(INSN_COST * 13);
7956 
7957   format %{ "ldrs   $tmp, $mem\n\t"
7958             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
7959             "addv   $tmp, $tmp\t# vector (8B)\n\t"
7960             "mov    $dst, $tmp\t# vector (1D)" %}
7961   ins_encode %{
7962     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7963     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7964                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
7965     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7966     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7967     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7968   %}
7969 
7970   ins_pipe(pipe_class_default);
7971 %}
7972 
7973 // Note: Long.bitCount(long) returns an int.
7974 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7975   predicate(UsePopCountInstruction);
7976   match(Set dst (PopCountL src));
7977   effect(TEMP tmp);
7978   ins_cost(INSN_COST * 13);
7979 
7980   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
7981             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
7982             "addv   $tmp, $tmp\t# vector (8B)\n\t"
7983             "mov    $dst, $tmp\t# vector (1D)" %}
7984   ins_encode %{
7985     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
7986     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7987     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7988     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7989   %}
7990 
7991   ins_pipe(pipe_class_default);
7992 %}
7993 
7994 instruct popCountL_mem(iRegINoSp dst, memory mem, vRegD tmp) %{
7995   predicate(UsePopCountInstruction);
7996   match(Set dst (PopCountL (LoadL mem)));
7997   effect(TEMP tmp);
7998   ins_cost(INSN_COST * 13);
7999 
8000   format %{ "ldrd   $tmp, $mem\n\t"
8001             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
8002             "addv   $tmp, $tmp\t# vector (8B)\n\t"
8003             "mov    $dst, $tmp\t# vector (1D)" %}
8004   ins_encode %{
8005     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8006     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
8007                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
8008     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8009     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8010     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
8011   %}
8012 
8013   ins_pipe(pipe_class_default);
8014 %}
8015 
8016 // ============================================================================
8017 // MemBar Instruction
8018 
8019 instruct load_fence() %{
8020   match(LoadFence);
8021   ins_cost(VOLATILE_REF_COST);
8022 
8023   format %{ "load_fence" %}
8024 
8025   ins_encode %{
8026     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8027   %}
8028   ins_pipe(pipe_serial);
8029 %}
8030 
8031 instruct unnecessary_membar_acquire() %{
8032   predicate(unnecessary_acquire(n));
8033   match(MemBarAcquire);
8034   ins_cost(0);
8035 
8036   format %{ "membar_acquire (elided)" %}
8037 
8038   ins_encode %{
8039     __ block_comment("membar_acquire (elided)");
8040   %}
8041 
8042   ins_pipe(pipe_class_empty);
8043 %}
8044 
8045 instruct membar_acquire() %{
8046   match(MemBarAcquire);
8047   ins_cost(VOLATILE_REF_COST);
8048 
8049   format %{ "membar_acquire\n\t"
8050             "dmb ish" %}
8051 
8052   ins_encode %{
8053     __ block_comment("membar_acquire");
8054     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8055   %}
8056 
8057   ins_pipe(pipe_serial);
8058 %}
8059 
8060 
8061 instruct membar_acquire_lock() %{
8062   match(MemBarAcquireLock);
8063   ins_cost(VOLATILE_REF_COST);
8064 
8065   format %{ "membar_acquire_lock (elided)" %}
8066 
8067   ins_encode %{
8068     __ block_comment("membar_acquire_lock (elided)");
8069   %}
8070 
8071   ins_pipe(pipe_serial);
8072 %}
8073 
8074 instruct store_fence() %{
8075   match(StoreFence);
8076   ins_cost(VOLATILE_REF_COST);
8077 
8078   format %{ "store_fence" %}
8079 
8080   ins_encode %{
8081     __ membar(Assembler::LoadStore|Assembler::StoreStore);
8082   %}
8083   ins_pipe(pipe_serial);
8084 %}
8085 
8086 instruct unnecessary_membar_release() %{
8087   predicate(unnecessary_release(n));
8088   match(MemBarRelease);
8089   ins_cost(0);
8090 
8091   format %{ "membar_release (elided)" %}
8092 
8093   ins_encode %{
8094     __ block_comment("membar_release (elided)");
8095   %}
8096   ins_pipe(pipe_serial);
8097 %}
8098 
8099 instruct membar_release() %{
8100   match(MemBarRelease);
8101   ins_cost(VOLATILE_REF_COST);
8102 
8103   format %{ "membar_release\n\t"
8104             "dmb ish" %}
8105 
8106   ins_encode %{
8107     __ block_comment("membar_release");
8108     __ membar(Assembler::LoadStore|Assembler::StoreStore);
8109   %}
8110   ins_pipe(pipe_serial);
8111 %}
8112 
8113 instruct membar_storestore() %{
8114   match(MemBarStoreStore);
8115   ins_cost(VOLATILE_REF_COST);
8116 
8117   format %{ "MEMBAR-store-store" %}
8118 
8119   ins_encode %{
8120     __ membar(Assembler::StoreStore);
8121   %}
8122   ins_pipe(pipe_serial);
8123 %}
8124 
8125 instruct membar_release_lock() %{
8126   match(MemBarReleaseLock);
8127   ins_cost(VOLATILE_REF_COST);
8128 
8129   format %{ "membar_release_lock (elided)" %}
8130 
8131   ins_encode %{
8132     __ block_comment("membar_release_lock (elided)");
8133   %}
8134 
8135   ins_pipe(pipe_serial);
8136 %}
8137 
8138 instruct unnecessary_membar_volatile() %{
8139   predicate(unnecessary_volatile(n));
8140   match(MemBarVolatile);
8141   ins_cost(0);
8142 
8143   format %{ "membar_volatile (elided)" %}
8144 
8145   ins_encode %{
8146     __ block_comment("membar_volatile (elided)");
8147   %}
8148 
8149   ins_pipe(pipe_serial);
8150 %}
8151 
8152 instruct membar_volatile() %{
8153   match(MemBarVolatile);
8154   ins_cost(VOLATILE_REF_COST*100);
8155 
8156   format %{ "membar_volatile\n\t"
8157              "dmb ish"%}
8158 
8159   ins_encode %{
8160     __ block_comment("membar_volatile");
8161     __ membar(Assembler::StoreLoad);
8162   %}
8163 
8164   ins_pipe(pipe_serial);
8165 %}
8166 
8167 // ============================================================================
8168 // Cast/Convert Instructions
8169 
8170 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8171   match(Set dst (CastX2P src));
8172 
8173   ins_cost(INSN_COST);
8174   format %{ "mov $dst, $src\t# long -> ptr" %}
8175 
8176   ins_encode %{
8177     if ($dst$$reg != $src$$reg) {
8178       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8179     }
8180   %}
8181 
8182   ins_pipe(ialu_reg);
8183 %}
8184 
8185 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8186   match(Set dst (CastP2X src));
8187 
8188   ins_cost(INSN_COST);
8189   format %{ "mov $dst, $src\t# ptr -> long" %}
8190 
8191   ins_encode %{
8192     if ($dst$$reg != $src$$reg) {
8193       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8194     }
8195   %}
8196 
8197   ins_pipe(ialu_reg);
8198 %}
8199 
8200 // Convert oop into int for vectors alignment masking
8201 instruct convP2I(iRegINoSp dst, iRegP src) %{
8202   match(Set dst (ConvL2I (CastP2X src)));
8203 
8204   ins_cost(INSN_COST);
8205   format %{ "movw $dst, $src\t# ptr -> int" %}
8206   ins_encode %{
8207     __ movw($dst$$Register, $src$$Register);
8208   %}
8209 
8210   ins_pipe(ialu_reg);
8211 %}
8212 
8213 // Convert compressed oop into int for vectors alignment masking
8214 // in case of 32bit oops (heap < 4Gb).
8215 instruct convN2I(iRegINoSp dst, iRegN src)
8216 %{
8217   predicate(CompressedOops::shift() == 0);
8218   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8219 
8220   ins_cost(INSN_COST);
8221   format %{ "mov dst, $src\t# compressed ptr -> int" %}
8222   ins_encode %{
8223     __ movw($dst$$Register, $src$$Register);
8224   %}
8225 
8226   ins_pipe(ialu_reg);
8227 %}
8228 
8229 
8230 // Convert oop pointer into compressed form
8231 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8232   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8233   match(Set dst (EncodeP src));
8234   effect(KILL cr);
8235   ins_cost(INSN_COST * 3);
8236   format %{ "encode_heap_oop $dst, $src" %}
8237   ins_encode %{
8238     Register s = $src$$Register;
8239     Register d = $dst$$Register;
8240     __ encode_heap_oop(d, s);
8241   %}
8242   ins_pipe(ialu_reg);
8243 %}
8244 
8245 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8246   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8247   match(Set dst (EncodeP src));
8248   ins_cost(INSN_COST * 3);
8249   format %{ "encode_heap_oop_not_null $dst, $src" %}
8250   ins_encode %{
8251     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8252   %}
8253   ins_pipe(ialu_reg);
8254 %}
8255 
8256 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8257   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8258             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8259   match(Set dst (DecodeN src));
8260   ins_cost(INSN_COST * 3);
8261   format %{ "decode_heap_oop $dst, $src" %}
8262   ins_encode %{
8263     Register s = $src$$Register;
8264     Register d = $dst$$Register;
8265     __ decode_heap_oop(d, s);
8266   %}
8267   ins_pipe(ialu_reg);
8268 %}
8269 
8270 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8271   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8272             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8273   match(Set dst (DecodeN src));
8274   ins_cost(INSN_COST * 3);
8275   format %{ "decode_heap_oop_not_null $dst, $src" %}
8276   ins_encode %{
8277     Register s = $src$$Register;
8278     Register d = $dst$$Register;
8279     __ decode_heap_oop_not_null(d, s);
8280   %}
8281   ins_pipe(ialu_reg);
8282 %}
8283 
8284 // n.b. AArch64 implementations of encode_klass_not_null and
8285 // decode_klass_not_null do not modify the flags register so, unlike
8286 // Intel, we don't kill CR as a side effect here
8287 
8288 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8289   match(Set dst (EncodePKlass src));
8290 
8291   ins_cost(INSN_COST * 3);
8292   format %{ "encode_klass_not_null $dst,$src" %}
8293 
8294   ins_encode %{
8295     Register src_reg = as_Register($src$$reg);
8296     Register dst_reg = as_Register($dst$$reg);
8297     __ encode_klass_not_null(dst_reg, src_reg);
8298   %}
8299 
8300    ins_pipe(ialu_reg);
8301 %}
8302 
8303 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8304   match(Set dst (DecodeNKlass src));
8305 
8306   ins_cost(INSN_COST * 3);
8307   format %{ "decode_klass_not_null $dst,$src" %}
8308 
8309   ins_encode %{
8310     Register src_reg = as_Register($src$$reg);
8311     Register dst_reg = as_Register($dst$$reg);
8312     if (dst_reg != src_reg) {
8313       __ decode_klass_not_null(dst_reg, src_reg);
8314     } else {
8315       __ decode_klass_not_null(dst_reg);
8316     }
8317   %}
8318 
8319    ins_pipe(ialu_reg);
8320 %}
8321 
8322 instruct checkCastPP(iRegPNoSp dst)
8323 %{
8324   match(Set dst (CheckCastPP dst));
8325 
8326   size(0);
8327   format %{ "# checkcastPP of $dst" %}
8328   ins_encode(/* empty encoding */);
8329   ins_pipe(pipe_class_empty);
8330 %}
8331 
8332 instruct castPP(iRegPNoSp dst)
8333 %{
8334   match(Set dst (CastPP dst));
8335 
8336   size(0);
8337   format %{ "# castPP of $dst" %}
8338   ins_encode(/* empty encoding */);
8339   ins_pipe(pipe_class_empty);
8340 %}
8341 
8342 instruct castII(iRegI dst)
8343 %{
8344   match(Set dst (CastII dst));
8345 
8346   size(0);
8347   format %{ "# castII of $dst" %}
8348   ins_encode(/* empty encoding */);
8349   ins_cost(0);
8350   ins_pipe(pipe_class_empty);
8351 %}
8352 
8353 // ============================================================================
8354 // Atomic operation instructions
8355 //
8356 // Intel and SPARC both implement Ideal Node LoadPLocked and
8357 // Store{PIL}Conditional instructions using a normal load for the
8358 // LoadPLocked and a CAS for the Store{PIL}Conditional.
8359 //
8360 // The ideal code appears only to use LoadPLocked/StorePLocked as a
8361 // pair to lock object allocations from Eden space when not using
8362 // TLABs.
8363 //
8364 // There does not appear to be a Load{IL}Locked Ideal Node and the
8365 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
8366 // and to use StoreIConditional only for 32-bit and StoreLConditional
8367 // only for 64-bit.
8368 //
8369 // We implement LoadPLocked and StorePLocked instructions using,
8370 // respectively the AArch64 hw load-exclusive and store-conditional
8371 // instructions. Whereas we must implement each of
8372 // Store{IL}Conditional using a CAS which employs a pair of
8373 // instructions comprising a load-exclusive followed by a
8374 // store-conditional.
8375 
8376 
8377 // Locked-load (linked load) of the current heap-top
8378 // used when updating the eden heap top
8379 // implemented using ldaxr on AArch64
8380 
8381 instruct loadPLocked(iRegPNoSp dst, indirect mem)
8382 %{
8383   match(Set dst (LoadPLocked mem));
8384 
8385   ins_cost(VOLATILE_REF_COST);
8386 
8387   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
8388 
8389   ins_encode(aarch64_enc_ldaxr(dst, mem));
8390 
8391   ins_pipe(pipe_serial);
8392 %}
8393 
8394 // Conditional-store of the updated heap-top.
8395 // Used during allocation of the shared heap.
8396 // Sets flag (EQ) on success.
8397 // implemented using stlxr on AArch64.
8398 
8399 instruct storePConditional(memory heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
8400 %{
8401   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
8402 
8403   ins_cost(VOLATILE_REF_COST);
8404 
8405  // TODO
8406  // do we need to do a store-conditional release or can we just use a
8407  // plain store-conditional?
8408 
8409   format %{
8410     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
8411     "cmpw rscratch1, zr\t# EQ on successful write"
8412   %}
8413 
8414   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
8415 
8416   ins_pipe(pipe_serial);
8417 %}
8418 
8419 
8420 // storeLConditional is used by PhaseMacroExpand::expand_lock_node
8421 // when attempting to rebias a lock towards the current thread.  We
8422 // must use the acquire form of cmpxchg in order to guarantee acquire
8423 // semantics in this case.
8424 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
8425 %{
8426   match(Set cr (StoreLConditional mem (Binary oldval newval)));
8427 
8428   ins_cost(VOLATILE_REF_COST);
8429 
8430   format %{
8431     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
8432     "cmpw rscratch1, zr\t# EQ on successful write"
8433   %}
8434 
8435   ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
8436 
8437   ins_pipe(pipe_slow);
8438 %}
8439 
8440 // storeIConditional also has acquire semantics, for no better reason
8441 // than matching storeLConditional.  At the time of writing this
8442 // comment storeIConditional was not used anywhere by AArch64.
8443 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
8444 %{
8445   match(Set cr (StoreIConditional mem (Binary oldval newval)));
8446 
8447   ins_cost(VOLATILE_REF_COST);
8448 
8449   format %{
8450     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
8451     "cmpw rscratch1, zr\t# EQ on successful write"
8452   %}
8453 
8454   ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
8455 
8456   ins_pipe(pipe_slow);
8457 %}
8458 
8459 // standard CompareAndSwapX when we are using barriers
8460 // these have higher priority than the rules selected by a predicate
8461 
8462 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8463 // can't match them
8464 
8465 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8466 
8467   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8468   ins_cost(2 * VOLATILE_REF_COST);
8469 
8470   effect(KILL cr);
8471 
8472   format %{
8473     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8474     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8475   %}
8476 
8477   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8478             aarch64_enc_cset_eq(res));
8479 
8480   ins_pipe(pipe_slow);
8481 %}
8482 
8483 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8484 
8485   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8486   ins_cost(2 * VOLATILE_REF_COST);
8487 
8488   effect(KILL cr);
8489 
8490   format %{
8491     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8492     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8493   %}
8494 
8495   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8496             aarch64_enc_cset_eq(res));
8497 
8498   ins_pipe(pipe_slow);
8499 %}
8500 
8501 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8502 
8503   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8504   ins_cost(2 * VOLATILE_REF_COST);
8505 
8506   effect(KILL cr);
8507 
8508  format %{
8509     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8510     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8511  %}
8512 
8513  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8514             aarch64_enc_cset_eq(res));
8515 
8516   ins_pipe(pipe_slow);
8517 %}
8518 
8519 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8520 
8521   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8522   ins_cost(2 * VOLATILE_REF_COST);
8523 
8524   effect(KILL cr);
8525 
8526  format %{
8527     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8528     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8529  %}
8530 
8531  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8532             aarch64_enc_cset_eq(res));
8533 
8534   ins_pipe(pipe_slow);
8535 %}
8536 
8537 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8538 
8539   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8540   ins_cost(2 * VOLATILE_REF_COST);
8541 
8542   effect(KILL cr);
8543 
8544  format %{
8545     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8546     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8547  %}
8548 
8549  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8550             aarch64_enc_cset_eq(res));
8551 
8552   ins_pipe(pipe_slow);
8553 %}
8554 
8555 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8556 
8557   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8558   ins_cost(2 * VOLATILE_REF_COST);
8559 
8560   effect(KILL cr);
8561 
8562  format %{
8563     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8564     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8565  %}
8566 
8567  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8568             aarch64_enc_cset_eq(res));
8569 
8570   ins_pipe(pipe_slow);
8571 %}
8572 
8573 // alternative CompareAndSwapX when we are eliding barriers
8574 
8575 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8576 
8577   predicate(needs_acquiring_load_exclusive(n));
8578   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8579   ins_cost(VOLATILE_REF_COST);
8580 
8581   effect(KILL cr);
8582 
8583   format %{
8584     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8585     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8586   %}
8587 
8588   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8589             aarch64_enc_cset_eq(res));
8590 
8591   ins_pipe(pipe_slow);
8592 %}
8593 
8594 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8595 
8596   predicate(needs_acquiring_load_exclusive(n));
8597   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8598   ins_cost(VOLATILE_REF_COST);
8599 
8600   effect(KILL cr);
8601 
8602   format %{
8603     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8604     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8605   %}
8606 
8607   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8608             aarch64_enc_cset_eq(res));
8609 
8610   ins_pipe(pipe_slow);
8611 %}
8612 
8613 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8614 
8615   predicate(needs_acquiring_load_exclusive(n));
8616   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8617   ins_cost(VOLATILE_REF_COST);
8618 
8619   effect(KILL cr);
8620 
8621  format %{
8622     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8623     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8624  %}
8625 
8626  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8627             aarch64_enc_cset_eq(res));
8628 
8629   ins_pipe(pipe_slow);
8630 %}
8631 
8632 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8633 
8634   predicate(needs_acquiring_load_exclusive(n));
8635   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8636   ins_cost(VOLATILE_REF_COST);
8637 
8638   effect(KILL cr);
8639 
8640  format %{
8641     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8642     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8643  %}
8644 
8645  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8646             aarch64_enc_cset_eq(res));
8647 
8648   ins_pipe(pipe_slow);
8649 %}
8650 
8651 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8652 
8653   predicate(needs_acquiring_load_exclusive(n));
8654   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8655   ins_cost(VOLATILE_REF_COST);
8656 
8657   effect(KILL cr);
8658 
8659  format %{
8660     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8661     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8662  %}
8663 
8664  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8665             aarch64_enc_cset_eq(res));
8666 
8667   ins_pipe(pipe_slow);
8668 %}
8669 
8670 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8671 
8672   predicate(needs_acquiring_load_exclusive(n));
8673   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8674   ins_cost(VOLATILE_REF_COST);
8675 
8676   effect(KILL cr);
8677 
8678  format %{
8679     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8680     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8681  %}
8682 
8683  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8684             aarch64_enc_cset_eq(res));
8685 
8686   ins_pipe(pipe_slow);
8687 %}
8688 
8689 
8690 // ---------------------------------------------------------------------
8691 
8692 
8693 // BEGIN This section of the file is automatically generated. Do not edit --------------
8694 
8695 // Sundry CAS operations.  Note that release is always true,
8696 // regardless of the memory ordering of the CAS.  This is because we
8697 // need the volatile case to be sequentially consistent but there is
8698 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
8699 // can't check the type of memory ordering here, so we always emit a
8700 // STLXR.
8701 
8702 // This section is generated from aarch64_ad_cas.m4
8703 
8704 
8705 
8706 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8707   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8708   ins_cost(2 * VOLATILE_REF_COST);
8709   effect(TEMP_DEF res, KILL cr);
8710   format %{
8711     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8712   %}
8713   ins_encode %{
8714     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8715                Assembler::byte, /*acquire*/ false, /*release*/ true,
8716                /*weak*/ false, $res$$Register);
8717     __ sxtbw($res$$Register, $res$$Register);
8718   %}
8719   ins_pipe(pipe_slow);
8720 %}
8721 
8722 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8723   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8724   ins_cost(2 * VOLATILE_REF_COST);
8725   effect(TEMP_DEF res, KILL cr);
8726   format %{
8727     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8728   %}
8729   ins_encode %{
8730     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8731                Assembler::halfword, /*acquire*/ false, /*release*/ true,
8732                /*weak*/ false, $res$$Register);
8733     __ sxthw($res$$Register, $res$$Register);
8734   %}
8735   ins_pipe(pipe_slow);
8736 %}
8737 
8738 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8739   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8740   ins_cost(2 * VOLATILE_REF_COST);
8741   effect(TEMP_DEF res, KILL cr);
8742   format %{
8743     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8744   %}
8745   ins_encode %{
8746     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8747                Assembler::word, /*acquire*/ false, /*release*/ true,
8748                /*weak*/ false, $res$$Register);
8749   %}
8750   ins_pipe(pipe_slow);
8751 %}
8752 
8753 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8754   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8755   ins_cost(2 * VOLATILE_REF_COST);
8756   effect(TEMP_DEF res, KILL cr);
8757   format %{
8758     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8759   %}
8760   ins_encode %{
8761     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8762                Assembler::xword, /*acquire*/ false, /*release*/ true,
8763                /*weak*/ false, $res$$Register);
8764   %}
8765   ins_pipe(pipe_slow);
8766 %}
8767 
8768 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8769   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8770   ins_cost(2 * VOLATILE_REF_COST);
8771   effect(TEMP_DEF res, KILL cr);
8772   format %{
8773     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8774   %}
8775   ins_encode %{
8776     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8777                Assembler::word, /*acquire*/ false, /*release*/ true,
8778                /*weak*/ false, $res$$Register);
8779   %}
8780   ins_pipe(pipe_slow);
8781 %}
8782 
8783 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8784   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8785   ins_cost(2 * VOLATILE_REF_COST);
8786   effect(TEMP_DEF res, KILL cr);
8787   format %{
8788     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8789   %}
8790   ins_encode %{
8791     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8792                Assembler::xword, /*acquire*/ false, /*release*/ true,
8793                /*weak*/ false, $res$$Register);
8794   %}
8795   ins_pipe(pipe_slow);
8796 %}
8797 
8798 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8799   predicate(needs_acquiring_load_exclusive(n));
8800   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8801   ins_cost(VOLATILE_REF_COST);
8802   effect(TEMP_DEF res, KILL cr);
8803   format %{
8804     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8805   %}
8806   ins_encode %{
8807     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8808                Assembler::byte, /*acquire*/ true, /*release*/ true,
8809                /*weak*/ false, $res$$Register);
8810     __ sxtbw($res$$Register, $res$$Register);
8811   %}
8812   ins_pipe(pipe_slow);
8813 %}
8814 
8815 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8816   predicate(needs_acquiring_load_exclusive(n));
8817   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8818   ins_cost(VOLATILE_REF_COST);
8819   effect(TEMP_DEF res, KILL cr);
8820   format %{
8821     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8822   %}
8823   ins_encode %{
8824     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8825                Assembler::halfword, /*acquire*/ true, /*release*/ true,
8826                /*weak*/ false, $res$$Register);
8827     __ sxthw($res$$Register, $res$$Register);
8828   %}
8829   ins_pipe(pipe_slow);
8830 %}
8831 
8832 
8833 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8834   predicate(needs_acquiring_load_exclusive(n));
8835   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8836   ins_cost(VOLATILE_REF_COST);
8837   effect(TEMP_DEF res, KILL cr);
8838   format %{
8839     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8840   %}
8841   ins_encode %{
8842     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8843                Assembler::word, /*acquire*/ true, /*release*/ true,
8844                /*weak*/ false, $res$$Register);
8845   %}
8846   ins_pipe(pipe_slow);
8847 %}
8848 
8849 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8850   predicate(needs_acquiring_load_exclusive(n));
8851   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8852   ins_cost(VOLATILE_REF_COST);
8853   effect(TEMP_DEF res, KILL cr);
8854   format %{
8855     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8856   %}
8857   ins_encode %{
8858     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8859                Assembler::xword, /*acquire*/ true, /*release*/ true,
8860                /*weak*/ false, $res$$Register);
8861   %}
8862   ins_pipe(pipe_slow);
8863 %}
8864 
8865 
8866 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8867   predicate(needs_acquiring_load_exclusive(n));
8868   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8869   ins_cost(VOLATILE_REF_COST);
8870   effect(TEMP_DEF res, KILL cr);
8871   format %{
8872     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8873   %}
8874   ins_encode %{
8875     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8876                Assembler::word, /*acquire*/ true, /*release*/ true,
8877                /*weak*/ false, $res$$Register);
8878   %}
8879   ins_pipe(pipe_slow);
8880 %}
8881 
8882 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8883   predicate(needs_acquiring_load_exclusive(n));
8884   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8885   ins_cost(VOLATILE_REF_COST);
8886   effect(TEMP_DEF res, KILL cr);
8887   format %{
8888     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8889   %}
8890   ins_encode %{
8891     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8892                Assembler::xword, /*acquire*/ true, /*release*/ true,
8893                /*weak*/ false, $res$$Register);
8894   %}
8895   ins_pipe(pipe_slow);
8896 %}
8897 
8898 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8899   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8900   ins_cost(2 * VOLATILE_REF_COST);
8901   effect(KILL cr);
8902   format %{
8903     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8904     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8905   %}
8906   ins_encode %{
8907     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8908                Assembler::byte, /*acquire*/ false, /*release*/ true,
8909                /*weak*/ true, noreg);
8910     __ csetw($res$$Register, Assembler::EQ);
8911   %}
8912   ins_pipe(pipe_slow);
8913 %}
8914 
8915 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8916   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8917   ins_cost(2 * VOLATILE_REF_COST);
8918   effect(KILL cr);
8919   format %{
8920     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8921     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8922   %}
8923   ins_encode %{
8924     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8925                Assembler::halfword, /*acquire*/ false, /*release*/ true,
8926                /*weak*/ true, noreg);
8927     __ csetw($res$$Register, Assembler::EQ);
8928   %}
8929   ins_pipe(pipe_slow);
8930 %}
8931 
8932 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8933   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8934   ins_cost(2 * VOLATILE_REF_COST);
8935   effect(KILL cr);
8936   format %{
8937     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8938     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8939   %}
8940   ins_encode %{
8941     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8942                Assembler::word, /*acquire*/ false, /*release*/ true,
8943                /*weak*/ true, noreg);
8944     __ csetw($res$$Register, Assembler::EQ);
8945   %}
8946   ins_pipe(pipe_slow);
8947 %}
8948 
8949 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8950   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8951   ins_cost(2 * VOLATILE_REF_COST);
8952   effect(KILL cr);
8953   format %{
8954     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8955     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8956   %}
8957   ins_encode %{
8958     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8959                Assembler::xword, /*acquire*/ false, /*release*/ true,
8960                /*weak*/ true, noreg);
8961     __ csetw($res$$Register, Assembler::EQ);
8962   %}
8963   ins_pipe(pipe_slow);
8964 %}
8965 
8966 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8967   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8968   ins_cost(2 * VOLATILE_REF_COST);
8969   effect(KILL cr);
8970   format %{
8971     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8972     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8973   %}
8974   ins_encode %{
8975     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8976                Assembler::word, /*acquire*/ false, /*release*/ true,
8977                /*weak*/ true, noreg);
8978     __ csetw($res$$Register, Assembler::EQ);
8979   %}
8980   ins_pipe(pipe_slow);
8981 %}
8982 
8983 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8984   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8985   ins_cost(2 * VOLATILE_REF_COST);
8986   effect(KILL cr);
8987   format %{
8988     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8989     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8990   %}
8991   ins_encode %{
8992     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8993                Assembler::xword, /*acquire*/ false, /*release*/ true,
8994                /*weak*/ true, noreg);
8995     __ csetw($res$$Register, Assembler::EQ);
8996   %}
8997   ins_pipe(pipe_slow);
8998 %}
8999 
9000 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9001   predicate(needs_acquiring_load_exclusive(n));
9002   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9003   ins_cost(VOLATILE_REF_COST);
9004   effect(KILL cr);
9005   format %{
9006     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9007     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9008   %}
9009   ins_encode %{
9010     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9011                Assembler::byte, /*acquire*/ true, /*release*/ true,
9012                /*weak*/ true, noreg);
9013     __ csetw($res$$Register, Assembler::EQ);
9014   %}
9015   ins_pipe(pipe_slow);
9016 %}
9017 
9018 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9019   predicate(needs_acquiring_load_exclusive(n));
9020   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9021   ins_cost(VOLATILE_REF_COST);
9022   effect(KILL cr);
9023   format %{
9024     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9025     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9026   %}
9027   ins_encode %{
9028     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9029                Assembler::halfword, /*acquire*/ true, /*release*/ true,
9030                /*weak*/ true, noreg);
9031     __ csetw($res$$Register, Assembler::EQ);
9032   %}
9033   ins_pipe(pipe_slow);
9034 %}
9035 
9036 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9037   predicate(needs_acquiring_load_exclusive(n));
9038   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9039   ins_cost(VOLATILE_REF_COST);
9040   effect(KILL cr);
9041   format %{
9042     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9043     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9044   %}
9045   ins_encode %{
9046     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9047                Assembler::word, /*acquire*/ true, /*release*/ true,
9048                /*weak*/ true, noreg);
9049     __ csetw($res$$Register, Assembler::EQ);
9050   %}
9051   ins_pipe(pipe_slow);
9052 %}
9053 
9054 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9055   predicate(needs_acquiring_load_exclusive(n));
9056   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9057   ins_cost(VOLATILE_REF_COST);
9058   effect(KILL cr);
9059   format %{
9060     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9061     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9062   %}
9063   ins_encode %{
9064     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9065                Assembler::xword, /*acquire*/ true, /*release*/ true,
9066                /*weak*/ true, noreg);
9067     __ csetw($res$$Register, Assembler::EQ);
9068   %}
9069   ins_pipe(pipe_slow);
9070 %}
9071 
9072 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9073   predicate(needs_acquiring_load_exclusive(n));
9074   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9075   ins_cost(VOLATILE_REF_COST);
9076   effect(KILL cr);
9077   format %{
9078     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9079     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9080   %}
9081   ins_encode %{
9082     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9083                Assembler::word, /*acquire*/ true, /*release*/ true,
9084                /*weak*/ true, noreg);
9085     __ csetw($res$$Register, Assembler::EQ);
9086   %}
9087   ins_pipe(pipe_slow);
9088 %}
9089 
9090 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9091   predicate(needs_acquiring_load_exclusive(n));
9092   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9093   ins_cost(VOLATILE_REF_COST);
9094   effect(KILL cr);
9095   format %{
9096     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9097     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9098   %}
9099   ins_encode %{
9100     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9101                Assembler::xword, /*acquire*/ true, /*release*/ true,
9102                /*weak*/ true, noreg);
9103     __ csetw($res$$Register, Assembler::EQ);
9104   %}
9105   ins_pipe(pipe_slow);
9106 %}
9107 
9108 // END This section of the file is automatically generated. Do not edit --------------
9109 // ---------------------------------------------------------------------
9110 
9111 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9112   match(Set prev (GetAndSetI mem newv));
9113   ins_cost(2 * VOLATILE_REF_COST);
9114   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
9115   ins_encode %{
9116     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9117   %}
9118   ins_pipe(pipe_serial);
9119 %}
9120 
9121 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9122   match(Set prev (GetAndSetL mem newv));
9123   ins_cost(2 * VOLATILE_REF_COST);
9124   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
9125   ins_encode %{
9126     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9127   %}
9128   ins_pipe(pipe_serial);
9129 %}
9130 
9131 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9132   match(Set prev (GetAndSetN mem newv));
9133   ins_cost(2 * VOLATILE_REF_COST);
9134   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9135   ins_encode %{
9136     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9137   %}
9138   ins_pipe(pipe_serial);
9139 %}
9140 
9141 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9142   match(Set prev (GetAndSetP mem newv));
9143   ins_cost(2 * VOLATILE_REF_COST);
9144   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
9145   ins_encode %{
9146     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9147   %}
9148   ins_pipe(pipe_serial);
9149 %}
9150 
9151 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9152   predicate(needs_acquiring_load_exclusive(n));
9153   match(Set prev (GetAndSetI mem newv));
9154   ins_cost(VOLATILE_REF_COST);
9155   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
9156   ins_encode %{
9157     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9158   %}
9159   ins_pipe(pipe_serial);
9160 %}
9161 
9162 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9163   predicate(needs_acquiring_load_exclusive(n));
9164   match(Set prev (GetAndSetL mem newv));
9165   ins_cost(VOLATILE_REF_COST);
9166   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
9167   ins_encode %{
9168     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9169   %}
9170   ins_pipe(pipe_serial);
9171 %}
9172 
9173 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9174   predicate(needs_acquiring_load_exclusive(n));
9175   match(Set prev (GetAndSetN mem newv));
9176   ins_cost(VOLATILE_REF_COST);
9177   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9178   ins_encode %{
9179     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9180   %}
9181   ins_pipe(pipe_serial);
9182 %}
9183 
9184 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9185   predicate(needs_acquiring_load_exclusive(n));
9186   match(Set prev (GetAndSetP mem newv));
9187   ins_cost(VOLATILE_REF_COST);
9188   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
9189   ins_encode %{
9190     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9191   %}
9192   ins_pipe(pipe_serial);
9193 %}
9194 
9195 
9196 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9197   match(Set newval (GetAndAddL mem incr));
9198   ins_cost(2 * VOLATILE_REF_COST + 1);
9199   format %{ "get_and_addL $newval, [$mem], $incr" %}
9200   ins_encode %{
9201     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9202   %}
9203   ins_pipe(pipe_serial);
9204 %}
9205 
9206 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9207   predicate(n->as_LoadStore()->result_not_used());
9208   match(Set dummy (GetAndAddL mem incr));
9209   ins_cost(2 * VOLATILE_REF_COST);
9210   format %{ "get_and_addL [$mem], $incr" %}
9211   ins_encode %{
9212     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9213   %}
9214   ins_pipe(pipe_serial);
9215 %}
9216 
9217 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9218   match(Set newval (GetAndAddL mem incr));
9219   ins_cost(2 * VOLATILE_REF_COST + 1);
9220   format %{ "get_and_addL $newval, [$mem], $incr" %}
9221   ins_encode %{
9222     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9223   %}
9224   ins_pipe(pipe_serial);
9225 %}
9226 
9227 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9228   predicate(n->as_LoadStore()->result_not_used());
9229   match(Set dummy (GetAndAddL mem incr));
9230   ins_cost(2 * VOLATILE_REF_COST);
9231   format %{ "get_and_addL [$mem], $incr" %}
9232   ins_encode %{
9233     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9234   %}
9235   ins_pipe(pipe_serial);
9236 %}
9237 
9238 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9239   match(Set newval (GetAndAddI mem incr));
9240   ins_cost(2 * VOLATILE_REF_COST + 1);
9241   format %{ "get_and_addI $newval, [$mem], $incr" %}
9242   ins_encode %{
9243     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9244   %}
9245   ins_pipe(pipe_serial);
9246 %}
9247 
9248 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9249   predicate(n->as_LoadStore()->result_not_used());
9250   match(Set dummy (GetAndAddI mem incr));
9251   ins_cost(2 * VOLATILE_REF_COST);
9252   format %{ "get_and_addI [$mem], $incr" %}
9253   ins_encode %{
9254     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9255   %}
9256   ins_pipe(pipe_serial);
9257 %}
9258 
9259 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9260   match(Set newval (GetAndAddI mem incr));
9261   ins_cost(2 * VOLATILE_REF_COST + 1);
9262   format %{ "get_and_addI $newval, [$mem], $incr" %}
9263   ins_encode %{
9264     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9265   %}
9266   ins_pipe(pipe_serial);
9267 %}
9268 
9269 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9270   predicate(n->as_LoadStore()->result_not_used());
9271   match(Set dummy (GetAndAddI mem incr));
9272   ins_cost(2 * VOLATILE_REF_COST);
9273   format %{ "get_and_addI [$mem], $incr" %}
9274   ins_encode %{
9275     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9276   %}
9277   ins_pipe(pipe_serial);
9278 %}
9279 
9280 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9281   predicate(needs_acquiring_load_exclusive(n));
9282   match(Set newval (GetAndAddL mem incr));
9283   ins_cost(VOLATILE_REF_COST + 1);
9284   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9285   ins_encode %{
9286     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9287   %}
9288   ins_pipe(pipe_serial);
9289 %}
9290 
9291 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9292   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9293   match(Set dummy (GetAndAddL mem incr));
9294   ins_cost(VOLATILE_REF_COST);
9295   format %{ "get_and_addL_acq [$mem], $incr" %}
9296   ins_encode %{
9297     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9298   %}
9299   ins_pipe(pipe_serial);
9300 %}
9301 
9302 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9303   predicate(needs_acquiring_load_exclusive(n));
9304   match(Set newval (GetAndAddL mem incr));
9305   ins_cost(VOLATILE_REF_COST + 1);
9306   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9307   ins_encode %{
9308     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9309   %}
9310   ins_pipe(pipe_serial);
9311 %}
9312 
9313 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9314   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9315   match(Set dummy (GetAndAddL mem incr));
9316   ins_cost(VOLATILE_REF_COST);
9317   format %{ "get_and_addL_acq [$mem], $incr" %}
9318   ins_encode %{
9319     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9320   %}
9321   ins_pipe(pipe_serial);
9322 %}
9323 
9324 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9325   predicate(needs_acquiring_load_exclusive(n));
9326   match(Set newval (GetAndAddI mem incr));
9327   ins_cost(VOLATILE_REF_COST + 1);
9328   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9329   ins_encode %{
9330     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9331   %}
9332   ins_pipe(pipe_serial);
9333 %}
9334 
9335 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9336   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9337   match(Set dummy (GetAndAddI mem incr));
9338   ins_cost(VOLATILE_REF_COST);
9339   format %{ "get_and_addI_acq [$mem], $incr" %}
9340   ins_encode %{
9341     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9342   %}
9343   ins_pipe(pipe_serial);
9344 %}
9345 
9346 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9347   predicate(needs_acquiring_load_exclusive(n));
9348   match(Set newval (GetAndAddI mem incr));
9349   ins_cost(VOLATILE_REF_COST + 1);
9350   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9351   ins_encode %{
9352     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9353   %}
9354   ins_pipe(pipe_serial);
9355 %}
9356 
9357 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9358   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9359   match(Set dummy (GetAndAddI mem incr));
9360   ins_cost(VOLATILE_REF_COST);
9361   format %{ "get_and_addI_acq [$mem], $incr" %}
9362   ins_encode %{
9363     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9364   %}
9365   ins_pipe(pipe_serial);
9366 %}
9367 
9368 // Manifest a CmpL result in an integer register.
9369 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9370 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9371 %{
9372   match(Set dst (CmpL3 src1 src2));
9373   effect(KILL flags);
9374 
9375   ins_cost(INSN_COST * 6);
9376   format %{
9377       "cmp $src1, $src2"
9378       "csetw $dst, ne"
9379       "cnegw $dst, lt"
9380   %}
9381   // format %{ "CmpL3 $dst, $src1, $src2" %}
9382   ins_encode %{
9383     __ cmp($src1$$Register, $src2$$Register);
9384     __ csetw($dst$$Register, Assembler::NE);
9385     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9386   %}
9387 
9388   ins_pipe(pipe_class_default);
9389 %}
9390 
9391 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9392 %{
9393   match(Set dst (CmpL3 src1 src2));
9394   effect(KILL flags);
9395 
9396   ins_cost(INSN_COST * 6);
9397   format %{
9398       "cmp $src1, $src2"
9399       "csetw $dst, ne"
9400       "cnegw $dst, lt"
9401   %}
9402   ins_encode %{
9403     int32_t con = (int32_t)$src2$$constant;
9404      if (con < 0) {
9405       __ adds(zr, $src1$$Register, -con);
9406     } else {
9407       __ subs(zr, $src1$$Register, con);
9408     }
9409     __ csetw($dst$$Register, Assembler::NE);
9410     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9411   %}
9412 
9413   ins_pipe(pipe_class_default);
9414 %}
9415 
9416 // ============================================================================
9417 // Conditional Move Instructions
9418 
9419 // n.b. we have identical rules for both a signed compare op (cmpOp)
9420 // and an unsigned compare op (cmpOpU). it would be nice if we could
9421 // define an op class which merged both inputs and use it to type the
9422 // argument to a single rule. unfortunatelyt his fails because the
9423 // opclass does not live up to the COND_INTER interface of its
9424 // component operands. When the generic code tries to negate the
9425 // operand it ends up running the generci Machoper::negate method
9426 // which throws a ShouldNotHappen. So, we have to provide two flavours
9427 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9428 
9429 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9430   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9431 
9432   ins_cost(INSN_COST * 2);
9433   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
9434 
9435   ins_encode %{
9436     __ cselw(as_Register($dst$$reg),
9437              as_Register($src2$$reg),
9438              as_Register($src1$$reg),
9439              (Assembler::Condition)$cmp$$cmpcode);
9440   %}
9441 
9442   ins_pipe(icond_reg_reg);
9443 %}
9444 
9445 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9446   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9447 
9448   ins_cost(INSN_COST * 2);
9449   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
9450 
9451   ins_encode %{
9452     __ cselw(as_Register($dst$$reg),
9453              as_Register($src2$$reg),
9454              as_Register($src1$$reg),
9455              (Assembler::Condition)$cmp$$cmpcode);
9456   %}
9457 
9458   ins_pipe(icond_reg_reg);
9459 %}
9460 
9461 // special cases where one arg is zero
9462 
9463 // n.b. this is selected in preference to the rule above because it
9464 // avoids loading constant 0 into a source register
9465 
9466 // TODO
9467 // we ought only to be able to cull one of these variants as the ideal
9468 // transforms ought always to order the zero consistently (to left/right?)
9469 
9470 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9471   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9472 
9473   ins_cost(INSN_COST * 2);
9474   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
9475 
9476   ins_encode %{
9477     __ cselw(as_Register($dst$$reg),
9478              as_Register($src$$reg),
9479              zr,
9480              (Assembler::Condition)$cmp$$cmpcode);
9481   %}
9482 
9483   ins_pipe(icond_reg);
9484 %}
9485 
9486 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9487   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9488 
9489   ins_cost(INSN_COST * 2);
9490   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
9491 
9492   ins_encode %{
9493     __ cselw(as_Register($dst$$reg),
9494              as_Register($src$$reg),
9495              zr,
9496              (Assembler::Condition)$cmp$$cmpcode);
9497   %}
9498 
9499   ins_pipe(icond_reg);
9500 %}
9501 
9502 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9503   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9504 
9505   ins_cost(INSN_COST * 2);
9506   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
9507 
9508   ins_encode %{
9509     __ cselw(as_Register($dst$$reg),
9510              zr,
9511              as_Register($src$$reg),
9512              (Assembler::Condition)$cmp$$cmpcode);
9513   %}
9514 
9515   ins_pipe(icond_reg);
9516 %}
9517 
9518 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9519   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9520 
9521   ins_cost(INSN_COST * 2);
9522   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
9523 
9524   ins_encode %{
9525     __ cselw(as_Register($dst$$reg),
9526              zr,
9527              as_Register($src$$reg),
9528              (Assembler::Condition)$cmp$$cmpcode);
9529   %}
9530 
9531   ins_pipe(icond_reg);
9532 %}
9533 
9534 // special case for creating a boolean 0 or 1
9535 
9536 // n.b. this is selected in preference to the rule above because it
9537 // avoids loading constants 0 and 1 into a source register
9538 
9539 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9540   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9541 
9542   ins_cost(INSN_COST * 2);
9543   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
9544 
9545   ins_encode %{
9546     // equivalently
9547     // cset(as_Register($dst$$reg),
9548     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
9549     __ csincw(as_Register($dst$$reg),
9550              zr,
9551              zr,
9552              (Assembler::Condition)$cmp$$cmpcode);
9553   %}
9554 
9555   ins_pipe(icond_none);
9556 %}
9557 
9558 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9559   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9560 
9561   ins_cost(INSN_COST * 2);
9562   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
9563 
9564   ins_encode %{
9565     // equivalently
9566     // cset(as_Register($dst$$reg),
9567     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
9568     __ csincw(as_Register($dst$$reg),
9569              zr,
9570              zr,
9571              (Assembler::Condition)$cmp$$cmpcode);
9572   %}
9573 
9574   ins_pipe(icond_none);
9575 %}
9576 
9577 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9578   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9579 
9580   ins_cost(INSN_COST * 2);
9581   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
9582 
9583   ins_encode %{
9584     __ csel(as_Register($dst$$reg),
9585             as_Register($src2$$reg),
9586             as_Register($src1$$reg),
9587             (Assembler::Condition)$cmp$$cmpcode);
9588   %}
9589 
9590   ins_pipe(icond_reg_reg);
9591 %}
9592 
9593 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9594   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9595 
9596   ins_cost(INSN_COST * 2);
9597   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
9598 
9599   ins_encode %{
9600     __ csel(as_Register($dst$$reg),
9601             as_Register($src2$$reg),
9602             as_Register($src1$$reg),
9603             (Assembler::Condition)$cmp$$cmpcode);
9604   %}
9605 
9606   ins_pipe(icond_reg_reg);
9607 %}
9608 
9609 // special cases where one arg is zero
9610 
9611 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9612   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9613 
9614   ins_cost(INSN_COST * 2);
9615   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
9616 
9617   ins_encode %{
9618     __ csel(as_Register($dst$$reg),
9619             zr,
9620             as_Register($src$$reg),
9621             (Assembler::Condition)$cmp$$cmpcode);
9622   %}
9623 
9624   ins_pipe(icond_reg);
9625 %}
9626 
9627 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9628   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9629 
9630   ins_cost(INSN_COST * 2);
9631   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
9632 
9633   ins_encode %{
9634     __ csel(as_Register($dst$$reg),
9635             zr,
9636             as_Register($src$$reg),
9637             (Assembler::Condition)$cmp$$cmpcode);
9638   %}
9639 
9640   ins_pipe(icond_reg);
9641 %}
9642 
9643 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9644   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9645 
9646   ins_cost(INSN_COST * 2);
9647   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
9648 
9649   ins_encode %{
9650     __ csel(as_Register($dst$$reg),
9651             as_Register($src$$reg),
9652             zr,
9653             (Assembler::Condition)$cmp$$cmpcode);
9654   %}
9655 
9656   ins_pipe(icond_reg);
9657 %}
9658 
9659 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9660   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9661 
9662   ins_cost(INSN_COST * 2);
9663   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
9664 
9665   ins_encode %{
9666     __ csel(as_Register($dst$$reg),
9667             as_Register($src$$reg),
9668             zr,
9669             (Assembler::Condition)$cmp$$cmpcode);
9670   %}
9671 
9672   ins_pipe(icond_reg);
9673 %}
9674 
9675 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9676   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9677 
9678   ins_cost(INSN_COST * 2);
9679   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
9680 
9681   ins_encode %{
9682     __ csel(as_Register($dst$$reg),
9683             as_Register($src2$$reg),
9684             as_Register($src1$$reg),
9685             (Assembler::Condition)$cmp$$cmpcode);
9686   %}
9687 
9688   ins_pipe(icond_reg_reg);
9689 %}
9690 
9691 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9692   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9693 
9694   ins_cost(INSN_COST * 2);
9695   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
9696 
9697   ins_encode %{
9698     __ csel(as_Register($dst$$reg),
9699             as_Register($src2$$reg),
9700             as_Register($src1$$reg),
9701             (Assembler::Condition)$cmp$$cmpcode);
9702   %}
9703 
9704   ins_pipe(icond_reg_reg);
9705 %}
9706 
9707 // special cases where one arg is zero
9708 
9709 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9710   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9711 
9712   ins_cost(INSN_COST * 2);
9713   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
9714 
9715   ins_encode %{
9716     __ csel(as_Register($dst$$reg),
9717             zr,
9718             as_Register($src$$reg),
9719             (Assembler::Condition)$cmp$$cmpcode);
9720   %}
9721 
9722   ins_pipe(icond_reg);
9723 %}
9724 
9725 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9726   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9727 
9728   ins_cost(INSN_COST * 2);
9729   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
9730 
9731   ins_encode %{
9732     __ csel(as_Register($dst$$reg),
9733             zr,
9734             as_Register($src$$reg),
9735             (Assembler::Condition)$cmp$$cmpcode);
9736   %}
9737 
9738   ins_pipe(icond_reg);
9739 %}
9740 
9741 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9742   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9743 
9744   ins_cost(INSN_COST * 2);
9745   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
9746 
9747   ins_encode %{
9748     __ csel(as_Register($dst$$reg),
9749             as_Register($src$$reg),
9750             zr,
9751             (Assembler::Condition)$cmp$$cmpcode);
9752   %}
9753 
9754   ins_pipe(icond_reg);
9755 %}
9756 
9757 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9758   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9759 
9760   ins_cost(INSN_COST * 2);
9761   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
9762 
9763   ins_encode %{
9764     __ csel(as_Register($dst$$reg),
9765             as_Register($src$$reg),
9766             zr,
9767             (Assembler::Condition)$cmp$$cmpcode);
9768   %}
9769 
9770   ins_pipe(icond_reg);
9771 %}
9772 
9773 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9774   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9775 
9776   ins_cost(INSN_COST * 2);
9777   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
9778 
9779   ins_encode %{
9780     __ cselw(as_Register($dst$$reg),
9781              as_Register($src2$$reg),
9782              as_Register($src1$$reg),
9783              (Assembler::Condition)$cmp$$cmpcode);
9784   %}
9785 
9786   ins_pipe(icond_reg_reg);
9787 %}
9788 
9789 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9790   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9791 
9792   ins_cost(INSN_COST * 2);
9793   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
9794 
9795   ins_encode %{
9796     __ cselw(as_Register($dst$$reg),
9797              as_Register($src2$$reg),
9798              as_Register($src1$$reg),
9799              (Assembler::Condition)$cmp$$cmpcode);
9800   %}
9801 
9802   ins_pipe(icond_reg_reg);
9803 %}
9804 
9805 // special cases where one arg is zero
9806 
9807 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9808   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9809 
9810   ins_cost(INSN_COST * 2);
9811   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
9812 
9813   ins_encode %{
9814     __ cselw(as_Register($dst$$reg),
9815              zr,
9816              as_Register($src$$reg),
9817              (Assembler::Condition)$cmp$$cmpcode);
9818   %}
9819 
9820   ins_pipe(icond_reg);
9821 %}
9822 
9823 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9824   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9825 
9826   ins_cost(INSN_COST * 2);
9827   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
9828 
9829   ins_encode %{
9830     __ cselw(as_Register($dst$$reg),
9831              zr,
9832              as_Register($src$$reg),
9833              (Assembler::Condition)$cmp$$cmpcode);
9834   %}
9835 
9836   ins_pipe(icond_reg);
9837 %}
9838 
9839 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9840   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9841 
9842   ins_cost(INSN_COST * 2);
9843   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
9844 
9845   ins_encode %{
9846     __ cselw(as_Register($dst$$reg),
9847              as_Register($src$$reg),
9848              zr,
9849              (Assembler::Condition)$cmp$$cmpcode);
9850   %}
9851 
9852   ins_pipe(icond_reg);
9853 %}
9854 
9855 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9856   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9857 
9858   ins_cost(INSN_COST * 2);
9859   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
9860 
9861   ins_encode %{
9862     __ cselw(as_Register($dst$$reg),
9863              as_Register($src$$reg),
9864              zr,
9865              (Assembler::Condition)$cmp$$cmpcode);
9866   %}
9867 
9868   ins_pipe(icond_reg);
9869 %}
9870 
9871 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
9872 %{
9873   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9874 
9875   ins_cost(INSN_COST * 3);
9876 
9877   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9878   ins_encode %{
9879     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9880     __ fcsels(as_FloatRegister($dst$$reg),
9881               as_FloatRegister($src2$$reg),
9882               as_FloatRegister($src1$$reg),
9883               cond);
9884   %}
9885 
9886   ins_pipe(fp_cond_reg_reg_s);
9887 %}
9888 
9889 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
9890 %{
9891   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9892 
9893   ins_cost(INSN_COST * 3);
9894 
9895   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9896   ins_encode %{
9897     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9898     __ fcsels(as_FloatRegister($dst$$reg),
9899               as_FloatRegister($src2$$reg),
9900               as_FloatRegister($src1$$reg),
9901               cond);
9902   %}
9903 
9904   ins_pipe(fp_cond_reg_reg_s);
9905 %}
9906 
9907 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
9908 %{
9909   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9910 
9911   ins_cost(INSN_COST * 3);
9912 
9913   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9914   ins_encode %{
9915     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9916     __ fcseld(as_FloatRegister($dst$$reg),
9917               as_FloatRegister($src2$$reg),
9918               as_FloatRegister($src1$$reg),
9919               cond);
9920   %}
9921 
9922   ins_pipe(fp_cond_reg_reg_d);
9923 %}
9924 
9925 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
9926 %{
9927   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9928 
9929   ins_cost(INSN_COST * 3);
9930 
9931   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9932   ins_encode %{
9933     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9934     __ fcseld(as_FloatRegister($dst$$reg),
9935               as_FloatRegister($src2$$reg),
9936               as_FloatRegister($src1$$reg),
9937               cond);
9938   %}
9939 
9940   ins_pipe(fp_cond_reg_reg_d);
9941 %}
9942 
9943 // ============================================================================
9944 // Arithmetic Instructions
9945 //
9946 
9947 // Integer Addition
9948 
9949 // TODO
9950 // these currently employ operations which do not set CR and hence are
9951 // not flagged as killing CR but we would like to isolate the cases
9952 // where we want to set flags from those where we don't. need to work
9953 // out how to do that.
9954 
9955 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9956   match(Set dst (AddI src1 src2));
9957 
9958   ins_cost(INSN_COST);
9959   format %{ "addw  $dst, $src1, $src2" %}
9960 
9961   ins_encode %{
9962     __ addw(as_Register($dst$$reg),
9963             as_Register($src1$$reg),
9964             as_Register($src2$$reg));
9965   %}
9966 
9967   ins_pipe(ialu_reg_reg);
9968 %}
9969 
9970 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9971   match(Set dst (AddI src1 src2));
9972 
9973   ins_cost(INSN_COST);
9974   format %{ "addw $dst, $src1, $src2" %}
9975 
9976   // use opcode to indicate that this is an add not a sub
9977   opcode(0x0);
9978 
9979   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9980 
9981   ins_pipe(ialu_reg_imm);
9982 %}
9983 
9984 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9985   match(Set dst (AddI (ConvL2I src1) src2));
9986 
9987   ins_cost(INSN_COST);
9988   format %{ "addw $dst, $src1, $src2" %}
9989 
9990   // use opcode to indicate that this is an add not a sub
9991   opcode(0x0);
9992 
9993   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9994 
9995   ins_pipe(ialu_reg_imm);
9996 %}
9997 
9998 // Pointer Addition
9999 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10000   match(Set dst (AddP src1 src2));
10001 
10002   ins_cost(INSN_COST);
10003   format %{ "add $dst, $src1, $src2\t# ptr" %}
10004 
10005   ins_encode %{
10006     __ add(as_Register($dst$$reg),
10007            as_Register($src1$$reg),
10008            as_Register($src2$$reg));
10009   %}
10010 
10011   ins_pipe(ialu_reg_reg);
10012 %}
10013 
10014 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10015   match(Set dst (AddP src1 (ConvI2L src2)));
10016 
10017   ins_cost(1.9 * INSN_COST);
10018   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10019 
10020   ins_encode %{
10021     __ add(as_Register($dst$$reg),
10022            as_Register($src1$$reg),
10023            as_Register($src2$$reg), ext::sxtw);
10024   %}
10025 
10026   ins_pipe(ialu_reg_reg);
10027 %}
10028 
10029 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10030   match(Set dst (AddP src1 (LShiftL src2 scale)));
10031 
10032   ins_cost(1.9 * INSN_COST);
10033   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10034 
10035   ins_encode %{
10036     __ lea(as_Register($dst$$reg),
10037            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10038                    Address::lsl($scale$$constant)));
10039   %}
10040 
10041   ins_pipe(ialu_reg_reg_shift);
10042 %}
10043 
10044 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10045   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10046 
10047   ins_cost(1.9 * INSN_COST);
10048   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10049 
10050   ins_encode %{
10051     __ lea(as_Register($dst$$reg),
10052            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10053                    Address::sxtw($scale$$constant)));
10054   %}
10055 
10056   ins_pipe(ialu_reg_reg_shift);
10057 %}
10058 
10059 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10060   match(Set dst (LShiftL (ConvI2L src) scale));
10061 
10062   ins_cost(INSN_COST);
10063   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10064 
10065   ins_encode %{
10066     __ sbfiz(as_Register($dst$$reg),
10067           as_Register($src$$reg),
10068           $scale$$constant & 63, MIN(32, (-$scale$$constant) & 63));
10069   %}
10070 
10071   ins_pipe(ialu_reg_shift);
10072 %}
10073 
10074 // Pointer Immediate Addition
10075 // n.b. this needs to be more expensive than using an indirect memory
10076 // operand
10077 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10078   match(Set dst (AddP src1 src2));
10079 
10080   ins_cost(INSN_COST);
10081   format %{ "add $dst, $src1, $src2\t# ptr" %}
10082 
10083   // use opcode to indicate that this is an add not a sub
10084   opcode(0x0);
10085 
10086   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10087 
10088   ins_pipe(ialu_reg_imm);
10089 %}
10090 
10091 // Long Addition
10092 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10093 
10094   match(Set dst (AddL src1 src2));
10095 
10096   ins_cost(INSN_COST);
10097   format %{ "add  $dst, $src1, $src2" %}
10098 
10099   ins_encode %{
10100     __ add(as_Register($dst$$reg),
10101            as_Register($src1$$reg),
10102            as_Register($src2$$reg));
10103   %}
10104 
10105   ins_pipe(ialu_reg_reg);
10106 %}
10107 
10108 // No constant pool entries requiredLong Immediate Addition.
10109 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10110   match(Set dst (AddL src1 src2));
10111 
10112   ins_cost(INSN_COST);
10113   format %{ "add $dst, $src1, $src2" %}
10114 
10115   // use opcode to indicate that this is an add not a sub
10116   opcode(0x0);
10117 
10118   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10119 
10120   ins_pipe(ialu_reg_imm);
10121 %}
10122 
10123 // Integer Subtraction
10124 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10125   match(Set dst (SubI src1 src2));
10126 
10127   ins_cost(INSN_COST);
10128   format %{ "subw  $dst, $src1, $src2" %}
10129 
10130   ins_encode %{
10131     __ subw(as_Register($dst$$reg),
10132             as_Register($src1$$reg),
10133             as_Register($src2$$reg));
10134   %}
10135 
10136   ins_pipe(ialu_reg_reg);
10137 %}
10138 
10139 // Immediate Subtraction
10140 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10141   match(Set dst (SubI src1 src2));
10142 
10143   ins_cost(INSN_COST);
10144   format %{ "subw $dst, $src1, $src2" %}
10145 
10146   // use opcode to indicate that this is a sub not an add
10147   opcode(0x1);
10148 
10149   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10150 
10151   ins_pipe(ialu_reg_imm);
10152 %}
10153 
10154 // Long Subtraction
10155 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10156 
10157   match(Set dst (SubL src1 src2));
10158 
10159   ins_cost(INSN_COST);
10160   format %{ "sub  $dst, $src1, $src2" %}
10161 
10162   ins_encode %{
10163     __ sub(as_Register($dst$$reg),
10164            as_Register($src1$$reg),
10165            as_Register($src2$$reg));
10166   %}
10167 
10168   ins_pipe(ialu_reg_reg);
10169 %}
10170 
10171 // No constant pool entries requiredLong Immediate Subtraction.
10172 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10173   match(Set dst (SubL src1 src2));
10174 
10175   ins_cost(INSN_COST);
10176   format %{ "sub$dst, $src1, $src2" %}
10177 
10178   // use opcode to indicate that this is a sub not an add
10179   opcode(0x1);
10180 
10181   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10182 
10183   ins_pipe(ialu_reg_imm);
10184 %}
10185 
10186 // Integer Negation (special case for sub)
10187 
10188 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10189   match(Set dst (SubI zero src));
10190 
10191   ins_cost(INSN_COST);
10192   format %{ "negw $dst, $src\t# int" %}
10193 
10194   ins_encode %{
10195     __ negw(as_Register($dst$$reg),
10196             as_Register($src$$reg));
10197   %}
10198 
10199   ins_pipe(ialu_reg);
10200 %}
10201 
10202 // Long Negation
10203 
10204 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10205   match(Set dst (SubL zero src));
10206 
10207   ins_cost(INSN_COST);
10208   format %{ "neg $dst, $src\t# long" %}
10209 
10210   ins_encode %{
10211     __ neg(as_Register($dst$$reg),
10212            as_Register($src$$reg));
10213   %}
10214 
10215   ins_pipe(ialu_reg);
10216 %}
10217 
10218 // Integer Multiply
10219 
10220 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10221   match(Set dst (MulI src1 src2));
10222 
10223   ins_cost(INSN_COST * 3);
10224   format %{ "mulw  $dst, $src1, $src2" %}
10225 
10226   ins_encode %{
10227     __ mulw(as_Register($dst$$reg),
10228             as_Register($src1$$reg),
10229             as_Register($src2$$reg));
10230   %}
10231 
10232   ins_pipe(imul_reg_reg);
10233 %}
10234 
10235 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10236   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10237 
10238   ins_cost(INSN_COST * 3);
10239   format %{ "smull  $dst, $src1, $src2" %}
10240 
10241   ins_encode %{
10242     __ smull(as_Register($dst$$reg),
10243              as_Register($src1$$reg),
10244              as_Register($src2$$reg));
10245   %}
10246 
10247   ins_pipe(imul_reg_reg);
10248 %}
10249 
10250 // Long Multiply
10251 
10252 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10253   match(Set dst (MulL src1 src2));
10254 
10255   ins_cost(INSN_COST * 5);
10256   format %{ "mul  $dst, $src1, $src2" %}
10257 
10258   ins_encode %{
10259     __ mul(as_Register($dst$$reg),
10260            as_Register($src1$$reg),
10261            as_Register($src2$$reg));
10262   %}
10263 
10264   ins_pipe(lmul_reg_reg);
10265 %}
10266 
10267 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10268 %{
10269   match(Set dst (MulHiL src1 src2));
10270 
10271   ins_cost(INSN_COST * 7);
10272   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
10273 
10274   ins_encode %{
10275     __ smulh(as_Register($dst$$reg),
10276              as_Register($src1$$reg),
10277              as_Register($src2$$reg));
10278   %}
10279 
10280   ins_pipe(lmul_reg_reg);
10281 %}
10282 
10283 // Combined Integer Multiply & Add/Sub
10284 
10285 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10286   match(Set dst (AddI src3 (MulI src1 src2)));
10287 
10288   ins_cost(INSN_COST * 3);
10289   format %{ "madd  $dst, $src1, $src2, $src3" %}
10290 
10291   ins_encode %{
10292     __ maddw(as_Register($dst$$reg),
10293              as_Register($src1$$reg),
10294              as_Register($src2$$reg),
10295              as_Register($src3$$reg));
10296   %}
10297 
10298   ins_pipe(imac_reg_reg);
10299 %}
10300 
10301 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10302   match(Set dst (SubI src3 (MulI src1 src2)));
10303 
10304   ins_cost(INSN_COST * 3);
10305   format %{ "msub  $dst, $src1, $src2, $src3" %}
10306 
10307   ins_encode %{
10308     __ msubw(as_Register($dst$$reg),
10309              as_Register($src1$$reg),
10310              as_Register($src2$$reg),
10311              as_Register($src3$$reg));
10312   %}
10313 
10314   ins_pipe(imac_reg_reg);
10315 %}
10316 
10317 // Combined Integer Multiply & Neg
10318 
10319 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10320   match(Set dst (MulI (SubI zero src1) src2));
10321   match(Set dst (MulI src1 (SubI zero src2)));
10322 
10323   ins_cost(INSN_COST * 3);
10324   format %{ "mneg  $dst, $src1, $src2" %}
10325 
10326   ins_encode %{
10327     __ mnegw(as_Register($dst$$reg),
10328              as_Register($src1$$reg),
10329              as_Register($src2$$reg));
10330   %}
10331 
10332   ins_pipe(imac_reg_reg);
10333 %}
10334 
10335 // Combined Long Multiply & Add/Sub
10336 
10337 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10338   match(Set dst (AddL src3 (MulL src1 src2)));
10339 
10340   ins_cost(INSN_COST * 5);
10341   format %{ "madd  $dst, $src1, $src2, $src3" %}
10342 
10343   ins_encode %{
10344     __ madd(as_Register($dst$$reg),
10345             as_Register($src1$$reg),
10346             as_Register($src2$$reg),
10347             as_Register($src3$$reg));
10348   %}
10349 
10350   ins_pipe(lmac_reg_reg);
10351 %}
10352 
10353 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10354   match(Set dst (SubL src3 (MulL src1 src2)));
10355 
10356   ins_cost(INSN_COST * 5);
10357   format %{ "msub  $dst, $src1, $src2, $src3" %}
10358 
10359   ins_encode %{
10360     __ msub(as_Register($dst$$reg),
10361             as_Register($src1$$reg),
10362             as_Register($src2$$reg),
10363             as_Register($src3$$reg));
10364   %}
10365 
10366   ins_pipe(lmac_reg_reg);
10367 %}
10368 
10369 // Combined Long Multiply & Neg
10370 
10371 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10372   match(Set dst (MulL (SubL zero src1) src2));
10373   match(Set dst (MulL src1 (SubL zero src2)));
10374 
10375   ins_cost(INSN_COST * 5);
10376   format %{ "mneg  $dst, $src1, $src2" %}
10377 
10378   ins_encode %{
10379     __ mneg(as_Register($dst$$reg),
10380             as_Register($src1$$reg),
10381             as_Register($src2$$reg));
10382   %}
10383 
10384   ins_pipe(lmac_reg_reg);
10385 %}
10386 
10387 // Integer Divide
10388 
10389 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10390   match(Set dst (DivI src1 src2));
10391 
10392   ins_cost(INSN_COST * 19);
10393   format %{ "sdivw  $dst, $src1, $src2" %}
10394 
10395   ins_encode(aarch64_enc_divw(dst, src1, src2));
10396   ins_pipe(idiv_reg_reg);
10397 %}
10398 
10399 instruct signExtract(iRegINoSp dst, iRegIorL2I src1, immI_31 div1, immI_31 div2) %{
10400   match(Set dst (URShiftI (RShiftI src1 div1) div2));
10401   ins_cost(INSN_COST);
10402   format %{ "lsrw $dst, $src1, $div1" %}
10403   ins_encode %{
10404     __ lsrw(as_Register($dst$$reg), as_Register($src1$$reg), 31);
10405   %}
10406   ins_pipe(ialu_reg_shift);
10407 %}
10408 
10409 instruct div2Round(iRegINoSp dst, iRegIorL2I src, immI_31 div1, immI_31 div2) %{
10410   match(Set dst (AddI src (URShiftI (RShiftI src div1) div2)));
10411   ins_cost(INSN_COST);
10412   format %{ "addw $dst, $src, LSR $div1" %}
10413 
10414   ins_encode %{
10415     __ addw(as_Register($dst$$reg),
10416               as_Register($src$$reg),
10417               as_Register($src$$reg),
10418               Assembler::LSR, 31);
10419   %}
10420   ins_pipe(ialu_reg);
10421 %}
10422 
10423 // Long Divide
10424 
10425 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10426   match(Set dst (DivL src1 src2));
10427 
10428   ins_cost(INSN_COST * 35);
10429   format %{ "sdiv   $dst, $src1, $src2" %}
10430 
10431   ins_encode(aarch64_enc_div(dst, src1, src2));
10432   ins_pipe(ldiv_reg_reg);
10433 %}
10434 
10435 instruct signExtractL(iRegLNoSp dst, iRegL src1, immI_63 div1, immI_63 div2) %{
10436   match(Set dst (URShiftL (RShiftL src1 div1) div2));
10437   ins_cost(INSN_COST);
10438   format %{ "lsr $dst, $src1, $div1" %}
10439   ins_encode %{
10440     __ lsr(as_Register($dst$$reg), as_Register($src1$$reg), 63);
10441   %}
10442   ins_pipe(ialu_reg_shift);
10443 %}
10444 
10445 instruct div2RoundL(iRegLNoSp dst, iRegL src, immI_63 div1, immI_63 div2) %{
10446   match(Set dst (AddL src (URShiftL (RShiftL src div1) div2)));
10447   ins_cost(INSN_COST);
10448   format %{ "add $dst, $src, $div1" %}
10449 
10450   ins_encode %{
10451     __ add(as_Register($dst$$reg),
10452               as_Register($src$$reg),
10453               as_Register($src$$reg),
10454               Assembler::LSR, 63);
10455   %}
10456   ins_pipe(ialu_reg);
10457 %}
10458 
10459 // Integer Remainder
10460 
10461 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10462   match(Set dst (ModI src1 src2));
10463 
10464   ins_cost(INSN_COST * 22);
10465   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
10466             "msubw($dst, rscratch1, $src2, $src1" %}
10467 
10468   ins_encode(aarch64_enc_modw(dst, src1, src2));
10469   ins_pipe(idiv_reg_reg);
10470 %}
10471 
10472 // Long Remainder
10473 
10474 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10475   match(Set dst (ModL src1 src2));
10476 
10477   ins_cost(INSN_COST * 38);
10478   format %{ "sdiv   rscratch1, $src1, $src2\n"
10479             "msub($dst, rscratch1, $src2, $src1" %}
10480 
10481   ins_encode(aarch64_enc_mod(dst, src1, src2));
10482   ins_pipe(ldiv_reg_reg);
10483 %}
10484 
10485 // Integer Shifts
10486 
10487 // Shift Left Register
10488 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10489   match(Set dst (LShiftI src1 src2));
10490 
10491   ins_cost(INSN_COST * 2);
10492   format %{ "lslvw  $dst, $src1, $src2" %}
10493 
10494   ins_encode %{
10495     __ lslvw(as_Register($dst$$reg),
10496              as_Register($src1$$reg),
10497              as_Register($src2$$reg));
10498   %}
10499 
10500   ins_pipe(ialu_reg_reg_vshift);
10501 %}
10502 
10503 // Shift Left Immediate
10504 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10505   match(Set dst (LShiftI src1 src2));
10506 
10507   ins_cost(INSN_COST);
10508   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10509 
10510   ins_encode %{
10511     __ lslw(as_Register($dst$$reg),
10512             as_Register($src1$$reg),
10513             $src2$$constant & 0x1f);
10514   %}
10515 
10516   ins_pipe(ialu_reg_shift);
10517 %}
10518 
10519 // Shift Right Logical Register
10520 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10521   match(Set dst (URShiftI src1 src2));
10522 
10523   ins_cost(INSN_COST * 2);
10524   format %{ "lsrvw  $dst, $src1, $src2" %}
10525 
10526   ins_encode %{
10527     __ lsrvw(as_Register($dst$$reg),
10528              as_Register($src1$$reg),
10529              as_Register($src2$$reg));
10530   %}
10531 
10532   ins_pipe(ialu_reg_reg_vshift);
10533 %}
10534 
10535 // Shift Right Logical Immediate
10536 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10537   match(Set dst (URShiftI src1 src2));
10538 
10539   ins_cost(INSN_COST);
10540   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10541 
10542   ins_encode %{
10543     __ lsrw(as_Register($dst$$reg),
10544             as_Register($src1$$reg),
10545             $src2$$constant & 0x1f);
10546   %}
10547 
10548   ins_pipe(ialu_reg_shift);
10549 %}
10550 
10551 // Shift Right Arithmetic Register
10552 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10553   match(Set dst (RShiftI src1 src2));
10554 
10555   ins_cost(INSN_COST * 2);
10556   format %{ "asrvw  $dst, $src1, $src2" %}
10557 
10558   ins_encode %{
10559     __ asrvw(as_Register($dst$$reg),
10560              as_Register($src1$$reg),
10561              as_Register($src2$$reg));
10562   %}
10563 
10564   ins_pipe(ialu_reg_reg_vshift);
10565 %}
10566 
10567 // Shift Right Arithmetic Immediate
10568 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10569   match(Set dst (RShiftI src1 src2));
10570 
10571   ins_cost(INSN_COST);
10572   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10573 
10574   ins_encode %{
10575     __ asrw(as_Register($dst$$reg),
10576             as_Register($src1$$reg),
10577             $src2$$constant & 0x1f);
10578   %}
10579 
10580   ins_pipe(ialu_reg_shift);
10581 %}
10582 
10583 // Combined Int Mask and Right Shift (using UBFM)
10584 // TODO
10585 
10586 // Long Shifts
10587 
10588 // Shift Left Register
10589 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10590   match(Set dst (LShiftL src1 src2));
10591 
10592   ins_cost(INSN_COST * 2);
10593   format %{ "lslv  $dst, $src1, $src2" %}
10594 
10595   ins_encode %{
10596     __ lslv(as_Register($dst$$reg),
10597             as_Register($src1$$reg),
10598             as_Register($src2$$reg));
10599   %}
10600 
10601   ins_pipe(ialu_reg_reg_vshift);
10602 %}
10603 
10604 // Shift Left Immediate
10605 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10606   match(Set dst (LShiftL src1 src2));
10607 
10608   ins_cost(INSN_COST);
10609   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10610 
10611   ins_encode %{
10612     __ lsl(as_Register($dst$$reg),
10613             as_Register($src1$$reg),
10614             $src2$$constant & 0x3f);
10615   %}
10616 
10617   ins_pipe(ialu_reg_shift);
10618 %}
10619 
10620 // Shift Right Logical Register
10621 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10622   match(Set dst (URShiftL src1 src2));
10623 
10624   ins_cost(INSN_COST * 2);
10625   format %{ "lsrv  $dst, $src1, $src2" %}
10626 
10627   ins_encode %{
10628     __ lsrv(as_Register($dst$$reg),
10629             as_Register($src1$$reg),
10630             as_Register($src2$$reg));
10631   %}
10632 
10633   ins_pipe(ialu_reg_reg_vshift);
10634 %}
10635 
10636 // Shift Right Logical Immediate
10637 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10638   match(Set dst (URShiftL src1 src2));
10639 
10640   ins_cost(INSN_COST);
10641   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10642 
10643   ins_encode %{
10644     __ lsr(as_Register($dst$$reg),
10645            as_Register($src1$$reg),
10646            $src2$$constant & 0x3f);
10647   %}
10648 
10649   ins_pipe(ialu_reg_shift);
10650 %}
10651 
10652 // A special-case pattern for card table stores.
10653 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10654   match(Set dst (URShiftL (CastP2X src1) src2));
10655 
10656   ins_cost(INSN_COST);
10657   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10658 
10659   ins_encode %{
10660     __ lsr(as_Register($dst$$reg),
10661            as_Register($src1$$reg),
10662            $src2$$constant & 0x3f);
10663   %}
10664 
10665   ins_pipe(ialu_reg_shift);
10666 %}
10667 
10668 // Shift Right Arithmetic Register
10669 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10670   match(Set dst (RShiftL src1 src2));
10671 
10672   ins_cost(INSN_COST * 2);
10673   format %{ "asrv  $dst, $src1, $src2" %}
10674 
10675   ins_encode %{
10676     __ asrv(as_Register($dst$$reg),
10677             as_Register($src1$$reg),
10678             as_Register($src2$$reg));
10679   %}
10680 
10681   ins_pipe(ialu_reg_reg_vshift);
10682 %}
10683 
10684 // Shift Right Arithmetic Immediate
10685 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10686   match(Set dst (RShiftL src1 src2));
10687 
10688   ins_cost(INSN_COST);
10689   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10690 
10691   ins_encode %{
10692     __ asr(as_Register($dst$$reg),
10693            as_Register($src1$$reg),
10694            $src2$$constant & 0x3f);
10695   %}
10696 
10697   ins_pipe(ialu_reg_shift);
10698 %}
10699 
10700 // BEGIN This section of the file is automatically generated. Do not edit --------------
10701 
10702 instruct regL_not_reg(iRegLNoSp dst,
10703                          iRegL src1, immL_M1 m1,
10704                          rFlagsReg cr) %{
10705   match(Set dst (XorL src1 m1));
10706   ins_cost(INSN_COST);
10707   format %{ "eon  $dst, $src1, zr" %}
10708 
10709   ins_encode %{
10710     __ eon(as_Register($dst$$reg),
10711               as_Register($src1$$reg),
10712               zr,
10713               Assembler::LSL, 0);
10714   %}
10715 
10716   ins_pipe(ialu_reg);
10717 %}
10718 instruct regI_not_reg(iRegINoSp dst,
10719                          iRegIorL2I src1, immI_M1 m1,
10720                          rFlagsReg cr) %{
10721   match(Set dst (XorI src1 m1));
10722   ins_cost(INSN_COST);
10723   format %{ "eonw  $dst, $src1, zr" %}
10724 
10725   ins_encode %{
10726     __ eonw(as_Register($dst$$reg),
10727               as_Register($src1$$reg),
10728               zr,
10729               Assembler::LSL, 0);
10730   %}
10731 
10732   ins_pipe(ialu_reg);
10733 %}
10734 
10735 instruct AndI_reg_not_reg(iRegINoSp dst,
10736                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
10737                          rFlagsReg cr) %{
10738   match(Set dst (AndI src1 (XorI src2 m1)));
10739   ins_cost(INSN_COST);
10740   format %{ "bicw  $dst, $src1, $src2" %}
10741 
10742   ins_encode %{
10743     __ bicw(as_Register($dst$$reg),
10744               as_Register($src1$$reg),
10745               as_Register($src2$$reg),
10746               Assembler::LSL, 0);
10747   %}
10748 
10749   ins_pipe(ialu_reg_reg);
10750 %}
10751 
10752 instruct AndL_reg_not_reg(iRegLNoSp dst,
10753                          iRegL src1, iRegL src2, immL_M1 m1,
10754                          rFlagsReg cr) %{
10755   match(Set dst (AndL src1 (XorL src2 m1)));
10756   ins_cost(INSN_COST);
10757   format %{ "bic  $dst, $src1, $src2" %}
10758 
10759   ins_encode %{
10760     __ bic(as_Register($dst$$reg),
10761               as_Register($src1$$reg),
10762               as_Register($src2$$reg),
10763               Assembler::LSL, 0);
10764   %}
10765 
10766   ins_pipe(ialu_reg_reg);
10767 %}
10768 
10769 instruct OrI_reg_not_reg(iRegINoSp dst,
10770                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
10771                          rFlagsReg cr) %{
10772   match(Set dst (OrI src1 (XorI src2 m1)));
10773   ins_cost(INSN_COST);
10774   format %{ "ornw  $dst, $src1, $src2" %}
10775 
10776   ins_encode %{
10777     __ ornw(as_Register($dst$$reg),
10778               as_Register($src1$$reg),
10779               as_Register($src2$$reg),
10780               Assembler::LSL, 0);
10781   %}
10782 
10783   ins_pipe(ialu_reg_reg);
10784 %}
10785 
10786 instruct OrL_reg_not_reg(iRegLNoSp dst,
10787                          iRegL src1, iRegL src2, immL_M1 m1,
10788                          rFlagsReg cr) %{
10789   match(Set dst (OrL src1 (XorL src2 m1)));
10790   ins_cost(INSN_COST);
10791   format %{ "orn  $dst, $src1, $src2" %}
10792 
10793   ins_encode %{
10794     __ orn(as_Register($dst$$reg),
10795               as_Register($src1$$reg),
10796               as_Register($src2$$reg),
10797               Assembler::LSL, 0);
10798   %}
10799 
10800   ins_pipe(ialu_reg_reg);
10801 %}
10802 
10803 instruct XorI_reg_not_reg(iRegINoSp dst,
10804                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
10805                          rFlagsReg cr) %{
10806   match(Set dst (XorI m1 (XorI src2 src1)));
10807   ins_cost(INSN_COST);
10808   format %{ "eonw  $dst, $src1, $src2" %}
10809 
10810   ins_encode %{
10811     __ eonw(as_Register($dst$$reg),
10812               as_Register($src1$$reg),
10813               as_Register($src2$$reg),
10814               Assembler::LSL, 0);
10815   %}
10816 
10817   ins_pipe(ialu_reg_reg);
10818 %}
10819 
10820 instruct XorL_reg_not_reg(iRegLNoSp dst,
10821                          iRegL src1, iRegL src2, immL_M1 m1,
10822                          rFlagsReg cr) %{
10823   match(Set dst (XorL m1 (XorL src2 src1)));
10824   ins_cost(INSN_COST);
10825   format %{ "eon  $dst, $src1, $src2" %}
10826 
10827   ins_encode %{
10828     __ eon(as_Register($dst$$reg),
10829               as_Register($src1$$reg),
10830               as_Register($src2$$reg),
10831               Assembler::LSL, 0);
10832   %}
10833 
10834   ins_pipe(ialu_reg_reg);
10835 %}
10836 
10837 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10838                          iRegIorL2I src1, iRegIorL2I src2,
10839                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10840   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10841   ins_cost(1.9 * INSN_COST);
10842   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
10843 
10844   ins_encode %{
10845     __ bicw(as_Register($dst$$reg),
10846               as_Register($src1$$reg),
10847               as_Register($src2$$reg),
10848               Assembler::LSR,
10849               $src3$$constant & 0x1f);
10850   %}
10851 
10852   ins_pipe(ialu_reg_reg_shift);
10853 %}
10854 
10855 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10856                          iRegL src1, iRegL src2,
10857                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10858   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10859   ins_cost(1.9 * INSN_COST);
10860   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
10861 
10862   ins_encode %{
10863     __ bic(as_Register($dst$$reg),
10864               as_Register($src1$$reg),
10865               as_Register($src2$$reg),
10866               Assembler::LSR,
10867               $src3$$constant & 0x3f);
10868   %}
10869 
10870   ins_pipe(ialu_reg_reg_shift);
10871 %}
10872 
10873 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10874                          iRegIorL2I src1, iRegIorL2I src2,
10875                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10876   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10877   ins_cost(1.9 * INSN_COST);
10878   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
10879 
10880   ins_encode %{
10881     __ bicw(as_Register($dst$$reg),
10882               as_Register($src1$$reg),
10883               as_Register($src2$$reg),
10884               Assembler::ASR,
10885               $src3$$constant & 0x1f);
10886   %}
10887 
10888   ins_pipe(ialu_reg_reg_shift);
10889 %}
10890 
10891 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10892                          iRegL src1, iRegL src2,
10893                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10894   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10895   ins_cost(1.9 * INSN_COST);
10896   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
10897 
10898   ins_encode %{
10899     __ bic(as_Register($dst$$reg),
10900               as_Register($src1$$reg),
10901               as_Register($src2$$reg),
10902               Assembler::ASR,
10903               $src3$$constant & 0x3f);
10904   %}
10905 
10906   ins_pipe(ialu_reg_reg_shift);
10907 %}
10908 
10909 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10910                          iRegIorL2I src1, iRegIorL2I src2,
10911                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10912   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10913   ins_cost(1.9 * INSN_COST);
10914   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
10915 
10916   ins_encode %{
10917     __ bicw(as_Register($dst$$reg),
10918               as_Register($src1$$reg),
10919               as_Register($src2$$reg),
10920               Assembler::LSL,
10921               $src3$$constant & 0x1f);
10922   %}
10923 
10924   ins_pipe(ialu_reg_reg_shift);
10925 %}
10926 
10927 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10928                          iRegL src1, iRegL src2,
10929                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10930   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10931   ins_cost(1.9 * INSN_COST);
10932   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
10933 
10934   ins_encode %{
10935     __ bic(as_Register($dst$$reg),
10936               as_Register($src1$$reg),
10937               as_Register($src2$$reg),
10938               Assembler::LSL,
10939               $src3$$constant & 0x3f);
10940   %}
10941 
10942   ins_pipe(ialu_reg_reg_shift);
10943 %}
10944 
10945 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10946                          iRegIorL2I src1, iRegIorL2I src2,
10947                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10948   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10949   ins_cost(1.9 * INSN_COST);
10950   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
10951 
10952   ins_encode %{
10953     __ eonw(as_Register($dst$$reg),
10954               as_Register($src1$$reg),
10955               as_Register($src2$$reg),
10956               Assembler::LSR,
10957               $src3$$constant & 0x1f);
10958   %}
10959 
10960   ins_pipe(ialu_reg_reg_shift);
10961 %}
10962 
10963 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10964                          iRegL src1, iRegL src2,
10965                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10966   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10967   ins_cost(1.9 * INSN_COST);
10968   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
10969 
10970   ins_encode %{
10971     __ eon(as_Register($dst$$reg),
10972               as_Register($src1$$reg),
10973               as_Register($src2$$reg),
10974               Assembler::LSR,
10975               $src3$$constant & 0x3f);
10976   %}
10977 
10978   ins_pipe(ialu_reg_reg_shift);
10979 %}
10980 
10981 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10982                          iRegIorL2I src1, iRegIorL2I src2,
10983                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10984   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10985   ins_cost(1.9 * INSN_COST);
10986   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
10987 
10988   ins_encode %{
10989     __ eonw(as_Register($dst$$reg),
10990               as_Register($src1$$reg),
10991               as_Register($src2$$reg),
10992               Assembler::ASR,
10993               $src3$$constant & 0x1f);
10994   %}
10995 
10996   ins_pipe(ialu_reg_reg_shift);
10997 %}
10998 
10999 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11000                          iRegL src1, iRegL src2,
11001                          immI src3, immL_M1 src4, rFlagsReg cr) %{
11002   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11003   ins_cost(1.9 * INSN_COST);
11004   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11005 
11006   ins_encode %{
11007     __ eon(as_Register($dst$$reg),
11008               as_Register($src1$$reg),
11009               as_Register($src2$$reg),
11010               Assembler::ASR,
11011               $src3$$constant & 0x3f);
11012   %}
11013 
11014   ins_pipe(ialu_reg_reg_shift);
11015 %}
11016 
11017 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11018                          iRegIorL2I src1, iRegIorL2I src2,
11019                          immI src3, immI_M1 src4, rFlagsReg cr) %{
11020   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11021   ins_cost(1.9 * INSN_COST);
11022   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11023 
11024   ins_encode %{
11025     __ eonw(as_Register($dst$$reg),
11026               as_Register($src1$$reg),
11027               as_Register($src2$$reg),
11028               Assembler::LSL,
11029               $src3$$constant & 0x1f);
11030   %}
11031 
11032   ins_pipe(ialu_reg_reg_shift);
11033 %}
11034 
11035 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11036                          iRegL src1, iRegL src2,
11037                          immI src3, immL_M1 src4, rFlagsReg cr) %{
11038   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11039   ins_cost(1.9 * INSN_COST);
11040   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11041 
11042   ins_encode %{
11043     __ eon(as_Register($dst$$reg),
11044               as_Register($src1$$reg),
11045               as_Register($src2$$reg),
11046               Assembler::LSL,
11047               $src3$$constant & 0x3f);
11048   %}
11049 
11050   ins_pipe(ialu_reg_reg_shift);
11051 %}
11052 
11053 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11054                          iRegIorL2I src1, iRegIorL2I src2,
11055                          immI src3, immI_M1 src4, rFlagsReg cr) %{
11056   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11057   ins_cost(1.9 * INSN_COST);
11058   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11059 
11060   ins_encode %{
11061     __ ornw(as_Register($dst$$reg),
11062               as_Register($src1$$reg),
11063               as_Register($src2$$reg),
11064               Assembler::LSR,
11065               $src3$$constant & 0x1f);
11066   %}
11067 
11068   ins_pipe(ialu_reg_reg_shift);
11069 %}
11070 
11071 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11072                          iRegL src1, iRegL src2,
11073                          immI src3, immL_M1 src4, rFlagsReg cr) %{
11074   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11075   ins_cost(1.9 * INSN_COST);
11076   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11077 
11078   ins_encode %{
11079     __ orn(as_Register($dst$$reg),
11080               as_Register($src1$$reg),
11081               as_Register($src2$$reg),
11082               Assembler::LSR,
11083               $src3$$constant & 0x3f);
11084   %}
11085 
11086   ins_pipe(ialu_reg_reg_shift);
11087 %}
11088 
11089 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11090                          iRegIorL2I src1, iRegIorL2I src2,
11091                          immI src3, immI_M1 src4, rFlagsReg cr) %{
11092   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11093   ins_cost(1.9 * INSN_COST);
11094   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11095 
11096   ins_encode %{
11097     __ ornw(as_Register($dst$$reg),
11098               as_Register($src1$$reg),
11099               as_Register($src2$$reg),
11100               Assembler::ASR,
11101               $src3$$constant & 0x1f);
11102   %}
11103 
11104   ins_pipe(ialu_reg_reg_shift);
11105 %}
11106 
11107 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11108                          iRegL src1, iRegL src2,
11109                          immI src3, immL_M1 src4, rFlagsReg cr) %{
11110   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11111   ins_cost(1.9 * INSN_COST);
11112   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11113 
11114   ins_encode %{
11115     __ orn(as_Register($dst$$reg),
11116               as_Register($src1$$reg),
11117               as_Register($src2$$reg),
11118               Assembler::ASR,
11119               $src3$$constant & 0x3f);
11120   %}
11121 
11122   ins_pipe(ialu_reg_reg_shift);
11123 %}
11124 
11125 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11126                          iRegIorL2I src1, iRegIorL2I src2,
11127                          immI src3, immI_M1 src4, rFlagsReg cr) %{
11128   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11129   ins_cost(1.9 * INSN_COST);
11130   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11131 
11132   ins_encode %{
11133     __ ornw(as_Register($dst$$reg),
11134               as_Register($src1$$reg),
11135               as_Register($src2$$reg),
11136               Assembler::LSL,
11137               $src3$$constant & 0x1f);
11138   %}
11139 
11140   ins_pipe(ialu_reg_reg_shift);
11141 %}
11142 
11143 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11144                          iRegL src1, iRegL src2,
11145                          immI src3, immL_M1 src4, rFlagsReg cr) %{
11146   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11147   ins_cost(1.9 * INSN_COST);
11148   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11149 
11150   ins_encode %{
11151     __ orn(as_Register($dst$$reg),
11152               as_Register($src1$$reg),
11153               as_Register($src2$$reg),
11154               Assembler::LSL,
11155               $src3$$constant & 0x3f);
11156   %}
11157 
11158   ins_pipe(ialu_reg_reg_shift);
11159 %}
11160 
11161 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11162                          iRegIorL2I src1, iRegIorL2I src2,
11163                          immI src3, rFlagsReg cr) %{
11164   match(Set dst (AndI src1 (URShiftI src2 src3)));
11165 
11166   ins_cost(1.9 * INSN_COST);
11167   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
11168 
11169   ins_encode %{
11170     __ andw(as_Register($dst$$reg),
11171               as_Register($src1$$reg),
11172               as_Register($src2$$reg),
11173               Assembler::LSR,
11174               $src3$$constant & 0x1f);
11175   %}
11176 
11177   ins_pipe(ialu_reg_reg_shift);
11178 %}
11179 
11180 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11181                          iRegL src1, iRegL src2,
11182                          immI src3, rFlagsReg cr) %{
11183   match(Set dst (AndL src1 (URShiftL src2 src3)));
11184 
11185   ins_cost(1.9 * INSN_COST);
11186   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
11187 
11188   ins_encode %{
11189     __ andr(as_Register($dst$$reg),
11190               as_Register($src1$$reg),
11191               as_Register($src2$$reg),
11192               Assembler::LSR,
11193               $src3$$constant & 0x3f);
11194   %}
11195 
11196   ins_pipe(ialu_reg_reg_shift);
11197 %}
11198 
11199 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11200                          iRegIorL2I src1, iRegIorL2I src2,
11201                          immI src3, rFlagsReg cr) %{
11202   match(Set dst (AndI src1 (RShiftI src2 src3)));
11203 
11204   ins_cost(1.9 * INSN_COST);
11205   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
11206 
11207   ins_encode %{
11208     __ andw(as_Register($dst$$reg),
11209               as_Register($src1$$reg),
11210               as_Register($src2$$reg),
11211               Assembler::ASR,
11212               $src3$$constant & 0x1f);
11213   %}
11214 
11215   ins_pipe(ialu_reg_reg_shift);
11216 %}
11217 
11218 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11219                          iRegL src1, iRegL src2,
11220                          immI src3, rFlagsReg cr) %{
11221   match(Set dst (AndL src1 (RShiftL src2 src3)));
11222 
11223   ins_cost(1.9 * INSN_COST);
11224   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
11225 
11226   ins_encode %{
11227     __ andr(as_Register($dst$$reg),
11228               as_Register($src1$$reg),
11229               as_Register($src2$$reg),
11230               Assembler::ASR,
11231               $src3$$constant & 0x3f);
11232   %}
11233 
11234   ins_pipe(ialu_reg_reg_shift);
11235 %}
11236 
11237 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11238                          iRegIorL2I src1, iRegIorL2I src2,
11239                          immI src3, rFlagsReg cr) %{
11240   match(Set dst (AndI src1 (LShiftI src2 src3)));
11241 
11242   ins_cost(1.9 * INSN_COST);
11243   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
11244 
11245   ins_encode %{
11246     __ andw(as_Register($dst$$reg),
11247               as_Register($src1$$reg),
11248               as_Register($src2$$reg),
11249               Assembler::LSL,
11250               $src3$$constant & 0x1f);
11251   %}
11252 
11253   ins_pipe(ialu_reg_reg_shift);
11254 %}
11255 
11256 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11257                          iRegL src1, iRegL src2,
11258                          immI src3, rFlagsReg cr) %{
11259   match(Set dst (AndL src1 (LShiftL src2 src3)));
11260 
11261   ins_cost(1.9 * INSN_COST);
11262   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
11263 
11264   ins_encode %{
11265     __ andr(as_Register($dst$$reg),
11266               as_Register($src1$$reg),
11267               as_Register($src2$$reg),
11268               Assembler::LSL,
11269               $src3$$constant & 0x3f);
11270   %}
11271 
11272   ins_pipe(ialu_reg_reg_shift);
11273 %}
11274 
11275 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11276                          iRegIorL2I src1, iRegIorL2I src2,
11277                          immI src3, rFlagsReg cr) %{
11278   match(Set dst (XorI src1 (URShiftI src2 src3)));
11279 
11280   ins_cost(1.9 * INSN_COST);
11281   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
11282 
11283   ins_encode %{
11284     __ eorw(as_Register($dst$$reg),
11285               as_Register($src1$$reg),
11286               as_Register($src2$$reg),
11287               Assembler::LSR,
11288               $src3$$constant & 0x1f);
11289   %}
11290 
11291   ins_pipe(ialu_reg_reg_shift);
11292 %}
11293 
11294 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11295                          iRegL src1, iRegL src2,
11296                          immI src3, rFlagsReg cr) %{
11297   match(Set dst (XorL src1 (URShiftL src2 src3)));
11298 
11299   ins_cost(1.9 * INSN_COST);
11300   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
11301 
11302   ins_encode %{
11303     __ eor(as_Register($dst$$reg),
11304               as_Register($src1$$reg),
11305               as_Register($src2$$reg),
11306               Assembler::LSR,
11307               $src3$$constant & 0x3f);
11308   %}
11309 
11310   ins_pipe(ialu_reg_reg_shift);
11311 %}
11312 
11313 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11314                          iRegIorL2I src1, iRegIorL2I src2,
11315                          immI src3, rFlagsReg cr) %{
11316   match(Set dst (XorI src1 (RShiftI src2 src3)));
11317 
11318   ins_cost(1.9 * INSN_COST);
11319   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
11320 
11321   ins_encode %{
11322     __ eorw(as_Register($dst$$reg),
11323               as_Register($src1$$reg),
11324               as_Register($src2$$reg),
11325               Assembler::ASR,
11326               $src3$$constant & 0x1f);
11327   %}
11328 
11329   ins_pipe(ialu_reg_reg_shift);
11330 %}
11331 
11332 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11333                          iRegL src1, iRegL src2,
11334                          immI src3, rFlagsReg cr) %{
11335   match(Set dst (XorL src1 (RShiftL src2 src3)));
11336 
11337   ins_cost(1.9 * INSN_COST);
11338   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
11339 
11340   ins_encode %{
11341     __ eor(as_Register($dst$$reg),
11342               as_Register($src1$$reg),
11343               as_Register($src2$$reg),
11344               Assembler::ASR,
11345               $src3$$constant & 0x3f);
11346   %}
11347 
11348   ins_pipe(ialu_reg_reg_shift);
11349 %}
11350 
11351 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11352                          iRegIorL2I src1, iRegIorL2I src2,
11353                          immI src3, rFlagsReg cr) %{
11354   match(Set dst (XorI src1 (LShiftI src2 src3)));
11355 
11356   ins_cost(1.9 * INSN_COST);
11357   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
11358 
11359   ins_encode %{
11360     __ eorw(as_Register($dst$$reg),
11361               as_Register($src1$$reg),
11362               as_Register($src2$$reg),
11363               Assembler::LSL,
11364               $src3$$constant & 0x1f);
11365   %}
11366 
11367   ins_pipe(ialu_reg_reg_shift);
11368 %}
11369 
11370 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11371                          iRegL src1, iRegL src2,
11372                          immI src3, rFlagsReg cr) %{
11373   match(Set dst (XorL src1 (LShiftL src2 src3)));
11374 
11375   ins_cost(1.9 * INSN_COST);
11376   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
11377 
11378   ins_encode %{
11379     __ eor(as_Register($dst$$reg),
11380               as_Register($src1$$reg),
11381               as_Register($src2$$reg),
11382               Assembler::LSL,
11383               $src3$$constant & 0x3f);
11384   %}
11385 
11386   ins_pipe(ialu_reg_reg_shift);
11387 %}
11388 
11389 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11390                          iRegIorL2I src1, iRegIorL2I src2,
11391                          immI src3, rFlagsReg cr) %{
11392   match(Set dst (OrI src1 (URShiftI src2 src3)));
11393 
11394   ins_cost(1.9 * INSN_COST);
11395   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
11396 
11397   ins_encode %{
11398     __ orrw(as_Register($dst$$reg),
11399               as_Register($src1$$reg),
11400               as_Register($src2$$reg),
11401               Assembler::LSR,
11402               $src3$$constant & 0x1f);
11403   %}
11404 
11405   ins_pipe(ialu_reg_reg_shift);
11406 %}
11407 
11408 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11409                          iRegL src1, iRegL src2,
11410                          immI src3, rFlagsReg cr) %{
11411   match(Set dst (OrL src1 (URShiftL src2 src3)));
11412 
11413   ins_cost(1.9 * INSN_COST);
11414   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
11415 
11416   ins_encode %{
11417     __ orr(as_Register($dst$$reg),
11418               as_Register($src1$$reg),
11419               as_Register($src2$$reg),
11420               Assembler::LSR,
11421               $src3$$constant & 0x3f);
11422   %}
11423 
11424   ins_pipe(ialu_reg_reg_shift);
11425 %}
11426 
11427 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11428                          iRegIorL2I src1, iRegIorL2I src2,
11429                          immI src3, rFlagsReg cr) %{
11430   match(Set dst (OrI src1 (RShiftI src2 src3)));
11431 
11432   ins_cost(1.9 * INSN_COST);
11433   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
11434 
11435   ins_encode %{
11436     __ orrw(as_Register($dst$$reg),
11437               as_Register($src1$$reg),
11438               as_Register($src2$$reg),
11439               Assembler::ASR,
11440               $src3$$constant & 0x1f);
11441   %}
11442 
11443   ins_pipe(ialu_reg_reg_shift);
11444 %}
11445 
11446 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11447                          iRegL src1, iRegL src2,
11448                          immI src3, rFlagsReg cr) %{
11449   match(Set dst (OrL src1 (RShiftL src2 src3)));
11450 
11451   ins_cost(1.9 * INSN_COST);
11452   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
11453 
11454   ins_encode %{
11455     __ orr(as_Register($dst$$reg),
11456               as_Register($src1$$reg),
11457               as_Register($src2$$reg),
11458               Assembler::ASR,
11459               $src3$$constant & 0x3f);
11460   %}
11461 
11462   ins_pipe(ialu_reg_reg_shift);
11463 %}
11464 
11465 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11466                          iRegIorL2I src1, iRegIorL2I src2,
11467                          immI src3, rFlagsReg cr) %{
11468   match(Set dst (OrI src1 (LShiftI src2 src3)));
11469 
11470   ins_cost(1.9 * INSN_COST);
11471   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
11472 
11473   ins_encode %{
11474     __ orrw(as_Register($dst$$reg),
11475               as_Register($src1$$reg),
11476               as_Register($src2$$reg),
11477               Assembler::LSL,
11478               $src3$$constant & 0x1f);
11479   %}
11480 
11481   ins_pipe(ialu_reg_reg_shift);
11482 %}
11483 
11484 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11485                          iRegL src1, iRegL src2,
11486                          immI src3, rFlagsReg cr) %{
11487   match(Set dst (OrL src1 (LShiftL src2 src3)));
11488 
11489   ins_cost(1.9 * INSN_COST);
11490   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
11491 
11492   ins_encode %{
11493     __ orr(as_Register($dst$$reg),
11494               as_Register($src1$$reg),
11495               as_Register($src2$$reg),
11496               Assembler::LSL,
11497               $src3$$constant & 0x3f);
11498   %}
11499 
11500   ins_pipe(ialu_reg_reg_shift);
11501 %}
11502 
11503 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11504                          iRegIorL2I src1, iRegIorL2I src2,
11505                          immI src3, rFlagsReg cr) %{
11506   match(Set dst (AddI src1 (URShiftI src2 src3)));
11507 
11508   ins_cost(1.9 * INSN_COST);
11509   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
11510 
11511   ins_encode %{
11512     __ addw(as_Register($dst$$reg),
11513               as_Register($src1$$reg),
11514               as_Register($src2$$reg),
11515               Assembler::LSR,
11516               $src3$$constant & 0x1f);
11517   %}
11518 
11519   ins_pipe(ialu_reg_reg_shift);
11520 %}
11521 
11522 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11523                          iRegL src1, iRegL src2,
11524                          immI src3, rFlagsReg cr) %{
11525   match(Set dst (AddL src1 (URShiftL src2 src3)));
11526 
11527   ins_cost(1.9 * INSN_COST);
11528   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
11529 
11530   ins_encode %{
11531     __ add(as_Register($dst$$reg),
11532               as_Register($src1$$reg),
11533               as_Register($src2$$reg),
11534               Assembler::LSR,
11535               $src3$$constant & 0x3f);
11536   %}
11537 
11538   ins_pipe(ialu_reg_reg_shift);
11539 %}
11540 
11541 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11542                          iRegIorL2I src1, iRegIorL2I src2,
11543                          immI src3, rFlagsReg cr) %{
11544   match(Set dst (AddI src1 (RShiftI src2 src3)));
11545 
11546   ins_cost(1.9 * INSN_COST);
11547   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
11548 
11549   ins_encode %{
11550     __ addw(as_Register($dst$$reg),
11551               as_Register($src1$$reg),
11552               as_Register($src2$$reg),
11553               Assembler::ASR,
11554               $src3$$constant & 0x1f);
11555   %}
11556 
11557   ins_pipe(ialu_reg_reg_shift);
11558 %}
11559 
11560 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11561                          iRegL src1, iRegL src2,
11562                          immI src3, rFlagsReg cr) %{
11563   match(Set dst (AddL src1 (RShiftL src2 src3)));
11564 
11565   ins_cost(1.9 * INSN_COST);
11566   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
11567 
11568   ins_encode %{
11569     __ add(as_Register($dst$$reg),
11570               as_Register($src1$$reg),
11571               as_Register($src2$$reg),
11572               Assembler::ASR,
11573               $src3$$constant & 0x3f);
11574   %}
11575 
11576   ins_pipe(ialu_reg_reg_shift);
11577 %}
11578 
11579 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11580                          iRegIorL2I src1, iRegIorL2I src2,
11581                          immI src3, rFlagsReg cr) %{
11582   match(Set dst (AddI src1 (LShiftI src2 src3)));
11583 
11584   ins_cost(1.9 * INSN_COST);
11585   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
11586 
11587   ins_encode %{
11588     __ addw(as_Register($dst$$reg),
11589               as_Register($src1$$reg),
11590               as_Register($src2$$reg),
11591               Assembler::LSL,
11592               $src3$$constant & 0x1f);
11593   %}
11594 
11595   ins_pipe(ialu_reg_reg_shift);
11596 %}
11597 
11598 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11599                          iRegL src1, iRegL src2,
11600                          immI src3, rFlagsReg cr) %{
11601   match(Set dst (AddL src1 (LShiftL src2 src3)));
11602 
11603   ins_cost(1.9 * INSN_COST);
11604   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
11605 
11606   ins_encode %{
11607     __ add(as_Register($dst$$reg),
11608               as_Register($src1$$reg),
11609               as_Register($src2$$reg),
11610               Assembler::LSL,
11611               $src3$$constant & 0x3f);
11612   %}
11613 
11614   ins_pipe(ialu_reg_reg_shift);
11615 %}
11616 
11617 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11618                          iRegIorL2I src1, iRegIorL2I src2,
11619                          immI src3, rFlagsReg cr) %{
11620   match(Set dst (SubI src1 (URShiftI src2 src3)));
11621 
11622   ins_cost(1.9 * INSN_COST);
11623   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
11624 
11625   ins_encode %{
11626     __ subw(as_Register($dst$$reg),
11627               as_Register($src1$$reg),
11628               as_Register($src2$$reg),
11629               Assembler::LSR,
11630               $src3$$constant & 0x1f);
11631   %}
11632 
11633   ins_pipe(ialu_reg_reg_shift);
11634 %}
11635 
11636 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11637                          iRegL src1, iRegL src2,
11638                          immI src3, rFlagsReg cr) %{
11639   match(Set dst (SubL src1 (URShiftL src2 src3)));
11640 
11641   ins_cost(1.9 * INSN_COST);
11642   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
11643 
11644   ins_encode %{
11645     __ sub(as_Register($dst$$reg),
11646               as_Register($src1$$reg),
11647               as_Register($src2$$reg),
11648               Assembler::LSR,
11649               $src3$$constant & 0x3f);
11650   %}
11651 
11652   ins_pipe(ialu_reg_reg_shift);
11653 %}
11654 
11655 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11656                          iRegIorL2I src1, iRegIorL2I src2,
11657                          immI src3, rFlagsReg cr) %{
11658   match(Set dst (SubI src1 (RShiftI src2 src3)));
11659 
11660   ins_cost(1.9 * INSN_COST);
11661   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
11662 
11663   ins_encode %{
11664     __ subw(as_Register($dst$$reg),
11665               as_Register($src1$$reg),
11666               as_Register($src2$$reg),
11667               Assembler::ASR,
11668               $src3$$constant & 0x1f);
11669   %}
11670 
11671   ins_pipe(ialu_reg_reg_shift);
11672 %}
11673 
11674 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11675                          iRegL src1, iRegL src2,
11676                          immI src3, rFlagsReg cr) %{
11677   match(Set dst (SubL src1 (RShiftL src2 src3)));
11678 
11679   ins_cost(1.9 * INSN_COST);
11680   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
11681 
11682   ins_encode %{
11683     __ sub(as_Register($dst$$reg),
11684               as_Register($src1$$reg),
11685               as_Register($src2$$reg),
11686               Assembler::ASR,
11687               $src3$$constant & 0x3f);
11688   %}
11689 
11690   ins_pipe(ialu_reg_reg_shift);
11691 %}
11692 
11693 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11694                          iRegIorL2I src1, iRegIorL2I src2,
11695                          immI src3, rFlagsReg cr) %{
11696   match(Set dst (SubI src1 (LShiftI src2 src3)));
11697 
11698   ins_cost(1.9 * INSN_COST);
11699   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
11700 
11701   ins_encode %{
11702     __ subw(as_Register($dst$$reg),
11703               as_Register($src1$$reg),
11704               as_Register($src2$$reg),
11705               Assembler::LSL,
11706               $src3$$constant & 0x1f);
11707   %}
11708 
11709   ins_pipe(ialu_reg_reg_shift);
11710 %}
11711 
11712 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11713                          iRegL src1, iRegL src2,
11714                          immI src3, rFlagsReg cr) %{
11715   match(Set dst (SubL src1 (LShiftL src2 src3)));
11716 
11717   ins_cost(1.9 * INSN_COST);
11718   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
11719 
11720   ins_encode %{
11721     __ sub(as_Register($dst$$reg),
11722               as_Register($src1$$reg),
11723               as_Register($src2$$reg),
11724               Assembler::LSL,
11725               $src3$$constant & 0x3f);
11726   %}
11727 
11728   ins_pipe(ialu_reg_reg_shift);
11729 %}
11730 
11731 
11732 
11733 // Shift Left followed by Shift Right.
11734 // This idiom is used by the compiler for the i2b bytecode etc.
11735 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11736 %{
11737   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11738   ins_cost(INSN_COST * 2);
11739   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11740   ins_encode %{
11741     int lshift = $lshift_count$$constant & 63;
11742     int rshift = $rshift_count$$constant & 63;
11743     int s = 63 - lshift;
11744     int r = (rshift - lshift) & 63;
11745     __ sbfm(as_Register($dst$$reg),
11746             as_Register($src$$reg),
11747             r, s);
11748   %}
11749 
11750   ins_pipe(ialu_reg_shift);
11751 %}
11752 
11753 // Shift Left followed by Shift Right.
11754 // This idiom is used by the compiler for the i2b bytecode etc.
11755 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11756 %{
11757   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11758   ins_cost(INSN_COST * 2);
11759   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11760   ins_encode %{
11761     int lshift = $lshift_count$$constant & 31;
11762     int rshift = $rshift_count$$constant & 31;
11763     int s = 31 - lshift;
11764     int r = (rshift - lshift) & 31;
11765     __ sbfmw(as_Register($dst$$reg),
11766             as_Register($src$$reg),
11767             r, s);
11768   %}
11769 
11770   ins_pipe(ialu_reg_shift);
11771 %}
11772 
11773 // Shift Left followed by Shift Right.
11774 // This idiom is used by the compiler for the i2b bytecode etc.
11775 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11776 %{
11777   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11778   ins_cost(INSN_COST * 2);
11779   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11780   ins_encode %{
11781     int lshift = $lshift_count$$constant & 63;
11782     int rshift = $rshift_count$$constant & 63;
11783     int s = 63 - lshift;
11784     int r = (rshift - lshift) & 63;
11785     __ ubfm(as_Register($dst$$reg),
11786             as_Register($src$$reg),
11787             r, s);
11788   %}
11789 
11790   ins_pipe(ialu_reg_shift);
11791 %}
11792 
11793 // Shift Left followed by Shift Right.
11794 // This idiom is used by the compiler for the i2b bytecode etc.
11795 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11796 %{
11797   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11798   ins_cost(INSN_COST * 2);
11799   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11800   ins_encode %{
11801     int lshift = $lshift_count$$constant & 31;
11802     int rshift = $rshift_count$$constant & 31;
11803     int s = 31 - lshift;
11804     int r = (rshift - lshift) & 31;
11805     __ ubfmw(as_Register($dst$$reg),
11806             as_Register($src$$reg),
11807             r, s);
11808   %}
11809 
11810   ins_pipe(ialu_reg_shift);
11811 %}
11812 // Bitfield extract with shift & mask
11813 
11814 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11815 %{
11816   match(Set dst (AndI (URShiftI src rshift) mask));
11817   // Make sure we are not going to exceed what ubfxw can do.
11818   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11819 
11820   ins_cost(INSN_COST);
11821   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11822   ins_encode %{
11823     int rshift = $rshift$$constant & 31;
11824     long mask = $mask$$constant;
11825     int width = exact_log2(mask+1);
11826     __ ubfxw(as_Register($dst$$reg),
11827             as_Register($src$$reg), rshift, width);
11828   %}
11829   ins_pipe(ialu_reg_shift);
11830 %}
11831 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11832 %{
11833   match(Set dst (AndL (URShiftL src rshift) mask));
11834   // Make sure we are not going to exceed what ubfx can do.
11835   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11836 
11837   ins_cost(INSN_COST);
11838   format %{ "ubfx $dst, $src, $rshift, $mask" %}
11839   ins_encode %{
11840     int rshift = $rshift$$constant & 63;
11841     long mask = $mask$$constant;
11842     int width = exact_log2_long(mask+1);
11843     __ ubfx(as_Register($dst$$reg),
11844             as_Register($src$$reg), rshift, width);
11845   %}
11846   ins_pipe(ialu_reg_shift);
11847 %}
11848 
11849 // We can use ubfx when extending an And with a mask when we know mask
11850 // is positive.  We know that because immI_bitmask guarantees it.
11851 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11852 %{
11853   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11854   // Make sure we are not going to exceed what ubfxw can do.
11855   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11856 
11857   ins_cost(INSN_COST * 2);
11858   format %{ "ubfx $dst, $src, $rshift, $mask" %}
11859   ins_encode %{
11860     int rshift = $rshift$$constant & 31;
11861     long mask = $mask$$constant;
11862     int width = exact_log2(mask+1);
11863     __ ubfx(as_Register($dst$$reg),
11864             as_Register($src$$reg), rshift, width);
11865   %}
11866   ins_pipe(ialu_reg_shift);
11867 %}
11868 
11869 // We can use ubfiz when masking by a positive number and then left shifting the result.
11870 // We know that the mask is positive because immI_bitmask guarantees it.
11871 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11872 %{
11873   match(Set dst (LShiftI (AndI src mask) lshift));
11874   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11875 
11876   ins_cost(INSN_COST);
11877   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11878   ins_encode %{
11879     int lshift = $lshift$$constant & 31;
11880     long mask = $mask$$constant;
11881     int width = exact_log2(mask+1);
11882     __ ubfizw(as_Register($dst$$reg),
11883           as_Register($src$$reg), lshift, width);
11884   %}
11885   ins_pipe(ialu_reg_shift);
11886 %}
11887 // We can use ubfiz when masking by a positive number and then left shifting the result.
11888 // We know that the mask is positive because immL_bitmask guarantees it.
11889 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11890 %{
11891   match(Set dst (LShiftL (AndL src mask) lshift));
11892   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11893 
11894   ins_cost(INSN_COST);
11895   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11896   ins_encode %{
11897     int lshift = $lshift$$constant & 63;
11898     long mask = $mask$$constant;
11899     int width = exact_log2_long(mask+1);
11900     __ ubfiz(as_Register($dst$$reg),
11901           as_Register($src$$reg), lshift, width);
11902   %}
11903   ins_pipe(ialu_reg_shift);
11904 %}
11905 
11906 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11907 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11908 %{
11909   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11910   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11911 
11912   ins_cost(INSN_COST);
11913   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11914   ins_encode %{
11915     int lshift = $lshift$$constant & 63;
11916     long mask = $mask$$constant;
11917     int width = exact_log2(mask+1);
11918     __ ubfiz(as_Register($dst$$reg),
11919              as_Register($src$$reg), lshift, width);
11920   %}
11921   ins_pipe(ialu_reg_shift);
11922 %}
11923 
11924 // Rotations
11925 
11926 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11927 %{
11928   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11929   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11930 
11931   ins_cost(INSN_COST);
11932   format %{ "extr $dst, $src1, $src2, #$rshift" %}
11933 
11934   ins_encode %{
11935     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11936             $rshift$$constant & 63);
11937   %}
11938   ins_pipe(ialu_reg_reg_extr);
11939 %}
11940 
11941 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11942 %{
11943   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11944   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11945 
11946   ins_cost(INSN_COST);
11947   format %{ "extr $dst, $src1, $src2, #$rshift" %}
11948 
11949   ins_encode %{
11950     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11951             $rshift$$constant & 31);
11952   %}
11953   ins_pipe(ialu_reg_reg_extr);
11954 %}
11955 
11956 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11957 %{
11958   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11959   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11960 
11961   ins_cost(INSN_COST);
11962   format %{ "extr $dst, $src1, $src2, #$rshift" %}
11963 
11964   ins_encode %{
11965     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11966             $rshift$$constant & 63);
11967   %}
11968   ins_pipe(ialu_reg_reg_extr);
11969 %}
11970 
11971 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11972 %{
11973   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11974   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11975 
11976   ins_cost(INSN_COST);
11977   format %{ "extr $dst, $src1, $src2, #$rshift" %}
11978 
11979   ins_encode %{
11980     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11981             $rshift$$constant & 31);
11982   %}
11983   ins_pipe(ialu_reg_reg_extr);
11984 %}
11985 
11986 
11987 // rol expander
11988 
11989 instruct rolL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
11990 %{
11991   effect(DEF dst, USE src, USE shift);
11992 
11993   format %{ "rol    $dst, $src, $shift" %}
11994   ins_cost(INSN_COST * 3);
11995   ins_encode %{
11996     __ subw(rscratch1, zr, as_Register($shift$$reg));
11997     __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
11998             rscratch1);
11999     %}
12000   ins_pipe(ialu_reg_reg_vshift);
12001 %}
12002 
12003 // rol expander
12004 
12005 instruct rolI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
12006 %{
12007   effect(DEF dst, USE src, USE shift);
12008 
12009   format %{ "rol    $dst, $src, $shift" %}
12010   ins_cost(INSN_COST * 3);
12011   ins_encode %{
12012     __ subw(rscratch1, zr, as_Register($shift$$reg));
12013     __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
12014             rscratch1);
12015     %}
12016   ins_pipe(ialu_reg_reg_vshift);
12017 %}
12018 
12019 instruct rolL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
12020 %{
12021   match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c_64 shift))));
12022 
12023   expand %{
12024     rolL_rReg(dst, src, shift, cr);
12025   %}
12026 %}
12027 
12028 instruct rolL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
12029 %{
12030   match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c0 shift))));
12031 
12032   expand %{
12033     rolL_rReg(dst, src, shift, cr);
12034   %}
12035 %}
12036 
12037 instruct rolI_rReg_Var_C_32(iRegINoSp dst, iRegI src, iRegI shift, immI_32 c_32, rFlagsReg cr)
12038 %{
12039   match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c_32 shift))));
12040 
12041   expand %{
12042     rolI_rReg(dst, src, shift, cr);
12043   %}
12044 %}
12045 
12046 instruct rolI_rReg_Var_C0(iRegINoSp dst, iRegI src, iRegI shift, immI0 c0, rFlagsReg cr)
12047 %{
12048   match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c0 shift))));
12049 
12050   expand %{
12051     rolI_rReg(dst, src, shift, cr);
12052   %}
12053 %}
12054 
12055 // ror expander
12056 
12057 instruct rorL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
12058 %{
12059   effect(DEF dst, USE src, USE shift);
12060 
12061   format %{ "ror    $dst, $src, $shift" %}
12062   ins_cost(INSN_COST);
12063   ins_encode %{
12064     __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
12065             as_Register($shift$$reg));
12066     %}
12067   ins_pipe(ialu_reg_reg_vshift);
12068 %}
12069 
12070 // ror expander
12071 
12072 instruct rorI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
12073 %{
12074   effect(DEF dst, USE src, USE shift);
12075 
12076   format %{ "ror    $dst, $src, $shift" %}
12077   ins_cost(INSN_COST);
12078   ins_encode %{
12079     __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
12080             as_Register($shift$$reg));
12081     %}
12082   ins_pipe(ialu_reg_reg_vshift);
12083 %}
12084 
12085 instruct rorL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
12086 %{
12087   match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c_64 shift))));
12088 
12089   expand %{
12090     rorL_rReg(dst, src, shift, cr);
12091   %}
12092 %}
12093 
12094 instruct rorL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
12095 %{
12096   match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c0 shift))));
12097 
12098   expand %{
12099     rorL_rReg(dst, src, shift, cr);
12100   %}
12101 %}
12102 
12103 instruct rorI_rReg_Var_C_32(iRegINoSp dst, iRegI src, iRegI shift, immI_32 c_32, rFlagsReg cr)
12104 %{
12105   match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c_32 shift))));
12106 
12107   expand %{
12108     rorI_rReg(dst, src, shift, cr);
12109   %}
12110 %}
12111 
12112 instruct rorI_rReg_Var_C0(iRegINoSp dst, iRegI src, iRegI shift, immI0 c0, rFlagsReg cr)
12113 %{
12114   match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c0 shift))));
12115 
12116   expand %{
12117     rorI_rReg(dst, src, shift, cr);
12118   %}
12119 %}
12120 
12121 // Add/subtract (extended)
12122 
12123 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12124 %{
12125   match(Set dst (AddL src1 (ConvI2L src2)));
12126   ins_cost(INSN_COST);
12127   format %{ "add  $dst, $src1, $src2, sxtw" %}
12128 
12129    ins_encode %{
12130      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12131             as_Register($src2$$reg), ext::sxtw);
12132    %}
12133   ins_pipe(ialu_reg_reg);
12134 %};
12135 
12136 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12137 %{
12138   match(Set dst (SubL src1 (ConvI2L src2)));
12139   ins_cost(INSN_COST);
12140   format %{ "sub  $dst, $src1, $src2, sxtw" %}
12141 
12142    ins_encode %{
12143      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12144             as_Register($src2$$reg), ext::sxtw);
12145    %}
12146   ins_pipe(ialu_reg_reg);
12147 %};
12148 
12149 
12150 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12151 %{
12152   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12153   ins_cost(INSN_COST);
12154   format %{ "add  $dst, $src1, $src2, sxth" %}
12155 
12156    ins_encode %{
12157      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12158             as_Register($src2$$reg), ext::sxth);
12159    %}
12160   ins_pipe(ialu_reg_reg);
12161 %}
12162 
12163 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12164 %{
12165   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12166   ins_cost(INSN_COST);
12167   format %{ "add  $dst, $src1, $src2, sxtb" %}
12168 
12169    ins_encode %{
12170      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12171             as_Register($src2$$reg), ext::sxtb);
12172    %}
12173   ins_pipe(ialu_reg_reg);
12174 %}
12175 
12176 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12177 %{
12178   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12179   ins_cost(INSN_COST);
12180   format %{ "add  $dst, $src1, $src2, uxtb" %}
12181 
12182    ins_encode %{
12183      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12184             as_Register($src2$$reg), ext::uxtb);
12185    %}
12186   ins_pipe(ialu_reg_reg);
12187 %}
12188 
12189 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12190 %{
12191   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12192   ins_cost(INSN_COST);
12193   format %{ "add  $dst, $src1, $src2, sxth" %}
12194 
12195    ins_encode %{
12196      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12197             as_Register($src2$$reg), ext::sxth);
12198    %}
12199   ins_pipe(ialu_reg_reg);
12200 %}
12201 
12202 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12203 %{
12204   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12205   ins_cost(INSN_COST);
12206   format %{ "add  $dst, $src1, $src2, sxtw" %}
12207 
12208    ins_encode %{
12209      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12210             as_Register($src2$$reg), ext::sxtw);
12211    %}
12212   ins_pipe(ialu_reg_reg);
12213 %}
12214 
12215 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12216 %{
12217   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12218   ins_cost(INSN_COST);
12219   format %{ "add  $dst, $src1, $src2, sxtb" %}
12220 
12221    ins_encode %{
12222      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12223             as_Register($src2$$reg), ext::sxtb);
12224    %}
12225   ins_pipe(ialu_reg_reg);
12226 %}
12227 
12228 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12229 %{
12230   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12231   ins_cost(INSN_COST);
12232   format %{ "add  $dst, $src1, $src2, uxtb" %}
12233 
12234    ins_encode %{
12235      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12236             as_Register($src2$$reg), ext::uxtb);
12237    %}
12238   ins_pipe(ialu_reg_reg);
12239 %}
12240 
12241 
12242 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12243 %{
12244   match(Set dst (AddI src1 (AndI src2 mask)));
12245   ins_cost(INSN_COST);
12246   format %{ "addw  $dst, $src1, $src2, uxtb" %}
12247 
12248    ins_encode %{
12249      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12250             as_Register($src2$$reg), ext::uxtb);
12251    %}
12252   ins_pipe(ialu_reg_reg);
12253 %}
12254 
12255 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12256 %{
12257   match(Set dst (AddI src1 (AndI src2 mask)));
12258   ins_cost(INSN_COST);
12259   format %{ "addw  $dst, $src1, $src2, uxth" %}
12260 
12261    ins_encode %{
12262      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12263             as_Register($src2$$reg), ext::uxth);
12264    %}
12265   ins_pipe(ialu_reg_reg);
12266 %}
12267 
12268 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12269 %{
12270   match(Set dst (AddL src1 (AndL src2 mask)));
12271   ins_cost(INSN_COST);
12272   format %{ "add  $dst, $src1, $src2, uxtb" %}
12273 
12274    ins_encode %{
12275      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12276             as_Register($src2$$reg), ext::uxtb);
12277    %}
12278   ins_pipe(ialu_reg_reg);
12279 %}
12280 
12281 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12282 %{
12283   match(Set dst (AddL src1 (AndL src2 mask)));
12284   ins_cost(INSN_COST);
12285   format %{ "add  $dst, $src1, $src2, uxth" %}
12286 
12287    ins_encode %{
12288      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12289             as_Register($src2$$reg), ext::uxth);
12290    %}
12291   ins_pipe(ialu_reg_reg);
12292 %}
12293 
12294 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12295 %{
12296   match(Set dst (AddL src1 (AndL src2 mask)));
12297   ins_cost(INSN_COST);
12298   format %{ "add  $dst, $src1, $src2, uxtw" %}
12299 
12300    ins_encode %{
12301      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12302             as_Register($src2$$reg), ext::uxtw);
12303    %}
12304   ins_pipe(ialu_reg_reg);
12305 %}
12306 
12307 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12308 %{
12309   match(Set dst (SubI src1 (AndI src2 mask)));
12310   ins_cost(INSN_COST);
12311   format %{ "subw  $dst, $src1, $src2, uxtb" %}
12312 
12313    ins_encode %{
12314      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12315             as_Register($src2$$reg), ext::uxtb);
12316    %}
12317   ins_pipe(ialu_reg_reg);
12318 %}
12319 
12320 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12321 %{
12322   match(Set dst (SubI src1 (AndI src2 mask)));
12323   ins_cost(INSN_COST);
12324   format %{ "subw  $dst, $src1, $src2, uxth" %}
12325 
12326    ins_encode %{
12327      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12328             as_Register($src2$$reg), ext::uxth);
12329    %}
12330   ins_pipe(ialu_reg_reg);
12331 %}
12332 
12333 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12334 %{
12335   match(Set dst (SubL src1 (AndL src2 mask)));
12336   ins_cost(INSN_COST);
12337   format %{ "sub  $dst, $src1, $src2, uxtb" %}
12338 
12339    ins_encode %{
12340      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12341             as_Register($src2$$reg), ext::uxtb);
12342    %}
12343   ins_pipe(ialu_reg_reg);
12344 %}
12345 
12346 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12347 %{
12348   match(Set dst (SubL src1 (AndL src2 mask)));
12349   ins_cost(INSN_COST);
12350   format %{ "sub  $dst, $src1, $src2, uxth" %}
12351 
12352    ins_encode %{
12353      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12354             as_Register($src2$$reg), ext::uxth);
12355    %}
12356   ins_pipe(ialu_reg_reg);
12357 %}
12358 
12359 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12360 %{
12361   match(Set dst (SubL src1 (AndL src2 mask)));
12362   ins_cost(INSN_COST);
12363   format %{ "sub  $dst, $src1, $src2, uxtw" %}
12364 
12365    ins_encode %{
12366      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12367             as_Register($src2$$reg), ext::uxtw);
12368    %}
12369   ins_pipe(ialu_reg_reg);
12370 %}
12371 
12372 
12373 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12374 %{
12375   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12376   ins_cost(1.9 * INSN_COST);
12377   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
12378 
12379    ins_encode %{
12380      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12381             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12382    %}
12383   ins_pipe(ialu_reg_reg_shift);
12384 %}
12385 
12386 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12387 %{
12388   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12389   ins_cost(1.9 * INSN_COST);
12390   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
12391 
12392    ins_encode %{
12393      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12394             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12395    %}
12396   ins_pipe(ialu_reg_reg_shift);
12397 %}
12398 
12399 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12400 %{
12401   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12402   ins_cost(1.9 * INSN_COST);
12403   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
12404 
12405    ins_encode %{
12406      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12407             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12408    %}
12409   ins_pipe(ialu_reg_reg_shift);
12410 %}
12411 
12412 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12413 %{
12414   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12415   ins_cost(1.9 * INSN_COST);
12416   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
12417 
12418    ins_encode %{
12419      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12420             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12421    %}
12422   ins_pipe(ialu_reg_reg_shift);
12423 %}
12424 
12425 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12426 %{
12427   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12428   ins_cost(1.9 * INSN_COST);
12429   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
12430 
12431    ins_encode %{
12432      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12433             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12434    %}
12435   ins_pipe(ialu_reg_reg_shift);
12436 %}
12437 
12438 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12439 %{
12440   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12441   ins_cost(1.9 * INSN_COST);
12442   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
12443 
12444    ins_encode %{
12445      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12446             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12447    %}
12448   ins_pipe(ialu_reg_reg_shift);
12449 %}
12450 
12451 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12452 %{
12453   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12454   ins_cost(1.9 * INSN_COST);
12455   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
12456 
12457    ins_encode %{
12458      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12459             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12460    %}
12461   ins_pipe(ialu_reg_reg_shift);
12462 %}
12463 
12464 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12465 %{
12466   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12467   ins_cost(1.9 * INSN_COST);
12468   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
12469 
12470    ins_encode %{
12471      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12472             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12473    %}
12474   ins_pipe(ialu_reg_reg_shift);
12475 %}
12476 
12477 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12478 %{
12479   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12480   ins_cost(1.9 * INSN_COST);
12481   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
12482 
12483    ins_encode %{
12484      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12485             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12486    %}
12487   ins_pipe(ialu_reg_reg_shift);
12488 %}
12489 
12490 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12491 %{
12492   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12493   ins_cost(1.9 * INSN_COST);
12494   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
12495 
12496    ins_encode %{
12497      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12498             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12499    %}
12500   ins_pipe(ialu_reg_reg_shift);
12501 %}
12502 
12503 
12504 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12505 %{
12506   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12507   ins_cost(1.9 * INSN_COST);
12508   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
12509 
12510    ins_encode %{
12511      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12512             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12513    %}
12514   ins_pipe(ialu_reg_reg_shift);
12515 %};
12516 
12517 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12518 %{
12519   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12520   ins_cost(1.9 * INSN_COST);
12521   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
12522 
12523    ins_encode %{
12524      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12525             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12526    %}
12527   ins_pipe(ialu_reg_reg_shift);
12528 %};
12529 
12530 
12531 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12532 %{
12533   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12534   ins_cost(1.9 * INSN_COST);
12535   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
12536 
12537    ins_encode %{
12538      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12539             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12540    %}
12541   ins_pipe(ialu_reg_reg_shift);
12542 %}
12543 
12544 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12545 %{
12546   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12547   ins_cost(1.9 * INSN_COST);
12548   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
12549 
12550    ins_encode %{
12551      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12552             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12553    %}
12554   ins_pipe(ialu_reg_reg_shift);
12555 %}
12556 
12557 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12558 %{
12559   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12560   ins_cost(1.9 * INSN_COST);
12561   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
12562 
12563    ins_encode %{
12564      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12565             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12566    %}
12567   ins_pipe(ialu_reg_reg_shift);
12568 %}
12569 
12570 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12571 %{
12572   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12573   ins_cost(1.9 * INSN_COST);
12574   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
12575 
12576    ins_encode %{
12577      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12578             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12579    %}
12580   ins_pipe(ialu_reg_reg_shift);
12581 %}
12582 
12583 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12584 %{
12585   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12586   ins_cost(1.9 * INSN_COST);
12587   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
12588 
12589    ins_encode %{
12590      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12591             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12592    %}
12593   ins_pipe(ialu_reg_reg_shift);
12594 %}
12595 
12596 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12597 %{
12598   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12599   ins_cost(1.9 * INSN_COST);
12600   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
12601 
12602    ins_encode %{
12603      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12604             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12605    %}
12606   ins_pipe(ialu_reg_reg_shift);
12607 %}
12608 
12609 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12610 %{
12611   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12612   ins_cost(1.9 * INSN_COST);
12613   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
12614 
12615    ins_encode %{
12616      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12617             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12618    %}
12619   ins_pipe(ialu_reg_reg_shift);
12620 %}
12621 
12622 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12623 %{
12624   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12625   ins_cost(1.9 * INSN_COST);
12626   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
12627 
12628    ins_encode %{
12629      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12630             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12631    %}
12632   ins_pipe(ialu_reg_reg_shift);
12633 %}
12634 
12635 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12636 %{
12637   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12638   ins_cost(1.9 * INSN_COST);
12639   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
12640 
12641    ins_encode %{
12642      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12643             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12644    %}
12645   ins_pipe(ialu_reg_reg_shift);
12646 %}
12647 
12648 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12649 %{
12650   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12651   ins_cost(1.9 * INSN_COST);
12652   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
12653 
12654    ins_encode %{
12655      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12656             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12657    %}
12658   ins_pipe(ialu_reg_reg_shift);
12659 %}
12660 // END This section of the file is automatically generated. Do not edit --------------
12661 
12662 // ============================================================================
12663 // Floating Point Arithmetic Instructions
12664 
12665 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12666   match(Set dst (AddF src1 src2));
12667 
12668   ins_cost(INSN_COST * 5);
12669   format %{ "fadds   $dst, $src1, $src2" %}
12670 
12671   ins_encode %{
12672     __ fadds(as_FloatRegister($dst$$reg),
12673              as_FloatRegister($src1$$reg),
12674              as_FloatRegister($src2$$reg));
12675   %}
12676 
12677   ins_pipe(fp_dop_reg_reg_s);
12678 %}
12679 
12680 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12681   match(Set dst (AddD src1 src2));
12682 
12683   ins_cost(INSN_COST * 5);
12684   format %{ "faddd   $dst, $src1, $src2" %}
12685 
12686   ins_encode %{
12687     __ faddd(as_FloatRegister($dst$$reg),
12688              as_FloatRegister($src1$$reg),
12689              as_FloatRegister($src2$$reg));
12690   %}
12691 
12692   ins_pipe(fp_dop_reg_reg_d);
12693 %}
12694 
12695 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12696   match(Set dst (SubF src1 src2));
12697 
12698   ins_cost(INSN_COST * 5);
12699   format %{ "fsubs   $dst, $src1, $src2" %}
12700 
12701   ins_encode %{
12702     __ fsubs(as_FloatRegister($dst$$reg),
12703              as_FloatRegister($src1$$reg),
12704              as_FloatRegister($src2$$reg));
12705   %}
12706 
12707   ins_pipe(fp_dop_reg_reg_s);
12708 %}
12709 
12710 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12711   match(Set dst (SubD src1 src2));
12712 
12713   ins_cost(INSN_COST * 5);
12714   format %{ "fsubd   $dst, $src1, $src2" %}
12715 
12716   ins_encode %{
12717     __ fsubd(as_FloatRegister($dst$$reg),
12718              as_FloatRegister($src1$$reg),
12719              as_FloatRegister($src2$$reg));
12720   %}
12721 
12722   ins_pipe(fp_dop_reg_reg_d);
12723 %}
12724 
12725 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12726   match(Set dst (MulF src1 src2));
12727 
12728   ins_cost(INSN_COST * 6);
12729   format %{ "fmuls   $dst, $src1, $src2" %}
12730 
12731   ins_encode %{
12732     __ fmuls(as_FloatRegister($dst$$reg),
12733              as_FloatRegister($src1$$reg),
12734              as_FloatRegister($src2$$reg));
12735   %}
12736 
12737   ins_pipe(fp_dop_reg_reg_s);
12738 %}
12739 
12740 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12741   match(Set dst (MulD src1 src2));
12742 
12743   ins_cost(INSN_COST * 6);
12744   format %{ "fmuld   $dst, $src1, $src2" %}
12745 
12746   ins_encode %{
12747     __ fmuld(as_FloatRegister($dst$$reg),
12748              as_FloatRegister($src1$$reg),
12749              as_FloatRegister($src2$$reg));
12750   %}
12751 
12752   ins_pipe(fp_dop_reg_reg_d);
12753 %}
12754 
12755 // src1 * src2 + src3
12756 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12757   predicate(UseFMA);
12758   match(Set dst (FmaF src3 (Binary src1 src2)));
12759 
12760   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
12761 
12762   ins_encode %{
12763     __ fmadds(as_FloatRegister($dst$$reg),
12764              as_FloatRegister($src1$$reg),
12765              as_FloatRegister($src2$$reg),
12766              as_FloatRegister($src3$$reg));
12767   %}
12768 
12769   ins_pipe(pipe_class_default);
12770 %}
12771 
12772 // src1 * src2 + src3
12773 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12774   predicate(UseFMA);
12775   match(Set dst (FmaD src3 (Binary src1 src2)));
12776 
12777   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
12778 
12779   ins_encode %{
12780     __ fmaddd(as_FloatRegister($dst$$reg),
12781              as_FloatRegister($src1$$reg),
12782              as_FloatRegister($src2$$reg),
12783              as_FloatRegister($src3$$reg));
12784   %}
12785 
12786   ins_pipe(pipe_class_default);
12787 %}
12788 
12789 // -src1 * src2 + src3
12790 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12791   predicate(UseFMA);
12792   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
12793   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
12794 
12795   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
12796 
12797   ins_encode %{
12798     __ fmsubs(as_FloatRegister($dst$$reg),
12799               as_FloatRegister($src1$$reg),
12800               as_FloatRegister($src2$$reg),
12801               as_FloatRegister($src3$$reg));
12802   %}
12803 
12804   ins_pipe(pipe_class_default);
12805 %}
12806 
12807 // -src1 * src2 + src3
12808 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12809   predicate(UseFMA);
12810   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
12811   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
12812 
12813   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
12814 
12815   ins_encode %{
12816     __ fmsubd(as_FloatRegister($dst$$reg),
12817               as_FloatRegister($src1$$reg),
12818               as_FloatRegister($src2$$reg),
12819               as_FloatRegister($src3$$reg));
12820   %}
12821 
12822   ins_pipe(pipe_class_default);
12823 %}
12824 
12825 // -src1 * src2 - src3
12826 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12827   predicate(UseFMA);
12828   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
12829   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
12830 
12831   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
12832 
12833   ins_encode %{
12834     __ fnmadds(as_FloatRegister($dst$$reg),
12835                as_FloatRegister($src1$$reg),
12836                as_FloatRegister($src2$$reg),
12837                as_FloatRegister($src3$$reg));
12838   %}
12839 
12840   ins_pipe(pipe_class_default);
12841 %}
12842 
12843 // -src1 * src2 - src3
12844 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12845   predicate(UseFMA);
12846   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
12847   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
12848 
12849   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
12850 
12851   ins_encode %{
12852     __ fnmaddd(as_FloatRegister($dst$$reg),
12853                as_FloatRegister($src1$$reg),
12854                as_FloatRegister($src2$$reg),
12855                as_FloatRegister($src3$$reg));
12856   %}
12857 
12858   ins_pipe(pipe_class_default);
12859 %}
12860 
12861 // src1 * src2 - src3
12862 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
12863   predicate(UseFMA);
12864   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
12865 
12866   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
12867 
12868   ins_encode %{
12869     __ fnmsubs(as_FloatRegister($dst$$reg),
12870                as_FloatRegister($src1$$reg),
12871                as_FloatRegister($src2$$reg),
12872                as_FloatRegister($src3$$reg));
12873   %}
12874 
12875   ins_pipe(pipe_class_default);
12876 %}
12877 
12878 // src1 * src2 - src3
12879 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
12880   predicate(UseFMA);
12881   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
12882 
12883   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
12884 
12885   ins_encode %{
12886   // n.b. insn name should be fnmsubd
12887     __ fnmsub(as_FloatRegister($dst$$reg),
12888               as_FloatRegister($src1$$reg),
12889               as_FloatRegister($src2$$reg),
12890               as_FloatRegister($src3$$reg));
12891   %}
12892 
12893   ins_pipe(pipe_class_default);
12894 %}
12895 
12896 
12897 // Math.max(FF)F
12898 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12899   match(Set dst (MaxF src1 src2));
12900 
12901   format %{ "fmaxs   $dst, $src1, $src2" %}
12902   ins_encode %{
12903     __ fmaxs(as_FloatRegister($dst$$reg),
12904              as_FloatRegister($src1$$reg),
12905              as_FloatRegister($src2$$reg));
12906   %}
12907 
12908   ins_pipe(fp_dop_reg_reg_s);
12909 %}
12910 
12911 // Math.min(FF)F
12912 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12913   match(Set dst (MinF src1 src2));
12914 
12915   format %{ "fmins   $dst, $src1, $src2" %}
12916   ins_encode %{
12917     __ fmins(as_FloatRegister($dst$$reg),
12918              as_FloatRegister($src1$$reg),
12919              as_FloatRegister($src2$$reg));
12920   %}
12921 
12922   ins_pipe(fp_dop_reg_reg_s);
12923 %}
12924 
12925 // Math.max(DD)D
12926 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12927   match(Set dst (MaxD src1 src2));
12928 
12929   format %{ "fmaxd   $dst, $src1, $src2" %}
12930   ins_encode %{
12931     __ fmaxd(as_FloatRegister($dst$$reg),
12932              as_FloatRegister($src1$$reg),
12933              as_FloatRegister($src2$$reg));
12934   %}
12935 
12936   ins_pipe(fp_dop_reg_reg_d);
12937 %}
12938 
12939 // Math.min(DD)D
12940 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12941   match(Set dst (MinD src1 src2));
12942 
12943   format %{ "fmind   $dst, $src1, $src2" %}
12944   ins_encode %{
12945     __ fmind(as_FloatRegister($dst$$reg),
12946              as_FloatRegister($src1$$reg),
12947              as_FloatRegister($src2$$reg));
12948   %}
12949 
12950   ins_pipe(fp_dop_reg_reg_d);
12951 %}
12952 
12953 
12954 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12955   match(Set dst (DivF src1  src2));
12956 
12957   ins_cost(INSN_COST * 18);
12958   format %{ "fdivs   $dst, $src1, $src2" %}
12959 
12960   ins_encode %{
12961     __ fdivs(as_FloatRegister($dst$$reg),
12962              as_FloatRegister($src1$$reg),
12963              as_FloatRegister($src2$$reg));
12964   %}
12965 
12966   ins_pipe(fp_div_s);
12967 %}
12968 
12969 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12970   match(Set dst (DivD src1  src2));
12971 
12972   ins_cost(INSN_COST * 32);
12973   format %{ "fdivd   $dst, $src1, $src2" %}
12974 
12975   ins_encode %{
12976     __ fdivd(as_FloatRegister($dst$$reg),
12977              as_FloatRegister($src1$$reg),
12978              as_FloatRegister($src2$$reg));
12979   %}
12980 
12981   ins_pipe(fp_div_d);
12982 %}
12983 
12984 instruct negF_reg_reg(vRegF dst, vRegF src) %{
12985   match(Set dst (NegF src));
12986 
12987   ins_cost(INSN_COST * 3);
12988   format %{ "fneg   $dst, $src" %}
12989 
12990   ins_encode %{
12991     __ fnegs(as_FloatRegister($dst$$reg),
12992              as_FloatRegister($src$$reg));
12993   %}
12994 
12995   ins_pipe(fp_uop_s);
12996 %}
12997 
12998 instruct negD_reg_reg(vRegD dst, vRegD src) %{
12999   match(Set dst (NegD src));
13000 
13001   ins_cost(INSN_COST * 3);
13002   format %{ "fnegd   $dst, $src" %}
13003 
13004   ins_encode %{
13005     __ fnegd(as_FloatRegister($dst$$reg),
13006              as_FloatRegister($src$$reg));
13007   %}
13008 
13009   ins_pipe(fp_uop_d);
13010 %}
13011 
13012 instruct absF_reg(vRegF dst, vRegF src) %{
13013   match(Set dst (AbsF src));
13014 
13015   ins_cost(INSN_COST * 3);
13016   format %{ "fabss   $dst, $src" %}
13017   ins_encode %{
13018     __ fabss(as_FloatRegister($dst$$reg),
13019              as_FloatRegister($src$$reg));
13020   %}
13021 
13022   ins_pipe(fp_uop_s);
13023 %}
13024 
13025 instruct absD_reg(vRegD dst, vRegD src) %{
13026   match(Set dst (AbsD src));
13027 
13028   ins_cost(INSN_COST * 3);
13029   format %{ "fabsd   $dst, $src" %}
13030   ins_encode %{
13031     __ fabsd(as_FloatRegister($dst$$reg),
13032              as_FloatRegister($src$$reg));
13033   %}
13034 
13035   ins_pipe(fp_uop_d);
13036 %}
13037 
13038 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13039   match(Set dst (SqrtD src));
13040 
13041   ins_cost(INSN_COST * 50);
13042   format %{ "fsqrtd  $dst, $src" %}
13043   ins_encode %{
13044     __ fsqrtd(as_FloatRegister($dst$$reg),
13045              as_FloatRegister($src$$reg));
13046   %}
13047 
13048   ins_pipe(fp_div_s);
13049 %}
13050 
13051 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13052   match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
13053 
13054   ins_cost(INSN_COST * 50);
13055   format %{ "fsqrts  $dst, $src" %}
13056   ins_encode %{
13057     __ fsqrts(as_FloatRegister($dst$$reg),
13058              as_FloatRegister($src$$reg));
13059   %}
13060 
13061   ins_pipe(fp_div_d);
13062 %}
13063 
13064 // ============================================================================
13065 // Logical Instructions
13066 
13067 // Integer Logical Instructions
13068 
13069 // And Instructions
13070 
13071 
13072 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13073   match(Set dst (AndI src1 src2));
13074 
13075   format %{ "andw  $dst, $src1, $src2\t# int" %}
13076 
13077   ins_cost(INSN_COST);
13078   ins_encode %{
13079     __ andw(as_Register($dst$$reg),
13080             as_Register($src1$$reg),
13081             as_Register($src2$$reg));
13082   %}
13083 
13084   ins_pipe(ialu_reg_reg);
13085 %}
13086 
13087 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13088   match(Set dst (AndI src1 src2));
13089 
13090   format %{ "andsw  $dst, $src1, $src2\t# int" %}
13091 
13092   ins_cost(INSN_COST);
13093   ins_encode %{
13094     __ andw(as_Register($dst$$reg),
13095             as_Register($src1$$reg),
13096             (unsigned long)($src2$$constant));
13097   %}
13098 
13099   ins_pipe(ialu_reg_imm);
13100 %}
13101 
13102 // Or Instructions
13103 
13104 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13105   match(Set dst (OrI src1 src2));
13106 
13107   format %{ "orrw  $dst, $src1, $src2\t# int" %}
13108 
13109   ins_cost(INSN_COST);
13110   ins_encode %{
13111     __ orrw(as_Register($dst$$reg),
13112             as_Register($src1$$reg),
13113             as_Register($src2$$reg));
13114   %}
13115 
13116   ins_pipe(ialu_reg_reg);
13117 %}
13118 
13119 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13120   match(Set dst (OrI src1 src2));
13121 
13122   format %{ "orrw  $dst, $src1, $src2\t# int" %}
13123 
13124   ins_cost(INSN_COST);
13125   ins_encode %{
13126     __ orrw(as_Register($dst$$reg),
13127             as_Register($src1$$reg),
13128             (unsigned long)($src2$$constant));
13129   %}
13130 
13131   ins_pipe(ialu_reg_imm);
13132 %}
13133 
13134 // Xor Instructions
13135 
13136 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13137   match(Set dst (XorI src1 src2));
13138 
13139   format %{ "eorw  $dst, $src1, $src2\t# int" %}
13140 
13141   ins_cost(INSN_COST);
13142   ins_encode %{
13143     __ eorw(as_Register($dst$$reg),
13144             as_Register($src1$$reg),
13145             as_Register($src2$$reg));
13146   %}
13147 
13148   ins_pipe(ialu_reg_reg);
13149 %}
13150 
13151 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13152   match(Set dst (XorI src1 src2));
13153 
13154   format %{ "eorw  $dst, $src1, $src2\t# int" %}
13155 
13156   ins_cost(INSN_COST);
13157   ins_encode %{
13158     __ eorw(as_Register($dst$$reg),
13159             as_Register($src1$$reg),
13160             (unsigned long)($src2$$constant));
13161   %}
13162 
13163   ins_pipe(ialu_reg_imm);
13164 %}
13165 
13166 // Long Logical Instructions
13167 // TODO
13168 
13169 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13170   match(Set dst (AndL src1 src2));
13171 
13172   format %{ "and  $dst, $src1, $src2\t# int" %}
13173 
13174   ins_cost(INSN_COST);
13175   ins_encode %{
13176     __ andr(as_Register($dst$$reg),
13177             as_Register($src1$$reg),
13178             as_Register($src2$$reg));
13179   %}
13180 
13181   ins_pipe(ialu_reg_reg);
13182 %}
13183 
13184 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13185   match(Set dst (AndL src1 src2));
13186 
13187   format %{ "and  $dst, $src1, $src2\t# int" %}
13188 
13189   ins_cost(INSN_COST);
13190   ins_encode %{
13191     __ andr(as_Register($dst$$reg),
13192             as_Register($src1$$reg),
13193             (unsigned long)($src2$$constant));
13194   %}
13195 
13196   ins_pipe(ialu_reg_imm);
13197 %}
13198 
13199 // Or Instructions
13200 
13201 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13202   match(Set dst (OrL src1 src2));
13203 
13204   format %{ "orr  $dst, $src1, $src2\t# int" %}
13205 
13206   ins_cost(INSN_COST);
13207   ins_encode %{
13208     __ orr(as_Register($dst$$reg),
13209            as_Register($src1$$reg),
13210            as_Register($src2$$reg));
13211   %}
13212 
13213   ins_pipe(ialu_reg_reg);
13214 %}
13215 
13216 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13217   match(Set dst (OrL src1 src2));
13218 
13219   format %{ "orr  $dst, $src1, $src2\t# int" %}
13220 
13221   ins_cost(INSN_COST);
13222   ins_encode %{
13223     __ orr(as_Register($dst$$reg),
13224            as_Register($src1$$reg),
13225            (unsigned long)($src2$$constant));
13226   %}
13227 
13228   ins_pipe(ialu_reg_imm);
13229 %}
13230 
13231 // Xor Instructions
13232 
13233 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13234   match(Set dst (XorL src1 src2));
13235 
13236   format %{ "eor  $dst, $src1, $src2\t# int" %}
13237 
13238   ins_cost(INSN_COST);
13239   ins_encode %{
13240     __ eor(as_Register($dst$$reg),
13241            as_Register($src1$$reg),
13242            as_Register($src2$$reg));
13243   %}
13244 
13245   ins_pipe(ialu_reg_reg);
13246 %}
13247 
13248 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13249   match(Set dst (XorL src1 src2));
13250 
13251   ins_cost(INSN_COST);
13252   format %{ "eor  $dst, $src1, $src2\t# int" %}
13253 
13254   ins_encode %{
13255     __ eor(as_Register($dst$$reg),
13256            as_Register($src1$$reg),
13257            (unsigned long)($src2$$constant));
13258   %}
13259 
13260   ins_pipe(ialu_reg_imm);
13261 %}
13262 
13263 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13264 %{
13265   match(Set dst (ConvI2L src));
13266 
13267   ins_cost(INSN_COST);
13268   format %{ "sxtw  $dst, $src\t# i2l" %}
13269   ins_encode %{
13270     __ sbfm($dst$$Register, $src$$Register, 0, 31);
13271   %}
13272   ins_pipe(ialu_reg_shift);
13273 %}
13274 
13275 // this pattern occurs in bigmath arithmetic
13276 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13277 %{
13278   match(Set dst (AndL (ConvI2L src) mask));
13279 
13280   ins_cost(INSN_COST);
13281   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
13282   ins_encode %{
13283     __ ubfm($dst$$Register, $src$$Register, 0, 31);
13284   %}
13285 
13286   ins_pipe(ialu_reg_shift);
13287 %}
13288 
13289 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13290   match(Set dst (ConvL2I src));
13291 
13292   ins_cost(INSN_COST);
13293   format %{ "movw  $dst, $src \t// l2i" %}
13294 
13295   ins_encode %{
13296     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13297   %}
13298 
13299   ins_pipe(ialu_reg);
13300 %}
13301 
13302 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13303 %{
13304   match(Set dst (Conv2B src));
13305   effect(KILL cr);
13306 
13307   format %{
13308     "cmpw $src, zr\n\t"
13309     "cset $dst, ne"
13310   %}
13311 
13312   ins_encode %{
13313     __ cmpw(as_Register($src$$reg), zr);
13314     __ cset(as_Register($dst$$reg), Assembler::NE);
13315   %}
13316 
13317   ins_pipe(ialu_reg);
13318 %}
13319 
13320 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
13321 %{
13322   match(Set dst (Conv2B src));
13323   effect(KILL cr);
13324 
13325   format %{
13326     "cmp  $src, zr\n\t"
13327     "cset $dst, ne"
13328   %}
13329 
13330   ins_encode %{
13331     __ cmp(as_Register($src$$reg), zr);
13332     __ cset(as_Register($dst$$reg), Assembler::NE);
13333   %}
13334 
13335   ins_pipe(ialu_reg);
13336 %}
13337 
13338 instruct convD2F_reg(vRegF dst, vRegD src) %{
13339   match(Set dst (ConvD2F src));
13340 
13341   ins_cost(INSN_COST * 5);
13342   format %{ "fcvtd  $dst, $src \t// d2f" %}
13343 
13344   ins_encode %{
13345     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13346   %}
13347 
13348   ins_pipe(fp_d2f);
13349 %}
13350 
13351 instruct convF2D_reg(vRegD dst, vRegF src) %{
13352   match(Set dst (ConvF2D src));
13353 
13354   ins_cost(INSN_COST * 5);
13355   format %{ "fcvts  $dst, $src \t// f2d" %}
13356 
13357   ins_encode %{
13358     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13359   %}
13360 
13361   ins_pipe(fp_f2d);
13362 %}
13363 
13364 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13365   match(Set dst (ConvF2I src));
13366 
13367   ins_cost(INSN_COST * 5);
13368   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
13369 
13370   ins_encode %{
13371     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13372   %}
13373 
13374   ins_pipe(fp_f2i);
13375 %}
13376 
13377 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13378   match(Set dst (ConvF2L src));
13379 
13380   ins_cost(INSN_COST * 5);
13381   format %{ "fcvtzs  $dst, $src \t// f2l" %}
13382 
13383   ins_encode %{
13384     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13385   %}
13386 
13387   ins_pipe(fp_f2l);
13388 %}
13389 
13390 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13391   match(Set dst (ConvI2F src));
13392 
13393   ins_cost(INSN_COST * 5);
13394   format %{ "scvtfws  $dst, $src \t// i2f" %}
13395 
13396   ins_encode %{
13397     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13398   %}
13399 
13400   ins_pipe(fp_i2f);
13401 %}
13402 
13403 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13404   match(Set dst (ConvL2F src));
13405 
13406   ins_cost(INSN_COST * 5);
13407   format %{ "scvtfs  $dst, $src \t// l2f" %}
13408 
13409   ins_encode %{
13410     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13411   %}
13412 
13413   ins_pipe(fp_l2f);
13414 %}
13415 
13416 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13417   match(Set dst (ConvD2I src));
13418 
13419   ins_cost(INSN_COST * 5);
13420   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
13421 
13422   ins_encode %{
13423     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13424   %}
13425 
13426   ins_pipe(fp_d2i);
13427 %}
13428 
13429 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13430   match(Set dst (ConvD2L src));
13431 
13432   ins_cost(INSN_COST * 5);
13433   format %{ "fcvtzd  $dst, $src \t// d2l" %}
13434 
13435   ins_encode %{
13436     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13437   %}
13438 
13439   ins_pipe(fp_d2l);
13440 %}
13441 
13442 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13443   match(Set dst (ConvI2D src));
13444 
13445   ins_cost(INSN_COST * 5);
13446   format %{ "scvtfwd  $dst, $src \t// i2d" %}
13447 
13448   ins_encode %{
13449     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13450   %}
13451 
13452   ins_pipe(fp_i2d);
13453 %}
13454 
13455 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13456   match(Set dst (ConvL2D src));
13457 
13458   ins_cost(INSN_COST * 5);
13459   format %{ "scvtfd  $dst, $src \t// l2d" %}
13460 
13461   ins_encode %{
13462     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13463   %}
13464 
13465   ins_pipe(fp_l2d);
13466 %}
13467 
13468 // stack <-> reg and reg <-> reg shuffles with no conversion
13469 
13470 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13471 
13472   match(Set dst (MoveF2I src));
13473 
13474   effect(DEF dst, USE src);
13475 
13476   ins_cost(4 * INSN_COST);
13477 
13478   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13479 
13480   ins_encode %{
13481     __ ldrw($dst$$Register, Address(sp, $src$$disp));
13482   %}
13483 
13484   ins_pipe(iload_reg_reg);
13485 
13486 %}
13487 
13488 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13489 
13490   match(Set dst (MoveI2F src));
13491 
13492   effect(DEF dst, USE src);
13493 
13494   ins_cost(4 * INSN_COST);
13495 
13496   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13497 
13498   ins_encode %{
13499     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13500   %}
13501 
13502   ins_pipe(pipe_class_memory);
13503 
13504 %}
13505 
13506 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13507 
13508   match(Set dst (MoveD2L src));
13509 
13510   effect(DEF dst, USE src);
13511 
13512   ins_cost(4 * INSN_COST);
13513 
13514   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13515 
13516   ins_encode %{
13517     __ ldr($dst$$Register, Address(sp, $src$$disp));
13518   %}
13519 
13520   ins_pipe(iload_reg_reg);
13521 
13522 %}
13523 
13524 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13525 
13526   match(Set dst (MoveL2D src));
13527 
13528   effect(DEF dst, USE src);
13529 
13530   ins_cost(4 * INSN_COST);
13531 
13532   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13533 
13534   ins_encode %{
13535     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13536   %}
13537 
13538   ins_pipe(pipe_class_memory);
13539 
13540 %}
13541 
13542 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13543 
13544   match(Set dst (MoveF2I src));
13545 
13546   effect(DEF dst, USE src);
13547 
13548   ins_cost(INSN_COST);
13549 
13550   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13551 
13552   ins_encode %{
13553     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13554   %}
13555 
13556   ins_pipe(pipe_class_memory);
13557 
13558 %}
13559 
13560 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
13561 
13562   match(Set dst (MoveI2F src));
13563 
13564   effect(DEF dst, USE src);
13565 
13566   ins_cost(INSN_COST);
13567 
13568   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
13569 
13570   ins_encode %{
13571     __ strw($src$$Register, Address(sp, $dst$$disp));
13572   %}
13573 
13574   ins_pipe(istore_reg_reg);
13575 
13576 %}
13577 
13578 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
13579 
13580   match(Set dst (MoveD2L src));
13581 
13582   effect(DEF dst, USE src);
13583 
13584   ins_cost(INSN_COST);
13585 
13586   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
13587 
13588   ins_encode %{
13589     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13590   %}
13591 
13592   ins_pipe(pipe_class_memory);
13593 
13594 %}
13595 
13596 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
13597 
13598   match(Set dst (MoveL2D src));
13599 
13600   effect(DEF dst, USE src);
13601 
13602   ins_cost(INSN_COST);
13603 
13604   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
13605 
13606   ins_encode %{
13607     __ str($src$$Register, Address(sp, $dst$$disp));
13608   %}
13609 
13610   ins_pipe(istore_reg_reg);
13611 
13612 %}
13613 
13614 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13615 
13616   match(Set dst (MoveF2I src));
13617 
13618   effect(DEF dst, USE src);
13619 
13620   ins_cost(INSN_COST);
13621 
13622   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
13623 
13624   ins_encode %{
13625     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
13626   %}
13627 
13628   ins_pipe(fp_f2i);
13629 
13630 %}
13631 
13632 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
13633 
13634   match(Set dst (MoveI2F src));
13635 
13636   effect(DEF dst, USE src);
13637 
13638   ins_cost(INSN_COST);
13639 
13640   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
13641 
13642   ins_encode %{
13643     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
13644   %}
13645 
13646   ins_pipe(fp_i2f);
13647 
13648 %}
13649 
13650 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13651 
13652   match(Set dst (MoveD2L src));
13653 
13654   effect(DEF dst, USE src);
13655 
13656   ins_cost(INSN_COST);
13657 
13658   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
13659 
13660   ins_encode %{
13661     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
13662   %}
13663 
13664   ins_pipe(fp_d2l);
13665 
13666 %}
13667 
13668 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
13669 
13670   match(Set dst (MoveL2D src));
13671 
13672   effect(DEF dst, USE src);
13673 
13674   ins_cost(INSN_COST);
13675 
13676   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
13677 
13678   ins_encode %{
13679     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
13680   %}
13681 
13682   ins_pipe(fp_l2d);
13683 
13684 %}
13685 
13686 // ============================================================================
13687 // clearing of an array
13688 
13689 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
13690 %{
13691   match(Set dummy (ClearArray cnt base));
13692   effect(USE_KILL cnt, USE_KILL base);
13693 
13694   ins_cost(4 * INSN_COST);
13695   format %{ "ClearArray $cnt, $base" %}
13696 
13697   ins_encode %{
13698     __ zero_words($base$$Register, $cnt$$Register);
13699   %}
13700 
13701   ins_pipe(pipe_class_memory);
13702 %}
13703 
13704 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
13705 %{
13706   predicate((u_int64_t)n->in(2)->get_long()
13707             < (u_int64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
13708   match(Set dummy (ClearArray cnt base));
13709   effect(USE_KILL base);
13710 
13711   ins_cost(4 * INSN_COST);
13712   format %{ "ClearArray $cnt, $base" %}
13713 
13714   ins_encode %{
13715     __ zero_words($base$$Register, (u_int64_t)$cnt$$constant);
13716   %}
13717 
13718   ins_pipe(pipe_class_memory);
13719 %}
13720 
13721 // ============================================================================
13722 // Overflow Math Instructions
13723 
13724 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
13725 %{
13726   match(Set cr (OverflowAddI op1 op2));
13727 
13728   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
13729   ins_cost(INSN_COST);
13730   ins_encode %{
13731     __ cmnw($op1$$Register, $op2$$Register);
13732   %}
13733 
13734   ins_pipe(icmp_reg_reg);
13735 %}
13736 
13737 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
13738 %{
13739   match(Set cr (OverflowAddI op1 op2));
13740 
13741   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
13742   ins_cost(INSN_COST);
13743   ins_encode %{
13744     __ cmnw($op1$$Register, $op2$$constant);
13745   %}
13746 
13747   ins_pipe(icmp_reg_imm);
13748 %}
13749 
13750 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
13751 %{
13752   match(Set cr (OverflowAddL op1 op2));
13753 
13754   format %{ "cmn   $op1, $op2\t# overflow check long" %}
13755   ins_cost(INSN_COST);
13756   ins_encode %{
13757     __ cmn($op1$$Register, $op2$$Register);
13758   %}
13759 
13760   ins_pipe(icmp_reg_reg);
13761 %}
13762 
13763 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
13764 %{
13765   match(Set cr (OverflowAddL op1 op2));
13766 
13767   format %{ "cmn   $op1, $op2\t# overflow check long" %}
13768   ins_cost(INSN_COST);
13769   ins_encode %{
13770     __ cmn($op1$$Register, $op2$$constant);
13771   %}
13772 
13773   ins_pipe(icmp_reg_imm);
13774 %}
13775 
13776 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
13777 %{
13778   match(Set cr (OverflowSubI op1 op2));
13779 
13780   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
13781   ins_cost(INSN_COST);
13782   ins_encode %{
13783     __ cmpw($op1$$Register, $op2$$Register);
13784   %}
13785 
13786   ins_pipe(icmp_reg_reg);
13787 %}
13788 
13789 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
13790 %{
13791   match(Set cr (OverflowSubI op1 op2));
13792 
13793   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
13794   ins_cost(INSN_COST);
13795   ins_encode %{
13796     __ cmpw($op1$$Register, $op2$$constant);
13797   %}
13798 
13799   ins_pipe(icmp_reg_imm);
13800 %}
13801 
13802 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
13803 %{
13804   match(Set cr (OverflowSubL op1 op2));
13805 
13806   format %{ "cmp   $op1, $op2\t# overflow check long" %}
13807   ins_cost(INSN_COST);
13808   ins_encode %{
13809     __ cmp($op1$$Register, $op2$$Register);
13810   %}
13811 
13812   ins_pipe(icmp_reg_reg);
13813 %}
13814 
13815 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
13816 %{
13817   match(Set cr (OverflowSubL op1 op2));
13818 
13819   format %{ "cmp   $op1, $op2\t# overflow check long" %}
13820   ins_cost(INSN_COST);
13821   ins_encode %{
13822     __ subs(zr, $op1$$Register, $op2$$constant);
13823   %}
13824 
13825   ins_pipe(icmp_reg_imm);
13826 %}
13827 
13828 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
13829 %{
13830   match(Set cr (OverflowSubI zero op1));
13831 
13832   format %{ "cmpw  zr, $op1\t# overflow check int" %}
13833   ins_cost(INSN_COST);
13834   ins_encode %{
13835     __ cmpw(zr, $op1$$Register);
13836   %}
13837 
13838   ins_pipe(icmp_reg_imm);
13839 %}
13840 
13841 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
13842 %{
13843   match(Set cr (OverflowSubL zero op1));
13844 
13845   format %{ "cmp   zr, $op1\t# overflow check long" %}
13846   ins_cost(INSN_COST);
13847   ins_encode %{
13848     __ cmp(zr, $op1$$Register);
13849   %}
13850 
13851   ins_pipe(icmp_reg_imm);
13852 %}
13853 
13854 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
13855 %{
13856   match(Set cr (OverflowMulI op1 op2));
13857 
13858   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
13859             "cmp   rscratch1, rscratch1, sxtw\n\t"
13860             "movw  rscratch1, #0x80000000\n\t"
13861             "cselw rscratch1, rscratch1, zr, NE\n\t"
13862             "cmpw  rscratch1, #1" %}
13863   ins_cost(5 * INSN_COST);
13864   ins_encode %{
13865     __ smull(rscratch1, $op1$$Register, $op2$$Register);
13866     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
13867     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
13868     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
13869     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
13870   %}
13871 
13872   ins_pipe(pipe_slow);
13873 %}
13874 
13875 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
13876 %{
13877   match(If cmp (OverflowMulI op1 op2));
13878   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
13879             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
13880   effect(USE labl, KILL cr);
13881 
13882   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
13883             "cmp   rscratch1, rscratch1, sxtw\n\t"
13884             "b$cmp   $labl" %}
13885   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
13886   ins_encode %{
13887     Label* L = $labl$$label;
13888     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13889     __ smull(rscratch1, $op1$$Register, $op2$$Register);
13890     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
13891     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
13892   %}
13893 
13894   ins_pipe(pipe_serial);
13895 %}
13896 
13897 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
13898 %{
13899   match(Set cr (OverflowMulL op1 op2));
13900 
13901   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
13902             "smulh rscratch2, $op1, $op2\n\t"
13903             "cmp   rscratch2, rscratch1, ASR #63\n\t"
13904             "movw  rscratch1, #0x80000000\n\t"
13905             "cselw rscratch1, rscratch1, zr, NE\n\t"
13906             "cmpw  rscratch1, #1" %}
13907   ins_cost(6 * INSN_COST);
13908   ins_encode %{
13909     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
13910     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
13911     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
13912     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
13913     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
13914     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
13915   %}
13916 
13917   ins_pipe(pipe_slow);
13918 %}
13919 
13920 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
13921 %{
13922   match(If cmp (OverflowMulL op1 op2));
13923   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
13924             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
13925   effect(USE labl, KILL cr);
13926 
13927   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
13928             "smulh rscratch2, $op1, $op2\n\t"
13929             "cmp   rscratch2, rscratch1, ASR #63\n\t"
13930             "b$cmp $labl" %}
13931   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
13932   ins_encode %{
13933     Label* L = $labl$$label;
13934     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13935     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
13936     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
13937     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
13938     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
13939   %}
13940 
13941   ins_pipe(pipe_serial);
13942 %}
13943 
13944 // ============================================================================
13945 // Compare Instructions
13946 
13947 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
13948 %{
13949   match(Set cr (CmpI op1 op2));
13950 
13951   effect(DEF cr, USE op1, USE op2);
13952 
13953   ins_cost(INSN_COST);
13954   format %{ "cmpw  $op1, $op2" %}
13955 
13956   ins_encode(aarch64_enc_cmpw(op1, op2));
13957 
13958   ins_pipe(icmp_reg_reg);
13959 %}
13960 
13961 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
13962 %{
13963   match(Set cr (CmpI op1 zero));
13964 
13965   effect(DEF cr, USE op1);
13966 
13967   ins_cost(INSN_COST);
13968   format %{ "cmpw $op1, 0" %}
13969 
13970   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
13971 
13972   ins_pipe(icmp_reg_imm);
13973 %}
13974 
13975 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
13976 %{
13977   match(Set cr (CmpI op1 op2));
13978 
13979   effect(DEF cr, USE op1);
13980 
13981   ins_cost(INSN_COST);
13982   format %{ "cmpw  $op1, $op2" %}
13983 
13984   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
13985 
13986   ins_pipe(icmp_reg_imm);
13987 %}
13988 
13989 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
13990 %{
13991   match(Set cr (CmpI op1 op2));
13992 
13993   effect(DEF cr, USE op1);
13994 
13995   ins_cost(INSN_COST * 2);
13996   format %{ "cmpw  $op1, $op2" %}
13997 
13998   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
13999 
14000   ins_pipe(icmp_reg_imm);
14001 %}
14002 
14003 // Unsigned compare Instructions; really, same as signed compare
14004 // except it should only be used to feed an If or a CMovI which takes a
14005 // cmpOpU.
14006 
14007 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14008 %{
14009   match(Set cr (CmpU op1 op2));
14010 
14011   effect(DEF cr, USE op1, USE op2);
14012 
14013   ins_cost(INSN_COST);
14014   format %{ "cmpw  $op1, $op2\t# unsigned" %}
14015 
14016   ins_encode(aarch64_enc_cmpw(op1, op2));
14017 
14018   ins_pipe(icmp_reg_reg);
14019 %}
14020 
14021 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14022 %{
14023   match(Set cr (CmpU op1 zero));
14024 
14025   effect(DEF cr, USE op1);
14026 
14027   ins_cost(INSN_COST);
14028   format %{ "cmpw $op1, #0\t# unsigned" %}
14029 
14030   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14031 
14032   ins_pipe(icmp_reg_imm);
14033 %}
14034 
14035 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14036 %{
14037   match(Set cr (CmpU op1 op2));
14038 
14039   effect(DEF cr, USE op1);
14040 
14041   ins_cost(INSN_COST);
14042   format %{ "cmpw  $op1, $op2\t# unsigned" %}
14043 
14044   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14045 
14046   ins_pipe(icmp_reg_imm);
14047 %}
14048 
14049 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14050 %{
14051   match(Set cr (CmpU op1 op2));
14052 
14053   effect(DEF cr, USE op1);
14054 
14055   ins_cost(INSN_COST * 2);
14056   format %{ "cmpw  $op1, $op2\t# unsigned" %}
14057 
14058   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14059 
14060   ins_pipe(icmp_reg_imm);
14061 %}
14062 
14063 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14064 %{
14065   match(Set cr (CmpL op1 op2));
14066 
14067   effect(DEF cr, USE op1, USE op2);
14068 
14069   ins_cost(INSN_COST);
14070   format %{ "cmp  $op1, $op2" %}
14071 
14072   ins_encode(aarch64_enc_cmp(op1, op2));
14073 
14074   ins_pipe(icmp_reg_reg);
14075 %}
14076 
14077 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14078 %{
14079   match(Set cr (CmpL op1 zero));
14080 
14081   effect(DEF cr, USE op1);
14082 
14083   ins_cost(INSN_COST);
14084   format %{ "tst  $op1" %}
14085 
14086   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14087 
14088   ins_pipe(icmp_reg_imm);
14089 %}
14090 
14091 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14092 %{
14093   match(Set cr (CmpL op1 op2));
14094 
14095   effect(DEF cr, USE op1);
14096 
14097   ins_cost(INSN_COST);
14098   format %{ "cmp  $op1, $op2" %}
14099 
14100   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14101 
14102   ins_pipe(icmp_reg_imm);
14103 %}
14104 
14105 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14106 %{
14107   match(Set cr (CmpL op1 op2));
14108 
14109   effect(DEF cr, USE op1);
14110 
14111   ins_cost(INSN_COST * 2);
14112   format %{ "cmp  $op1, $op2" %}
14113 
14114   ins_encode(aarch64_enc_cmp_imm(op1, op2));
14115 
14116   ins_pipe(icmp_reg_imm);
14117 %}
14118 
14119 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14120 %{
14121   match(Set cr (CmpUL op1 op2));
14122 
14123   effect(DEF cr, USE op1, USE op2);
14124 
14125   ins_cost(INSN_COST);
14126   format %{ "cmp  $op1, $op2" %}
14127 
14128   ins_encode(aarch64_enc_cmp(op1, op2));
14129 
14130   ins_pipe(icmp_reg_reg);
14131 %}
14132 
14133 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14134 %{
14135   match(Set cr (CmpUL op1 zero));
14136 
14137   effect(DEF cr, USE op1);
14138 
14139   ins_cost(INSN_COST);
14140   format %{ "tst  $op1" %}
14141 
14142   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14143 
14144   ins_pipe(icmp_reg_imm);
14145 %}
14146 
14147 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14148 %{
14149   match(Set cr (CmpUL op1 op2));
14150 
14151   effect(DEF cr, USE op1);
14152 
14153   ins_cost(INSN_COST);
14154   format %{ "cmp  $op1, $op2" %}
14155 
14156   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14157 
14158   ins_pipe(icmp_reg_imm);
14159 %}
14160 
14161 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14162 %{
14163   match(Set cr (CmpUL op1 op2));
14164 
14165   effect(DEF cr, USE op1);
14166 
14167   ins_cost(INSN_COST * 2);
14168   format %{ "cmp  $op1, $op2" %}
14169 
14170   ins_encode(aarch64_enc_cmp_imm(op1, op2));
14171 
14172   ins_pipe(icmp_reg_imm);
14173 %}
14174 
14175 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14176 %{
14177   match(Set cr (CmpP op1 op2));
14178 
14179   effect(DEF cr, USE op1, USE op2);
14180 
14181   ins_cost(INSN_COST);
14182   format %{ "cmp  $op1, $op2\t // ptr" %}
14183 
14184   ins_encode(aarch64_enc_cmpp(op1, op2));
14185 
14186   ins_pipe(icmp_reg_reg);
14187 %}
14188 
14189 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14190 %{
14191   match(Set cr (CmpN op1 op2));
14192 
14193   effect(DEF cr, USE op1, USE op2);
14194 
14195   ins_cost(INSN_COST);
14196   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
14197 
14198   ins_encode(aarch64_enc_cmpn(op1, op2));
14199 
14200   ins_pipe(icmp_reg_reg);
14201 %}
14202 
14203 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14204 %{
14205   match(Set cr (CmpP op1 zero));
14206 
14207   effect(DEF cr, USE op1, USE zero);
14208 
14209   ins_cost(INSN_COST);
14210   format %{ "cmp  $op1, 0\t // ptr" %}
14211 
14212   ins_encode(aarch64_enc_testp(op1));
14213 
14214   ins_pipe(icmp_reg_imm);
14215 %}
14216 
14217 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14218 %{
14219   match(Set cr (CmpN op1 zero));
14220 
14221   effect(DEF cr, USE op1, USE zero);
14222 
14223   ins_cost(INSN_COST);
14224   format %{ "cmp  $op1, 0\t // compressed ptr" %}
14225 
14226   ins_encode(aarch64_enc_testn(op1));
14227 
14228   ins_pipe(icmp_reg_imm);
14229 %}
14230 
14231 // FP comparisons
14232 //
14233 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14234 // using normal cmpOp. See declaration of rFlagsReg for details.
14235 
14236 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14237 %{
14238   match(Set cr (CmpF src1 src2));
14239 
14240   ins_cost(3 * INSN_COST);
14241   format %{ "fcmps $src1, $src2" %}
14242 
14243   ins_encode %{
14244     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14245   %}
14246 
14247   ins_pipe(pipe_class_compare);
14248 %}
14249 
14250 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14251 %{
14252   match(Set cr (CmpF src1 src2));
14253 
14254   ins_cost(3 * INSN_COST);
14255   format %{ "fcmps $src1, 0.0" %}
14256 
14257   ins_encode %{
14258     __ fcmps(as_FloatRegister($src1$$reg), 0.0D);
14259   %}
14260 
14261   ins_pipe(pipe_class_compare);
14262 %}
14263 // FROM HERE
14264 
14265 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14266 %{
14267   match(Set cr (CmpD src1 src2));
14268 
14269   ins_cost(3 * INSN_COST);
14270   format %{ "fcmpd $src1, $src2" %}
14271 
14272   ins_encode %{
14273     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14274   %}
14275 
14276   ins_pipe(pipe_class_compare);
14277 %}
14278 
14279 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14280 %{
14281   match(Set cr (CmpD src1 src2));
14282 
14283   ins_cost(3 * INSN_COST);
14284   format %{ "fcmpd $src1, 0.0" %}
14285 
14286   ins_encode %{
14287     __ fcmpd(as_FloatRegister($src1$$reg), 0.0D);
14288   %}
14289 
14290   ins_pipe(pipe_class_compare);
14291 %}
14292 
14293 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14294 %{
14295   match(Set dst (CmpF3 src1 src2));
14296   effect(KILL cr);
14297 
14298   ins_cost(5 * INSN_COST);
14299   format %{ "fcmps $src1, $src2\n\t"
14300             "csinvw($dst, zr, zr, eq\n\t"
14301             "csnegw($dst, $dst, $dst, lt)"
14302   %}
14303 
14304   ins_encode %{
14305     Label done;
14306     FloatRegister s1 = as_FloatRegister($src1$$reg);
14307     FloatRegister s2 = as_FloatRegister($src2$$reg);
14308     Register d = as_Register($dst$$reg);
14309     __ fcmps(s1, s2);
14310     // installs 0 if EQ else -1
14311     __ csinvw(d, zr, zr, Assembler::EQ);
14312     // keeps -1 if less or unordered else installs 1
14313     __ csnegw(d, d, d, Assembler::LT);
14314     __ bind(done);
14315   %}
14316 
14317   ins_pipe(pipe_class_default);
14318 
14319 %}
14320 
14321 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14322 %{
14323   match(Set dst (CmpD3 src1 src2));
14324   effect(KILL cr);
14325 
14326   ins_cost(5 * INSN_COST);
14327   format %{ "fcmpd $src1, $src2\n\t"
14328             "csinvw($dst, zr, zr, eq\n\t"
14329             "csnegw($dst, $dst, $dst, lt)"
14330   %}
14331 
14332   ins_encode %{
14333     Label done;
14334     FloatRegister s1 = as_FloatRegister($src1$$reg);
14335     FloatRegister s2 = as_FloatRegister($src2$$reg);
14336     Register d = as_Register($dst$$reg);
14337     __ fcmpd(s1, s2);
14338     // installs 0 if EQ else -1
14339     __ csinvw(d, zr, zr, Assembler::EQ);
14340     // keeps -1 if less or unordered else installs 1
14341     __ csnegw(d, d, d, Assembler::LT);
14342     __ bind(done);
14343   %}
14344   ins_pipe(pipe_class_default);
14345 
14346 %}
14347 
14348 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14349 %{
14350   match(Set dst (CmpF3 src1 zero));
14351   effect(KILL cr);
14352 
14353   ins_cost(5 * INSN_COST);
14354   format %{ "fcmps $src1, 0.0\n\t"
14355             "csinvw($dst, zr, zr, eq\n\t"
14356             "csnegw($dst, $dst, $dst, lt)"
14357   %}
14358 
14359   ins_encode %{
14360     Label done;
14361     FloatRegister s1 = as_FloatRegister($src1$$reg);
14362     Register d = as_Register($dst$$reg);
14363     __ fcmps(s1, 0.0D);
14364     // installs 0 if EQ else -1
14365     __ csinvw(d, zr, zr, Assembler::EQ);
14366     // keeps -1 if less or unordered else installs 1
14367     __ csnegw(d, d, d, Assembler::LT);
14368     __ bind(done);
14369   %}
14370 
14371   ins_pipe(pipe_class_default);
14372 
14373 %}
14374 
14375 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14376 %{
14377   match(Set dst (CmpD3 src1 zero));
14378   effect(KILL cr);
14379 
14380   ins_cost(5 * INSN_COST);
14381   format %{ "fcmpd $src1, 0.0\n\t"
14382             "csinvw($dst, zr, zr, eq\n\t"
14383             "csnegw($dst, $dst, $dst, lt)"
14384   %}
14385 
14386   ins_encode %{
14387     Label done;
14388     FloatRegister s1 = as_FloatRegister($src1$$reg);
14389     Register d = as_Register($dst$$reg);
14390     __ fcmpd(s1, 0.0D);
14391     // installs 0 if EQ else -1
14392     __ csinvw(d, zr, zr, Assembler::EQ);
14393     // keeps -1 if less or unordered else installs 1
14394     __ csnegw(d, d, d, Assembler::LT);
14395     __ bind(done);
14396   %}
14397   ins_pipe(pipe_class_default);
14398 
14399 %}
14400 
14401 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14402 %{
14403   match(Set dst (CmpLTMask p q));
14404   effect(KILL cr);
14405 
14406   ins_cost(3 * INSN_COST);
14407 
14408   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14409             "csetw $dst, lt\n\t"
14410             "subw $dst, zr, $dst"
14411   %}
14412 
14413   ins_encode %{
14414     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14415     __ csetw(as_Register($dst$$reg), Assembler::LT);
14416     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14417   %}
14418 
14419   ins_pipe(ialu_reg_reg);
14420 %}
14421 
14422 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14423 %{
14424   match(Set dst (CmpLTMask src zero));
14425   effect(KILL cr);
14426 
14427   ins_cost(INSN_COST);
14428 
14429   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14430 
14431   ins_encode %{
14432     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14433   %}
14434 
14435   ins_pipe(ialu_reg_shift);
14436 %}
14437 
14438 // ============================================================================
14439 // Max and Min
14440 
14441 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
14442 %{
14443   effect( DEF dst, USE src1, USE src2, USE cr );
14444 
14445   ins_cost(INSN_COST * 2);
14446   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
14447 
14448   ins_encode %{
14449     __ cselw(as_Register($dst$$reg),
14450              as_Register($src1$$reg),
14451              as_Register($src2$$reg),
14452              Assembler::LT);
14453   %}
14454 
14455   ins_pipe(icond_reg_reg);
14456 %}
14457 
14458 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
14459 %{
14460   match(Set dst (MinI src1 src2));
14461   ins_cost(INSN_COST * 3);
14462 
14463   expand %{
14464     rFlagsReg cr;
14465     compI_reg_reg(cr, src1, src2);
14466     cmovI_reg_reg_lt(dst, src1, src2, cr);
14467   %}
14468 
14469 %}
14470 // FROM HERE
14471 
14472 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
14473 %{
14474   effect( DEF dst, USE src1, USE src2, USE cr );
14475 
14476   ins_cost(INSN_COST * 2);
14477   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
14478 
14479   ins_encode %{
14480     __ cselw(as_Register($dst$$reg),
14481              as_Register($src1$$reg),
14482              as_Register($src2$$reg),
14483              Assembler::GT);
14484   %}
14485 
14486   ins_pipe(icond_reg_reg);
14487 %}
14488 
14489 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
14490 %{
14491   match(Set dst (MaxI src1 src2));
14492   ins_cost(INSN_COST * 3);
14493   expand %{
14494     rFlagsReg cr;
14495     compI_reg_reg(cr, src1, src2);
14496     cmovI_reg_reg_gt(dst, src1, src2, cr);
14497   %}
14498 %}
14499 
14500 // ============================================================================
14501 // Branch Instructions
14502 
14503 // Direct Branch.
14504 instruct branch(label lbl)
14505 %{
14506   match(Goto);
14507 
14508   effect(USE lbl);
14509 
14510   ins_cost(BRANCH_COST);
14511   format %{ "b  $lbl" %}
14512 
14513   ins_encode(aarch64_enc_b(lbl));
14514 
14515   ins_pipe(pipe_branch);
14516 %}
14517 
14518 // Conditional Near Branch
14519 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14520 %{
14521   // Same match rule as `branchConFar'.
14522   match(If cmp cr);
14523 
14524   effect(USE lbl);
14525 
14526   ins_cost(BRANCH_COST);
14527   // If set to 1 this indicates that the current instruction is a
14528   // short variant of a long branch. This avoids using this
14529   // instruction in first-pass matching. It will then only be used in
14530   // the `Shorten_branches' pass.
14531   // ins_short_branch(1);
14532   format %{ "b$cmp  $lbl" %}
14533 
14534   ins_encode(aarch64_enc_br_con(cmp, lbl));
14535 
14536   ins_pipe(pipe_branch_cond);
14537 %}
14538 
14539 // Conditional Near Branch Unsigned
14540 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14541 %{
14542   // Same match rule as `branchConFar'.
14543   match(If cmp cr);
14544 
14545   effect(USE lbl);
14546 
14547   ins_cost(BRANCH_COST);
14548   // If set to 1 this indicates that the current instruction is a
14549   // short variant of a long branch. This avoids using this
14550   // instruction in first-pass matching. It will then only be used in
14551   // the `Shorten_branches' pass.
14552   // ins_short_branch(1);
14553   format %{ "b$cmp  $lbl\t# unsigned" %}
14554 
14555   ins_encode(aarch64_enc_br_conU(cmp, lbl));
14556 
14557   ins_pipe(pipe_branch_cond);
14558 %}
14559 
14560 // Make use of CBZ and CBNZ.  These instructions, as well as being
14561 // shorter than (cmp; branch), have the additional benefit of not
14562 // killing the flags.
14563 
14564 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14565   match(If cmp (CmpI op1 op2));
14566   effect(USE labl);
14567 
14568   ins_cost(BRANCH_COST);
14569   format %{ "cbw$cmp   $op1, $labl" %}
14570   ins_encode %{
14571     Label* L = $labl$$label;
14572     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14573     if (cond == Assembler::EQ)
14574       __ cbzw($op1$$Register, *L);
14575     else
14576       __ cbnzw($op1$$Register, *L);
14577   %}
14578   ins_pipe(pipe_cmp_branch);
14579 %}
14580 
14581 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
14582   match(If cmp (CmpL op1 op2));
14583   effect(USE labl);
14584 
14585   ins_cost(BRANCH_COST);
14586   format %{ "cb$cmp   $op1, $labl" %}
14587   ins_encode %{
14588     Label* L = $labl$$label;
14589     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14590     if (cond == Assembler::EQ)
14591       __ cbz($op1$$Register, *L);
14592     else
14593       __ cbnz($op1$$Register, *L);
14594   %}
14595   ins_pipe(pipe_cmp_branch);
14596 %}
14597 
14598 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
14599   match(If cmp (CmpP op1 op2));
14600   effect(USE labl);
14601 
14602   ins_cost(BRANCH_COST);
14603   format %{ "cb$cmp   $op1, $labl" %}
14604   ins_encode %{
14605     Label* L = $labl$$label;
14606     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14607     if (cond == Assembler::EQ)
14608       __ cbz($op1$$Register, *L);
14609     else
14610       __ cbnz($op1$$Register, *L);
14611   %}
14612   ins_pipe(pipe_cmp_branch);
14613 %}
14614 
14615 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
14616   match(If cmp (CmpN op1 op2));
14617   effect(USE labl);
14618 
14619   ins_cost(BRANCH_COST);
14620   format %{ "cbw$cmp   $op1, $labl" %}
14621   ins_encode %{
14622     Label* L = $labl$$label;
14623     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14624     if (cond == Assembler::EQ)
14625       __ cbzw($op1$$Register, *L);
14626     else
14627       __ cbnzw($op1$$Register, *L);
14628   %}
14629   ins_pipe(pipe_cmp_branch);
14630 %}
14631 
14632 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
14633   match(If cmp (CmpP (DecodeN oop) zero));
14634   effect(USE labl);
14635 
14636   ins_cost(BRANCH_COST);
14637   format %{ "cb$cmp   $oop, $labl" %}
14638   ins_encode %{
14639     Label* L = $labl$$label;
14640     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14641     if (cond == Assembler::EQ)
14642       __ cbzw($oop$$Register, *L);
14643     else
14644       __ cbnzw($oop$$Register, *L);
14645   %}
14646   ins_pipe(pipe_cmp_branch);
14647 %}
14648 
14649 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
14650   match(If cmp (CmpU op1 op2));
14651   effect(USE labl);
14652 
14653   ins_cost(BRANCH_COST);
14654   format %{ "cbw$cmp   $op1, $labl" %}
14655   ins_encode %{
14656     Label* L = $labl$$label;
14657     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14658     if (cond == Assembler::EQ || cond == Assembler::LS)
14659       __ cbzw($op1$$Register, *L);
14660     else
14661       __ cbnzw($op1$$Register, *L);
14662   %}
14663   ins_pipe(pipe_cmp_branch);
14664 %}
14665 
14666 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
14667   match(If cmp (CmpUL op1 op2));
14668   effect(USE labl);
14669 
14670   ins_cost(BRANCH_COST);
14671   format %{ "cb$cmp   $op1, $labl" %}
14672   ins_encode %{
14673     Label* L = $labl$$label;
14674     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14675     if (cond == Assembler::EQ || cond == Assembler::LS)
14676       __ cbz($op1$$Register, *L);
14677     else
14678       __ cbnz($op1$$Register, *L);
14679   %}
14680   ins_pipe(pipe_cmp_branch);
14681 %}
14682 
14683 // Test bit and Branch
14684 
14685 // Patterns for short (< 32KiB) variants
14686 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
14687   match(If cmp (CmpL op1 op2));
14688   effect(USE labl);
14689 
14690   ins_cost(BRANCH_COST);
14691   format %{ "cb$cmp   $op1, $labl # long" %}
14692   ins_encode %{
14693     Label* L = $labl$$label;
14694     Assembler::Condition cond =
14695       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
14696     __ tbr(cond, $op1$$Register, 63, *L);
14697   %}
14698   ins_pipe(pipe_cmp_branch);
14699   ins_short_branch(1);
14700 %}
14701 
14702 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
14703   match(If cmp (CmpI op1 op2));
14704   effect(USE labl);
14705 
14706   ins_cost(BRANCH_COST);
14707   format %{ "cb$cmp   $op1, $labl # int" %}
14708   ins_encode %{
14709     Label* L = $labl$$label;
14710     Assembler::Condition cond =
14711       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
14712     __ tbr(cond, $op1$$Register, 31, *L);
14713   %}
14714   ins_pipe(pipe_cmp_branch);
14715   ins_short_branch(1);
14716 %}
14717 
14718 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
14719   match(If cmp (CmpL (AndL op1 op2) op3));
14720   predicate(is_power_of_2(n->in(2)->in(1)->in(2)->get_long()));
14721   effect(USE labl);
14722 
14723   ins_cost(BRANCH_COST);
14724   format %{ "tb$cmp   $op1, $op2, $labl" %}
14725   ins_encode %{
14726     Label* L = $labl$$label;
14727     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14728     int bit = exact_log2($op2$$constant);
14729     __ tbr(cond, $op1$$Register, bit, *L);
14730   %}
14731   ins_pipe(pipe_cmp_branch);
14732   ins_short_branch(1);
14733 %}
14734 
14735 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
14736   match(If cmp (CmpI (AndI op1 op2) op3));
14737   predicate(is_power_of_2(n->in(2)->in(1)->in(2)->get_int()));
14738   effect(USE labl);
14739 
14740   ins_cost(BRANCH_COST);
14741   format %{ "tb$cmp   $op1, $op2, $labl" %}
14742   ins_encode %{
14743     Label* L = $labl$$label;
14744     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14745     int bit = exact_log2($op2$$constant);
14746     __ tbr(cond, $op1$$Register, bit, *L);
14747   %}
14748   ins_pipe(pipe_cmp_branch);
14749   ins_short_branch(1);
14750 %}
14751 
14752 // And far variants
14753 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
14754   match(If cmp (CmpL op1 op2));
14755   effect(USE labl);
14756 
14757   ins_cost(BRANCH_COST);
14758   format %{ "cb$cmp   $op1, $labl # long" %}
14759   ins_encode %{
14760     Label* L = $labl$$label;
14761     Assembler::Condition cond =
14762       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
14763     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
14764   %}
14765   ins_pipe(pipe_cmp_branch);
14766 %}
14767 
14768 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
14769   match(If cmp (CmpI op1 op2));
14770   effect(USE labl);
14771 
14772   ins_cost(BRANCH_COST);
14773   format %{ "cb$cmp   $op1, $labl # int" %}
14774   ins_encode %{
14775     Label* L = $labl$$label;
14776     Assembler::Condition cond =
14777       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
14778     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
14779   %}
14780   ins_pipe(pipe_cmp_branch);
14781 %}
14782 
14783 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
14784   match(If cmp (CmpL (AndL op1 op2) op3));
14785   predicate(is_power_of_2(n->in(2)->in(1)->in(2)->get_long()));
14786   effect(USE labl);
14787 
14788   ins_cost(BRANCH_COST);
14789   format %{ "tb$cmp   $op1, $op2, $labl" %}
14790   ins_encode %{
14791     Label* L = $labl$$label;
14792     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14793     int bit = exact_log2($op2$$constant);
14794     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
14795   %}
14796   ins_pipe(pipe_cmp_branch);
14797 %}
14798 
14799 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
14800   match(If cmp (CmpI (AndI op1 op2) op3));
14801   predicate(is_power_of_2(n->in(2)->in(1)->in(2)->get_int()));
14802   effect(USE labl);
14803 
14804   ins_cost(BRANCH_COST);
14805   format %{ "tb$cmp   $op1, $op2, $labl" %}
14806   ins_encode %{
14807     Label* L = $labl$$label;
14808     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14809     int bit = exact_log2($op2$$constant);
14810     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
14811   %}
14812   ins_pipe(pipe_cmp_branch);
14813 %}
14814 
14815 // Test bits
14816 
14817 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
14818   match(Set cr (CmpL (AndL op1 op2) op3));
14819   predicate(Assembler::operand_valid_for_logical_immediate
14820             (/*is_32*/false, n->in(1)->in(2)->get_long()));
14821 
14822   ins_cost(INSN_COST);
14823   format %{ "tst $op1, $op2 # long" %}
14824   ins_encode %{
14825     __ tst($op1$$Register, $op2$$constant);
14826   %}
14827   ins_pipe(ialu_reg_reg);
14828 %}
14829 
14830 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
14831   match(Set cr (CmpI (AndI op1 op2) op3));
14832   predicate(Assembler::operand_valid_for_logical_immediate
14833             (/*is_32*/true, n->in(1)->in(2)->get_int()));
14834 
14835   ins_cost(INSN_COST);
14836   format %{ "tst $op1, $op2 # int" %}
14837   ins_encode %{
14838     __ tstw($op1$$Register, $op2$$constant);
14839   %}
14840   ins_pipe(ialu_reg_reg);
14841 %}
14842 
14843 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
14844   match(Set cr (CmpL (AndL op1 op2) op3));
14845 
14846   ins_cost(INSN_COST);
14847   format %{ "tst $op1, $op2 # long" %}
14848   ins_encode %{
14849     __ tst($op1$$Register, $op2$$Register);
14850   %}
14851   ins_pipe(ialu_reg_reg);
14852 %}
14853 
14854 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
14855   match(Set cr (CmpI (AndI op1 op2) op3));
14856 
14857   ins_cost(INSN_COST);
14858   format %{ "tstw $op1, $op2 # int" %}
14859   ins_encode %{
14860     __ tstw($op1$$Register, $op2$$Register);
14861   %}
14862   ins_pipe(ialu_reg_reg);
14863 %}
14864 
14865 
14866 // Conditional Far Branch
14867 // Conditional Far Branch Unsigned
14868 // TODO: fixme
14869 
14870 // counted loop end branch near
14871 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
14872 %{
14873   match(CountedLoopEnd cmp cr);
14874 
14875   effect(USE lbl);
14876 
14877   ins_cost(BRANCH_COST);
14878   // short variant.
14879   // ins_short_branch(1);
14880   format %{ "b$cmp $lbl \t// counted loop end" %}
14881 
14882   ins_encode(aarch64_enc_br_con(cmp, lbl));
14883 
14884   ins_pipe(pipe_branch);
14885 %}
14886 
14887 // counted loop end branch near Unsigned
14888 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14889 %{
14890   match(CountedLoopEnd cmp cr);
14891 
14892   effect(USE lbl);
14893 
14894   ins_cost(BRANCH_COST);
14895   // short variant.
14896   // ins_short_branch(1);
14897   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
14898 
14899   ins_encode(aarch64_enc_br_conU(cmp, lbl));
14900 
14901   ins_pipe(pipe_branch);
14902 %}
14903 
14904 // counted loop end branch far
14905 // counted loop end branch far unsigned
14906 // TODO: fixme
14907 
14908 // ============================================================================
14909 // inlined locking and unlocking
14910 
14911 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
14912 %{
14913   match(Set cr (FastLock object box));
14914   effect(TEMP tmp, TEMP tmp2);
14915 
14916   // TODO
14917   // identify correct cost
14918   ins_cost(5 * INSN_COST);
14919   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
14920 
14921   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
14922 
14923   ins_pipe(pipe_serial);
14924 %}
14925 
14926 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
14927 %{
14928   match(Set cr (FastUnlock object box));
14929   effect(TEMP tmp, TEMP tmp2);
14930 
14931   ins_cost(5 * INSN_COST);
14932   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
14933 
14934   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
14935 
14936   ins_pipe(pipe_serial);
14937 %}
14938 
14939 
14940 // ============================================================================
14941 // Safepoint Instructions
14942 
14943 // TODO
14944 // provide a near and far version of this code
14945 
14946 instruct safePoint(rFlagsReg cr, iRegP poll)
14947 %{
14948   match(SafePoint poll);
14949   effect(KILL cr);
14950 
14951   format %{
14952     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
14953   %}
14954   ins_encode %{
14955     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
14956   %}
14957   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
14958 %}
14959 
14960 
14961 // ============================================================================
14962 // Procedure Call/Return Instructions
14963 
14964 // Call Java Static Instruction
14965 
14966 instruct CallStaticJavaDirect(method meth)
14967 %{
14968   match(CallStaticJava);
14969 
14970   effect(USE meth);
14971 
14972   ins_cost(CALL_COST);
14973 
14974   format %{ "call,static $meth \t// ==> " %}
14975 
14976   ins_encode( aarch64_enc_java_static_call(meth),
14977               aarch64_enc_call_epilog );
14978 
14979   ins_pipe(pipe_class_call);
14980 %}
14981 
14982 // TO HERE
14983 
14984 // Call Java Dynamic Instruction
14985 instruct CallDynamicJavaDirect(method meth)
14986 %{
14987   match(CallDynamicJava);
14988 
14989   effect(USE meth);
14990 
14991   ins_cost(CALL_COST);
14992 
14993   format %{ "CALL,dynamic $meth \t// ==> " %}
14994 
14995   ins_encode( aarch64_enc_java_dynamic_call(meth),
14996                aarch64_enc_call_epilog );
14997 
14998   ins_pipe(pipe_class_call);
14999 %}
15000 
15001 // Call Runtime Instruction
15002 
15003 instruct CallRuntimeDirect(method meth)
15004 %{
15005   match(CallRuntime);
15006 
15007   effect(USE meth);
15008 
15009   ins_cost(CALL_COST);
15010 
15011   format %{ "CALL, runtime $meth" %}
15012 
15013   ins_encode( aarch64_enc_java_to_runtime(meth) );
15014 
15015   ins_pipe(pipe_class_call);
15016 %}
15017 
15018 // Call Runtime Instruction
15019 
15020 instruct CallLeafDirect(method meth)
15021 %{
15022   match(CallLeaf);
15023 
15024   effect(USE meth);
15025 
15026   ins_cost(CALL_COST);
15027 
15028   format %{ "CALL, runtime leaf $meth" %}
15029 
15030   ins_encode( aarch64_enc_java_to_runtime(meth) );
15031 
15032   ins_pipe(pipe_class_call);
15033 %}
15034 
15035 // Call Runtime Instruction
15036 
15037 instruct CallLeafNoFPDirect(method meth)
15038 %{
15039   match(CallLeafNoFP);
15040 
15041   effect(USE meth);
15042 
15043   ins_cost(CALL_COST);
15044 
15045   format %{ "CALL, runtime leaf nofp $meth" %}
15046 
15047   ins_encode( aarch64_enc_java_to_runtime(meth) );
15048 
15049   ins_pipe(pipe_class_call);
15050 %}
15051 
15052 // Tail Call; Jump from runtime stub to Java code.
15053 // Also known as an 'interprocedural jump'.
15054 // Target of jump will eventually return to caller.
15055 // TailJump below removes the return address.
15056 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_oop)
15057 %{
15058   match(TailCall jump_target method_oop);
15059 
15060   ins_cost(CALL_COST);
15061 
15062   format %{ "br $jump_target\t# $method_oop holds method oop" %}
15063 
15064   ins_encode(aarch64_enc_tail_call(jump_target));
15065 
15066   ins_pipe(pipe_class_call);
15067 %}
15068 
15069 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
15070 %{
15071   match(TailJump jump_target ex_oop);
15072 
15073   ins_cost(CALL_COST);
15074 
15075   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15076 
15077   ins_encode(aarch64_enc_tail_jmp(jump_target));
15078 
15079   ins_pipe(pipe_class_call);
15080 %}
15081 
15082 // Create exception oop: created by stack-crawling runtime code.
15083 // Created exception is now available to this handler, and is setup
15084 // just prior to jumping to this handler. No code emitted.
15085 // TODO check
15086 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15087 instruct CreateException(iRegP_R0 ex_oop)
15088 %{
15089   match(Set ex_oop (CreateEx));
15090 
15091   format %{ " -- \t// exception oop; no code emitted" %}
15092 
15093   size(0);
15094 
15095   ins_encode( /*empty*/ );
15096 
15097   ins_pipe(pipe_class_empty);
15098 %}
15099 
15100 // Rethrow exception: The exception oop will come in the first
15101 // argument position. Then JUMP (not call) to the rethrow stub code.
15102 instruct RethrowException() %{
15103   match(Rethrow);
15104   ins_cost(CALL_COST);
15105 
15106   format %{ "b rethrow_stub" %}
15107 
15108   ins_encode( aarch64_enc_rethrow() );
15109 
15110   ins_pipe(pipe_class_call);
15111 %}
15112 
15113 
15114 // Return Instruction
15115 // epilog node loads ret address into lr as part of frame pop
15116 instruct Ret()
15117 %{
15118   match(Return);
15119 
15120   format %{ "ret\t// return register" %}
15121 
15122   ins_encode( aarch64_enc_ret() );
15123 
15124   ins_pipe(pipe_branch);
15125 %}
15126 
15127 // Die now.
15128 instruct ShouldNotReachHere() %{
15129   match(Halt);
15130 
15131   ins_cost(CALL_COST);
15132   format %{ "ShouldNotReachHere" %}
15133 
15134   ins_encode %{
15135     // +1 so NativeInstruction::is_sigill_zombie_not_entrant() doesn't
15136     // return true
15137     __ dpcs1(0xdead + 1);
15138   %}
15139 
15140   ins_pipe(pipe_class_default);
15141 %}
15142 
15143 // ============================================================================
15144 // Partial Subtype Check
15145 //
15146 // superklass array for an instance of the superklass.  Set a hidden
15147 // internal cache on a hit (cache is checked with exposed code in
15148 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
15149 // encoding ALSO sets flags.
15150 
15151 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15152 %{
15153   match(Set result (PartialSubtypeCheck sub super));
15154   effect(KILL cr, KILL temp);
15155 
15156   ins_cost(1100);  // slightly larger than the next version
15157   format %{ "partialSubtypeCheck $result, $sub, $super" %}
15158 
15159   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15160 
15161   opcode(0x1); // Force zero of result reg on hit
15162 
15163   ins_pipe(pipe_class_memory);
15164 %}
15165 
15166 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
15167 %{
15168   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
15169   effect(KILL temp, KILL result);
15170 
15171   ins_cost(1100);  // slightly larger than the next version
15172   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
15173 
15174   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15175 
15176   opcode(0x0); // Don't zero result reg on hit
15177 
15178   ins_pipe(pipe_class_memory);
15179 %}
15180 
15181 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15182                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15183 %{
15184   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
15185   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15186   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15187 
15188   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
15189   ins_encode %{
15190     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15191     __ string_compare($str1$$Register, $str2$$Register,
15192                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
15193                       $tmp1$$Register, $tmp2$$Register,
15194                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
15195   %}
15196   ins_pipe(pipe_class_memory);
15197 %}
15198 
15199 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15200                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15201 %{
15202   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
15203   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15204   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15205 
15206   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
15207   ins_encode %{
15208     __ string_compare($str1$$Register, $str2$$Register,
15209                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
15210                       $tmp1$$Register, $tmp2$$Register,
15211                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
15212   %}
15213   ins_pipe(pipe_class_memory);
15214 %}
15215 
15216 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15217                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15218                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15219 %{
15220   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
15221   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15222   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15223          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15224 
15225   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15226   ins_encode %{
15227     __ string_compare($str1$$Register, $str2$$Register,
15228                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
15229                       $tmp1$$Register, $tmp2$$Register,
15230                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15231                       $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
15232   %}
15233   ins_pipe(pipe_class_memory);
15234 %}
15235 
15236 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15237                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15238                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15239 %{
15240   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
15241   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15242   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15243          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15244 
15245   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15246   ins_encode %{
15247     __ string_compare($str1$$Register, $str2$$Register,
15248                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
15249                       $tmp1$$Register, $tmp2$$Register,
15250                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15251                       $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
15252   %}
15253   ins_pipe(pipe_class_memory);
15254 %}
15255 
15256 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15257        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15258        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
15259 %{
15260   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15261   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15262   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15263          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
15264   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
15265 
15266   ins_encode %{
15267     __ string_indexof($str1$$Register, $str2$$Register,
15268                       $cnt1$$Register, $cnt2$$Register,
15269                       $tmp1$$Register, $tmp2$$Register,
15270                       $tmp3$$Register, $tmp4$$Register,
15271                       $tmp5$$Register, $tmp6$$Register,
15272                       -1, $result$$Register, StrIntrinsicNode::UU);
15273   %}
15274   ins_pipe(pipe_class_memory);
15275 %}
15276 
15277 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15278        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15279        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
15280 %{
15281   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15282   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15283   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15284          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
15285   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
15286 
15287   ins_encode %{
15288     __ string_indexof($str1$$Register, $str2$$Register,
15289                       $cnt1$$Register, $cnt2$$Register,
15290                       $tmp1$$Register, $tmp2$$Register,
15291                       $tmp3$$Register, $tmp4$$Register,
15292                       $tmp5$$Register, $tmp6$$Register,
15293                       -1, $result$$Register, StrIntrinsicNode::LL);
15294   %}
15295   ins_pipe(pipe_class_memory);
15296 %}
15297 
15298 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15299        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15300        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
15301 %{
15302   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15303   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15304   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15305          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
15306   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
15307 
15308   ins_encode %{
15309     __ string_indexof($str1$$Register, $str2$$Register,
15310                       $cnt1$$Register, $cnt2$$Register,
15311                       $tmp1$$Register, $tmp2$$Register,
15312                       $tmp3$$Register, $tmp4$$Register,
15313                       $tmp5$$Register, $tmp6$$Register,
15314                       -1, $result$$Register, StrIntrinsicNode::UL);
15315   %}
15316   ins_pipe(pipe_class_memory);
15317 %}
15318 
15319 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15320                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15321                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15322 %{
15323   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15324   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15325   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15326          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15327   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
15328 
15329   ins_encode %{
15330     int icnt2 = (int)$int_cnt2$$constant;
15331     __ string_indexof($str1$$Register, $str2$$Register,
15332                       $cnt1$$Register, zr,
15333                       $tmp1$$Register, $tmp2$$Register,
15334                       $tmp3$$Register, $tmp4$$Register, zr, zr,
15335                       icnt2, $result$$Register, StrIntrinsicNode::UU);
15336   %}
15337   ins_pipe(pipe_class_memory);
15338 %}
15339 
15340 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15341                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15342                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15343 %{
15344   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15345   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15346   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15347          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15348   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
15349 
15350   ins_encode %{
15351     int icnt2 = (int)$int_cnt2$$constant;
15352     __ string_indexof($str1$$Register, $str2$$Register,
15353                       $cnt1$$Register, zr,
15354                       $tmp1$$Register, $tmp2$$Register,
15355                       $tmp3$$Register, $tmp4$$Register, zr, zr,
15356                       icnt2, $result$$Register, StrIntrinsicNode::LL);
15357   %}
15358   ins_pipe(pipe_class_memory);
15359 %}
15360 
15361 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15362                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15363                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15364 %{
15365   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15366   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15367   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15368          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15369   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
15370 
15371   ins_encode %{
15372     int icnt2 = (int)$int_cnt2$$constant;
15373     __ string_indexof($str1$$Register, $str2$$Register,
15374                       $cnt1$$Register, zr,
15375                       $tmp1$$Register, $tmp2$$Register,
15376                       $tmp3$$Register, $tmp4$$Register, zr, zr,
15377                       icnt2, $result$$Register, StrIntrinsicNode::UL);
15378   %}
15379   ins_pipe(pipe_class_memory);
15380 %}
15381 
15382 instruct string_indexofU_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15383                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15384                               iRegINoSp tmp3, rFlagsReg cr)
15385 %{
15386   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15387   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15388          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15389 
15390   format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15391 
15392   ins_encode %{
15393     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15394                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
15395                            $tmp3$$Register);
15396   %}
15397   ins_pipe(pipe_class_memory);
15398 %}
15399 
15400 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
15401                         iRegI_R0 result, rFlagsReg cr)
15402 %{
15403   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
15404   match(Set result (StrEquals (Binary str1 str2) cnt));
15405   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
15406 
15407   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
15408   ins_encode %{
15409     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15410     __ string_equals($str1$$Register, $str2$$Register,
15411                      $result$$Register, $cnt$$Register, 1);
15412   %}
15413   ins_pipe(pipe_class_memory);
15414 %}
15415 
15416 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
15417                         iRegI_R0 result, rFlagsReg cr)
15418 %{
15419   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
15420   match(Set result (StrEquals (Binary str1 str2) cnt));
15421   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
15422 
15423   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
15424   ins_encode %{
15425     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15426     __ string_equals($str1$$Register, $str2$$Register,
15427                      $result$$Register, $cnt$$Register, 2);
15428   %}
15429   ins_pipe(pipe_class_memory);
15430 %}
15431 
15432 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
15433                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
15434                        iRegP_R10 tmp, rFlagsReg cr)
15435 %{
15436   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
15437   match(Set result (AryEq ary1 ary2));
15438   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15439 
15440   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
15441   ins_encode %{
15442     __ arrays_equals($ary1$$Register, $ary2$$Register,
15443                      $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
15444                      $result$$Register, $tmp$$Register, 1);
15445     %}
15446   ins_pipe(pipe_class_memory);
15447 %}
15448 
15449 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
15450                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
15451                        iRegP_R10 tmp, rFlagsReg cr)
15452 %{
15453   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
15454   match(Set result (AryEq ary1 ary2));
15455   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15456 
15457   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
15458   ins_encode %{
15459     __ arrays_equals($ary1$$Register, $ary2$$Register,
15460                      $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
15461                      $result$$Register, $tmp$$Register, 2);
15462   %}
15463   ins_pipe(pipe_class_memory);
15464 %}
15465 
15466 instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
15467 %{
15468   match(Set result (HasNegatives ary1 len));
15469   effect(USE_KILL ary1, USE_KILL len, KILL cr);
15470   format %{ "has negatives byte[] $ary1,$len -> $result" %}
15471   ins_encode %{
15472     __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
15473   %}
15474   ins_pipe( pipe_slow );
15475 %}
15476 
15477 // fast char[] to byte[] compression
15478 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
15479                          vRegD_V0 tmp1, vRegD_V1 tmp2,
15480                          vRegD_V2 tmp3, vRegD_V3 tmp4,
15481                          iRegI_R0 result, rFlagsReg cr)
15482 %{
15483   match(Set result (StrCompressedCopy src (Binary dst len)));
15484   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
15485 
15486   format %{ "String Compress $src,$dst -> $result    // KILL R1, R2, R3, R4" %}
15487   ins_encode %{
15488     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
15489                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
15490                            $tmp3$$FloatRegister, $tmp4$$FloatRegister,
15491                            $result$$Register);
15492   %}
15493   ins_pipe( pipe_slow );
15494 %}
15495 
15496 // fast byte[] to char[] inflation
15497 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
15498                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
15499 %{
15500   match(Set dummy (StrInflatedCopy src (Binary dst len)));
15501   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
15502 
15503   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
15504   ins_encode %{
15505     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
15506                           $tmp1$$FloatRegister, $tmp2$$FloatRegister, $tmp3$$FloatRegister, $tmp4$$Register);
15507   %}
15508   ins_pipe(pipe_class_memory);
15509 %}
15510 
15511 // encode char[] to byte[] in ISO_8859_1
15512 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
15513                           vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
15514                           vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
15515                           iRegI_R0 result, rFlagsReg cr)
15516 %{
15517   match(Set result (EncodeISOArray src (Binary dst len)));
15518   effect(USE_KILL src, USE_KILL dst, USE_KILL len,
15519          KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
15520 
15521   format %{ "Encode array $src,$dst,$len -> $result" %}
15522   ins_encode %{
15523     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
15524          $result$$Register, $Vtmp1$$FloatRegister,  $Vtmp2$$FloatRegister,
15525          $Vtmp3$$FloatRegister,  $Vtmp4$$FloatRegister);
15526   %}
15527   ins_pipe( pipe_class_memory );
15528 %}
15529 
15530 // ============================================================================
15531 // This name is KNOWN by the ADLC and cannot be changed.
15532 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
15533 // for this guy.
15534 instruct tlsLoadP(thread_RegP dst)
15535 %{
15536   match(Set dst (ThreadLocal));
15537 
15538   ins_cost(0);
15539 
15540   format %{ " -- \t// $dst=Thread::current(), empty" %}
15541 
15542   size(0);
15543 
15544   ins_encode( /*empty*/ );
15545 
15546   ins_pipe(pipe_class_empty);
15547 %}
15548 
15549 // ====================VECTOR INSTRUCTIONS=====================================
15550 
15551 // Load vector (32 bits)
15552 instruct loadV4(vecD dst, vmem4 mem)
15553 %{
15554   predicate(n->as_LoadVector()->memory_size() == 4);
15555   match(Set dst (LoadVector mem));
15556   ins_cost(4 * INSN_COST);
15557   format %{ "ldrs   $dst,$mem\t# vector (32 bits)" %}
15558   ins_encode( aarch64_enc_ldrvS(dst, mem) );
15559   ins_pipe(vload_reg_mem64);
15560 %}
15561 
15562 // Load vector (64 bits)
15563 instruct loadV8(vecD dst, vmem8 mem)
15564 %{
15565   predicate(n->as_LoadVector()->memory_size() == 8);
15566   match(Set dst (LoadVector mem));
15567   ins_cost(4 * INSN_COST);
15568   format %{ "ldrd   $dst,$mem\t# vector (64 bits)" %}
15569   ins_encode( aarch64_enc_ldrvD(dst, mem) );
15570   ins_pipe(vload_reg_mem64);
15571 %}
15572 
15573 // Load Vector (128 bits)
15574 instruct loadV16(vecX dst, vmem16 mem)
15575 %{
15576   predicate(n->as_LoadVector()->memory_size() == 16);
15577   match(Set dst (LoadVector mem));
15578   ins_cost(4 * INSN_COST);
15579   format %{ "ldrq   $dst,$mem\t# vector (128 bits)" %}
15580   ins_encode( aarch64_enc_ldrvQ(dst, mem) );
15581   ins_pipe(vload_reg_mem128);
15582 %}
15583 
15584 // Store Vector (32 bits)
15585 instruct storeV4(vecD src, vmem4 mem)
15586 %{
15587   predicate(n->as_StoreVector()->memory_size() == 4);
15588   match(Set mem (StoreVector mem src));
15589   ins_cost(4 * INSN_COST);
15590   format %{ "strs   $mem,$src\t# vector (32 bits)" %}
15591   ins_encode( aarch64_enc_strvS(src, mem) );
15592   ins_pipe(vstore_reg_mem64);
15593 %}
15594 
15595 // Store Vector (64 bits)
15596 instruct storeV8(vecD src, vmem8 mem)
15597 %{
15598   predicate(n->as_StoreVector()->memory_size() == 8);
15599   match(Set mem (StoreVector mem src));
15600   ins_cost(4 * INSN_COST);
15601   format %{ "strd   $mem,$src\t# vector (64 bits)" %}
15602   ins_encode( aarch64_enc_strvD(src, mem) );
15603   ins_pipe(vstore_reg_mem64);
15604 %}
15605 
15606 // Store Vector (128 bits)
15607 instruct storeV16(vecX src, vmem16 mem)
15608 %{
15609   predicate(n->as_StoreVector()->memory_size() == 16);
15610   match(Set mem (StoreVector mem src));
15611   ins_cost(4 * INSN_COST);
15612   format %{ "strq   $mem,$src\t# vector (128 bits)" %}
15613   ins_encode( aarch64_enc_strvQ(src, mem) );
15614   ins_pipe(vstore_reg_mem128);
15615 %}
15616 
15617 instruct replicate8B(vecD dst, iRegIorL2I src)
15618 %{
15619   predicate(n->as_Vector()->length() == 4 ||
15620             n->as_Vector()->length() == 8);
15621   match(Set dst (ReplicateB src));
15622   ins_cost(INSN_COST);
15623   format %{ "dup  $dst, $src\t# vector (8B)" %}
15624   ins_encode %{
15625     __ dup(as_FloatRegister($dst$$reg), __ T8B, as_Register($src$$reg));
15626   %}
15627   ins_pipe(vdup_reg_reg64);
15628 %}
15629 
15630 instruct replicate16B(vecX dst, iRegIorL2I src)
15631 %{
15632   predicate(n->as_Vector()->length() == 16);
15633   match(Set dst (ReplicateB src));
15634   ins_cost(INSN_COST);
15635   format %{ "dup  $dst, $src\t# vector (16B)" %}
15636   ins_encode %{
15637     __ dup(as_FloatRegister($dst$$reg), __ T16B, as_Register($src$$reg));
15638   %}
15639   ins_pipe(vdup_reg_reg128);
15640 %}
15641 
15642 instruct replicate8B_imm(vecD dst, immI con)
15643 %{
15644   predicate(n->as_Vector()->length() == 4 ||
15645             n->as_Vector()->length() == 8);
15646   match(Set dst (ReplicateB con));
15647   ins_cost(INSN_COST);
15648   format %{ "movi  $dst, $con\t# vector(8B)" %}
15649   ins_encode %{
15650     __ mov(as_FloatRegister($dst$$reg), __ T8B, $con$$constant & 0xff);
15651   %}
15652   ins_pipe(vmovi_reg_imm64);
15653 %}
15654 
15655 instruct replicate16B_imm(vecX dst, immI con)
15656 %{
15657   predicate(n->as_Vector()->length() == 16);
15658   match(Set dst (ReplicateB con));
15659   ins_cost(INSN_COST);
15660   format %{ "movi  $dst, $con\t# vector(16B)" %}
15661   ins_encode %{
15662     __ mov(as_FloatRegister($dst$$reg), __ T16B, $con$$constant & 0xff);
15663   %}
15664   ins_pipe(vmovi_reg_imm128);
15665 %}
15666 
15667 instruct replicate4S(vecD dst, iRegIorL2I src)
15668 %{
15669   predicate(n->as_Vector()->length() == 2 ||
15670             n->as_Vector()->length() == 4);
15671   match(Set dst (ReplicateS src));
15672   ins_cost(INSN_COST);
15673   format %{ "dup  $dst, $src\t# vector (4S)" %}
15674   ins_encode %{
15675     __ dup(as_FloatRegister($dst$$reg), __ T4H, as_Register($src$$reg));
15676   %}
15677   ins_pipe(vdup_reg_reg64);
15678 %}
15679 
15680 instruct replicate8S(vecX dst, iRegIorL2I src)
15681 %{
15682   predicate(n->as_Vector()->length() == 8);
15683   match(Set dst (ReplicateS src));
15684   ins_cost(INSN_COST);
15685   format %{ "dup  $dst, $src\t# vector (8S)" %}
15686   ins_encode %{
15687     __ dup(as_FloatRegister($dst$$reg), __ T8H, as_Register($src$$reg));
15688   %}
15689   ins_pipe(vdup_reg_reg128);
15690 %}
15691 
15692 instruct replicate4S_imm(vecD dst, immI con)
15693 %{
15694   predicate(n->as_Vector()->length() == 2 ||
15695             n->as_Vector()->length() == 4);
15696   match(Set dst (ReplicateS con));
15697   ins_cost(INSN_COST);
15698   format %{ "movi  $dst, $con\t# vector(4H)" %}
15699   ins_encode %{
15700     __ mov(as_FloatRegister($dst$$reg), __ T4H, $con$$constant & 0xffff);
15701   %}
15702   ins_pipe(vmovi_reg_imm64);
15703 %}
15704 
15705 instruct replicate8S_imm(vecX dst, immI con)
15706 %{
15707   predicate(n->as_Vector()->length() == 8);
15708   match(Set dst (ReplicateS con));
15709   ins_cost(INSN_COST);
15710   format %{ "movi  $dst, $con\t# vector(8H)" %}
15711   ins_encode %{
15712     __ mov(as_FloatRegister($dst$$reg), __ T8H, $con$$constant & 0xffff);
15713   %}
15714   ins_pipe(vmovi_reg_imm128);
15715 %}
15716 
15717 instruct replicate2I(vecD dst, iRegIorL2I src)
15718 %{
15719   predicate(n->as_Vector()->length() == 2);
15720   match(Set dst (ReplicateI src));
15721   ins_cost(INSN_COST);
15722   format %{ "dup  $dst, $src\t# vector (2I)" %}
15723   ins_encode %{
15724     __ dup(as_FloatRegister($dst$$reg), __ T2S, as_Register($src$$reg));
15725   %}
15726   ins_pipe(vdup_reg_reg64);
15727 %}
15728 
15729 instruct replicate4I(vecX dst, iRegIorL2I src)
15730 %{
15731   predicate(n->as_Vector()->length() == 4);
15732   match(Set dst (ReplicateI src));
15733   ins_cost(INSN_COST);
15734   format %{ "dup  $dst, $src\t# vector (4I)" %}
15735   ins_encode %{
15736     __ dup(as_FloatRegister($dst$$reg), __ T4S, as_Register($src$$reg));
15737   %}
15738   ins_pipe(vdup_reg_reg128);
15739 %}
15740 
15741 instruct replicate2I_imm(vecD dst, immI con)
15742 %{
15743   predicate(n->as_Vector()->length() == 2);
15744   match(Set dst (ReplicateI con));
15745   ins_cost(INSN_COST);
15746   format %{ "movi  $dst, $con\t# vector(2I)" %}
15747   ins_encode %{
15748     __ mov(as_FloatRegister($dst$$reg), __ T2S, $con$$constant);
15749   %}
15750   ins_pipe(vmovi_reg_imm64);
15751 %}
15752 
15753 instruct replicate4I_imm(vecX dst, immI con)
15754 %{
15755   predicate(n->as_Vector()->length() == 4);
15756   match(Set dst (ReplicateI con));
15757   ins_cost(INSN_COST);
15758   format %{ "movi  $dst, $con\t# vector(4I)" %}
15759   ins_encode %{
15760     __ mov(as_FloatRegister($dst$$reg), __ T4S, $con$$constant);
15761   %}
15762   ins_pipe(vmovi_reg_imm128);
15763 %}
15764 
15765 instruct replicate2L(vecX dst, iRegL src)
15766 %{
15767   predicate(n->as_Vector()->length() == 2);
15768   match(Set dst (ReplicateL src));
15769   ins_cost(INSN_COST);
15770   format %{ "dup  $dst, $src\t# vector (2L)" %}
15771   ins_encode %{
15772     __ dup(as_FloatRegister($dst$$reg), __ T2D, as_Register($src$$reg));
15773   %}
15774   ins_pipe(vdup_reg_reg128);
15775 %}
15776 
15777 instruct replicate2L_zero(vecX dst, immI0 zero)
15778 %{
15779   predicate(n->as_Vector()->length() == 2);
15780   match(Set dst (ReplicateI zero));
15781   ins_cost(INSN_COST);
15782   format %{ "movi  $dst, $zero\t# vector(4I)" %}
15783   ins_encode %{
15784     __ eor(as_FloatRegister($dst$$reg), __ T16B,
15785            as_FloatRegister($dst$$reg),
15786            as_FloatRegister($dst$$reg));
15787   %}
15788   ins_pipe(vmovi_reg_imm128);
15789 %}
15790 
15791 instruct replicate2F(vecD dst, vRegF src)
15792 %{
15793   predicate(n->as_Vector()->length() == 2);
15794   match(Set dst (ReplicateF src));
15795   ins_cost(INSN_COST);
15796   format %{ "dup  $dst, $src\t# vector (2F)" %}
15797   ins_encode %{
15798     __ dup(as_FloatRegister($dst$$reg), __ T2S,
15799            as_FloatRegister($src$$reg));
15800   %}
15801   ins_pipe(vdup_reg_freg64);
15802 %}
15803 
15804 instruct replicate4F(vecX dst, vRegF src)
15805 %{
15806   predicate(n->as_Vector()->length() == 4);
15807   match(Set dst (ReplicateF src));
15808   ins_cost(INSN_COST);
15809   format %{ "dup  $dst, $src\t# vector (4F)" %}
15810   ins_encode %{
15811     __ dup(as_FloatRegister($dst$$reg), __ T4S,
15812            as_FloatRegister($src$$reg));
15813   %}
15814   ins_pipe(vdup_reg_freg128);
15815 %}
15816 
15817 instruct replicate2D(vecX dst, vRegD src)
15818 %{
15819   predicate(n->as_Vector()->length() == 2);
15820   match(Set dst (ReplicateD src));
15821   ins_cost(INSN_COST);
15822   format %{ "dup  $dst, $src\t# vector (2D)" %}
15823   ins_encode %{
15824     __ dup(as_FloatRegister($dst$$reg), __ T2D,
15825            as_FloatRegister($src$$reg));
15826   %}
15827   ins_pipe(vdup_reg_dreg128);
15828 %}
15829 
15830 // ====================REDUCTION ARITHMETIC====================================
15831 
15832 instruct reduce_add2I(iRegINoSp dst, iRegIorL2I src1, vecD src2, iRegINoSp tmp, iRegINoSp tmp2)
15833 %{
15834   match(Set dst (AddReductionVI src1 src2));
15835   ins_cost(INSN_COST);
15836   effect(TEMP tmp, TEMP tmp2);
15837   format %{ "umov  $tmp, $src2, S, 0\n\t"
15838             "umov  $tmp2, $src2, S, 1\n\t"
15839             "addw  $dst, $src1, $tmp\n\t"
15840             "addw  $dst, $dst, $tmp2\t add reduction2i"
15841   %}
15842   ins_encode %{
15843     __ umov($tmp$$Register, as_FloatRegister($src2$$reg), __ S, 0);
15844     __ umov($tmp2$$Register, as_FloatRegister($src2$$reg), __ S, 1);
15845     __ addw($dst$$Register, $src1$$Register, $tmp$$Register);
15846     __ addw($dst$$Register, $dst$$Register, $tmp2$$Register);
15847   %}
15848   ins_pipe(pipe_class_default);
15849 %}
15850 
15851 instruct reduce_add4I(iRegINoSp dst, iRegIorL2I src1, vecX src2, vecX tmp, iRegINoSp tmp2)
15852 %{
15853   match(Set dst (AddReductionVI src1 src2));
15854   ins_cost(INSN_COST);
15855   effect(TEMP tmp, TEMP tmp2);
15856   format %{ "addv  $tmp, T4S, $src2\n\t"
15857             "umov  $tmp2, $tmp, S, 0\n\t"
15858             "addw  $dst, $tmp2, $src1\t add reduction4i"
15859   %}
15860   ins_encode %{
15861     __ addv(as_FloatRegister($tmp$$reg), __ T4S,
15862             as_FloatRegister($src2$$reg));
15863     __ umov($tmp2$$Register, as_FloatRegister($tmp$$reg), __ S, 0);
15864     __ addw($dst$$Register, $tmp2$$Register, $src1$$Register);
15865   %}
15866   ins_pipe(pipe_class_default);
15867 %}
15868 
15869 instruct reduce_mul2I(iRegINoSp dst, iRegIorL2I src1, vecD src2, iRegINoSp tmp)
15870 %{
15871   match(Set dst (MulReductionVI src1 src2));
15872   ins_cost(INSN_COST);
15873   effect(TEMP tmp, TEMP dst);
15874   format %{ "umov  $tmp, $src2, S, 0\n\t"
15875             "mul   $dst, $tmp, $src1\n\t"
15876             "umov  $tmp, $src2, S, 1\n\t"
15877             "mul   $dst, $tmp, $dst\t mul reduction2i\n\t"
15878   %}
15879   ins_encode %{
15880     __ umov($tmp$$Register, as_FloatRegister($src2$$reg), __ S, 0);
15881     __ mul($dst$$Register, $tmp$$Register, $src1$$Register);
15882     __ umov($tmp$$Register, as_FloatRegister($src2$$reg), __ S, 1);
15883     __ mul($dst$$Register, $tmp$$Register, $dst$$Register);
15884   %}
15885   ins_pipe(pipe_class_default);
15886 %}
15887 
15888 instruct reduce_mul4I(iRegINoSp dst, iRegIorL2I src1, vecX src2, vecX tmp, iRegINoSp tmp2)
15889 %{
15890   match(Set dst (MulReductionVI src1 src2));
15891   ins_cost(INSN_COST);
15892   effect(TEMP tmp, TEMP tmp2, TEMP dst);
15893   format %{ "ins   $tmp, $src2, 0, 1\n\t"
15894             "mul   $tmp, $tmp, $src2\n\t"
15895             "umov  $tmp2, $tmp, S, 0\n\t"
15896             "mul   $dst, $tmp2, $src1\n\t"
15897             "umov  $tmp2, $tmp, S, 1\n\t"
15898             "mul   $dst, $tmp2, $dst\t mul reduction4i\n\t"
15899   %}
15900   ins_encode %{
15901     __ ins(as_FloatRegister($tmp$$reg), __ D,
15902            as_FloatRegister($src2$$reg), 0, 1);
15903     __ mulv(as_FloatRegister($tmp$$reg), __ T2S,
15904            as_FloatRegister($tmp$$reg), as_FloatRegister($src2$$reg));
15905     __ umov($tmp2$$Register, as_FloatRegister($tmp$$reg), __ S, 0);
15906     __ mul($dst$$Register, $tmp2$$Register, $src1$$Register);
15907     __ umov($tmp2$$Register, as_FloatRegister($tmp$$reg), __ S, 1);
15908     __ mul($dst$$Register, $tmp2$$Register, $dst$$Register);
15909   %}
15910   ins_pipe(pipe_class_default);
15911 %}
15912 
15913 instruct reduce_add2F(vRegF dst, vRegF src1, vecD src2, vecD tmp)
15914 %{
15915   match(Set dst (AddReductionVF src1 src2));
15916   ins_cost(INSN_COST);
15917   effect(TEMP tmp, TEMP dst);
15918   format %{ "fadds $dst, $src1, $src2\n\t"
15919             "ins   $tmp, S, $src2, 0, 1\n\t"
15920             "fadds $dst, $dst, $tmp\t add reduction2f"
15921   %}
15922   ins_encode %{
15923     __ fadds(as_FloatRegister($dst$$reg),
15924              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15925     __ ins(as_FloatRegister($tmp$$reg), __ S,
15926            as_FloatRegister($src2$$reg), 0, 1);
15927     __ fadds(as_FloatRegister($dst$$reg),
15928              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15929   %}
15930   ins_pipe(pipe_class_default);
15931 %}
15932 
15933 instruct reduce_add4F(vRegF dst, vRegF src1, vecX src2, vecX tmp)
15934 %{
15935   match(Set dst (AddReductionVF src1 src2));
15936   ins_cost(INSN_COST);
15937   effect(TEMP tmp, TEMP dst);
15938   format %{ "fadds $dst, $src1, $src2\n\t"
15939             "ins   $tmp, S, $src2, 0, 1\n\t"
15940             "fadds $dst, $dst, $tmp\n\t"
15941             "ins   $tmp, S, $src2, 0, 2\n\t"
15942             "fadds $dst, $dst, $tmp\n\t"
15943             "ins   $tmp, S, $src2, 0, 3\n\t"
15944             "fadds $dst, $dst, $tmp\t add reduction4f"
15945   %}
15946   ins_encode %{
15947     __ fadds(as_FloatRegister($dst$$reg),
15948              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15949     __ ins(as_FloatRegister($tmp$$reg), __ S,
15950            as_FloatRegister($src2$$reg), 0, 1);
15951     __ fadds(as_FloatRegister($dst$$reg),
15952              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15953     __ ins(as_FloatRegister($tmp$$reg), __ S,
15954            as_FloatRegister($src2$$reg), 0, 2);
15955     __ fadds(as_FloatRegister($dst$$reg),
15956              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15957     __ ins(as_FloatRegister($tmp$$reg), __ S,
15958            as_FloatRegister($src2$$reg), 0, 3);
15959     __ fadds(as_FloatRegister($dst$$reg),
15960              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15961   %}
15962   ins_pipe(pipe_class_default);
15963 %}
15964 
15965 instruct reduce_mul2F(vRegF dst, vRegF src1, vecD src2, vecD tmp)
15966 %{
15967   match(Set dst (MulReductionVF src1 src2));
15968   ins_cost(INSN_COST);
15969   effect(TEMP tmp, TEMP dst);
15970   format %{ "fmuls $dst, $src1, $src2\n\t"
15971             "ins   $tmp, S, $src2, 0, 1\n\t"
15972             "fmuls $dst, $dst, $tmp\t add reduction4f"
15973   %}
15974   ins_encode %{
15975     __ fmuls(as_FloatRegister($dst$$reg),
15976              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15977     __ ins(as_FloatRegister($tmp$$reg), __ S,
15978            as_FloatRegister($src2$$reg), 0, 1);
15979     __ fmuls(as_FloatRegister($dst$$reg),
15980              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15981   %}
15982   ins_pipe(pipe_class_default);
15983 %}
15984 
15985 instruct reduce_mul4F(vRegF dst, vRegF src1, vecX src2, vecX tmp)
15986 %{
15987   match(Set dst (MulReductionVF src1 src2));
15988   ins_cost(INSN_COST);
15989   effect(TEMP tmp, TEMP dst);
15990   format %{ "fmuls $dst, $src1, $src2\n\t"
15991             "ins   $tmp, S, $src2, 0, 1\n\t"
15992             "fmuls $dst, $dst, $tmp\n\t"
15993             "ins   $tmp, S, $src2, 0, 2\n\t"
15994             "fmuls $dst, $dst, $tmp\n\t"
15995             "ins   $tmp, S, $src2, 0, 3\n\t"
15996             "fmuls $dst, $dst, $tmp\t add reduction4f"
15997   %}
15998   ins_encode %{
15999     __ fmuls(as_FloatRegister($dst$$reg),
16000              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16001     __ ins(as_FloatRegister($tmp$$reg), __ S,
16002            as_FloatRegister($src2$$reg), 0, 1);
16003     __ fmuls(as_FloatRegister($dst$$reg),
16004              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16005     __ ins(as_FloatRegister($tmp$$reg), __ S,
16006            as_FloatRegister($src2$$reg), 0, 2);
16007     __ fmuls(as_FloatRegister($dst$$reg),
16008              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16009     __ ins(as_FloatRegister($tmp$$reg), __ S,
16010            as_FloatRegister($src2$$reg), 0, 3);
16011     __ fmuls(as_FloatRegister($dst$$reg),
16012              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16013   %}
16014   ins_pipe(pipe_class_default);
16015 %}
16016 
16017 instruct reduce_add2D(vRegD dst, vRegD src1, vecX src2, vecX tmp)
16018 %{
16019   match(Set dst (AddReductionVD src1 src2));
16020   ins_cost(INSN_COST);
16021   effect(TEMP tmp, TEMP dst);
16022   format %{ "faddd $dst, $src1, $src2\n\t"
16023             "ins   $tmp, D, $src2, 0, 1\n\t"
16024             "faddd $dst, $dst, $tmp\t add reduction2d"
16025   %}
16026   ins_encode %{
16027     __ faddd(as_FloatRegister($dst$$reg),
16028              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16029     __ ins(as_FloatRegister($tmp$$reg), __ D,
16030            as_FloatRegister($src2$$reg), 0, 1);
16031     __ faddd(as_FloatRegister($dst$$reg),
16032              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16033   %}
16034   ins_pipe(pipe_class_default);
16035 %}
16036 
16037 instruct reduce_mul2D(vRegD dst, vRegD src1, vecX src2, vecX tmp)
16038 %{
16039   match(Set dst (MulReductionVD src1 src2));
16040   ins_cost(INSN_COST);
16041   effect(TEMP tmp, TEMP dst);
16042   format %{ "fmuld $dst, $src1, $src2\n\t"
16043             "ins   $tmp, D, $src2, 0, 1\n\t"
16044             "fmuld $dst, $dst, $tmp\t add reduction2d"
16045   %}
16046   ins_encode %{
16047     __ fmuld(as_FloatRegister($dst$$reg),
16048              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16049     __ ins(as_FloatRegister($tmp$$reg), __ D,
16050            as_FloatRegister($src2$$reg), 0, 1);
16051     __ fmuld(as_FloatRegister($dst$$reg),
16052              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16053   %}
16054   ins_pipe(pipe_class_default);
16055 %}
16056 
16057 instruct reduce_max2F(vRegF dst, vRegF src1, vecD src2, vecD tmp) %{
16058   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
16059   match(Set dst (MaxReductionV src1 src2));
16060   ins_cost(INSN_COST);
16061   effect(TEMP_DEF dst, TEMP tmp);
16062   format %{ "fmaxs $dst, $src1, $src2\n\t"
16063             "ins   $tmp, S, $src2, 0, 1\n\t"
16064             "fmaxs $dst, $dst, $tmp\t max reduction2F" %}
16065   ins_encode %{
16066     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16067     __ ins(as_FloatRegister($tmp$$reg), __ S, as_FloatRegister($src2$$reg), 0, 1);
16068     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16069   %}
16070   ins_pipe(pipe_class_default);
16071 %}
16072 
16073 instruct reduce_max4F(vRegF dst, vRegF src1, vecX src2) %{
16074   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
16075   match(Set dst (MaxReductionV src1 src2));
16076   ins_cost(INSN_COST);
16077   effect(TEMP_DEF dst);
16078   format %{ "fmaxv $dst, T4S, $src2\n\t"
16079             "fmaxs $dst, $dst, $src1\t max reduction4F" %}
16080   ins_encode %{
16081     __ fmaxv(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($src2$$reg));
16082     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
16083   %}
16084   ins_pipe(pipe_class_default);
16085 %}
16086 
16087 instruct reduce_max2D(vRegD dst, vRegD src1, vecX src2, vecX tmp) %{
16088   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
16089   match(Set dst (MaxReductionV src1 src2));
16090   ins_cost(INSN_COST);
16091   effect(TEMP_DEF dst, TEMP tmp);
16092   format %{ "fmaxd $dst, $src1, $src2\n\t"
16093             "ins   $tmp, D, $src2, 0, 1\n\t"
16094             "fmaxd $dst, $dst, $tmp\t max reduction2D" %}
16095   ins_encode %{
16096     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16097     __ ins(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($src2$$reg), 0, 1);
16098     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16099   %}
16100   ins_pipe(pipe_class_default);
16101 %}
16102 
16103 instruct reduce_min2F(vRegF dst, vRegF src1, vecD src2, vecD tmp) %{
16104   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
16105   match(Set dst (MinReductionV src1 src2));
16106   ins_cost(INSN_COST);
16107   effect(TEMP_DEF dst, TEMP tmp);
16108   format %{ "fmins $dst, $src1, $src2\n\t"
16109             "ins   $tmp, S, $src2, 0, 1\n\t"
16110             "fmins $dst, $dst, $tmp\t min reduction2F" %}
16111   ins_encode %{
16112     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16113     __ ins(as_FloatRegister($tmp$$reg), __ S, as_FloatRegister($src2$$reg), 0, 1);
16114     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16115   %}
16116   ins_pipe(pipe_class_default);
16117 %}
16118 
16119 instruct reduce_min4F(vRegF dst, vRegF src1, vecX src2) %{
16120   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
16121   match(Set dst (MinReductionV src1 src2));
16122   ins_cost(INSN_COST);
16123   effect(TEMP_DEF dst);
16124   format %{ "fminv $dst, T4S, $src2\n\t"
16125             "fmins $dst, $dst, $src1\t min reduction4F" %}
16126   ins_encode %{
16127     __ fminv(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($src2$$reg));
16128     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
16129   %}
16130   ins_pipe(pipe_class_default);
16131 %}
16132 
16133 instruct reduce_min2D(vRegD dst, vRegD src1, vecX src2, vecX tmp) %{
16134   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
16135   match(Set dst (MinReductionV src1 src2));
16136   ins_cost(INSN_COST);
16137   effect(TEMP_DEF dst, TEMP tmp);
16138   format %{ "fmind $dst, $src1, $src2\n\t"
16139             "ins   $tmp, D, $src2, 0, 1\n\t"
16140             "fmind $dst, $dst, $tmp\t min reduction2D" %}
16141   ins_encode %{
16142     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16143     __ ins(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($src2$$reg), 0, 1);
16144     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16145   %}
16146   ins_pipe(pipe_class_default);
16147 %}
16148 
16149 // ====================VECTOR ARITHMETIC=======================================
16150 
16151 // --------------------------------- ADD --------------------------------------
16152 
16153 instruct vadd8B(vecD dst, vecD src1, vecD src2)
16154 %{
16155   predicate(n->as_Vector()->length() == 4 ||
16156             n->as_Vector()->length() == 8);
16157   match(Set dst (AddVB src1 src2));
16158   ins_cost(INSN_COST);
16159   format %{ "addv  $dst,$src1,$src2\t# vector (8B)" %}
16160   ins_encode %{
16161     __ addv(as_FloatRegister($dst$$reg), __ T8B,
16162             as_FloatRegister($src1$$reg),
16163             as_FloatRegister($src2$$reg));
16164   %}
16165   ins_pipe(vdop64);
16166 %}
16167 
16168 instruct vadd16B(vecX dst, vecX src1, vecX src2)
16169 %{
16170   predicate(n->as_Vector()->length() == 16);
16171   match(Set dst (AddVB src1 src2));
16172   ins_cost(INSN_COST);
16173   format %{ "addv  $dst,$src1,$src2\t# vector (16B)" %}
16174   ins_encode %{
16175     __ addv(as_FloatRegister($dst$$reg), __ T16B,
16176             as_FloatRegister($src1$$reg),
16177             as_FloatRegister($src2$$reg));
16178   %}
16179   ins_pipe(vdop128);
16180 %}
16181 
16182 instruct vadd4S(vecD dst, vecD src1, vecD src2)
16183 %{
16184   predicate(n->as_Vector()->length() == 2 ||
16185             n->as_Vector()->length() == 4);
16186   match(Set dst (AddVS src1 src2));
16187   ins_cost(INSN_COST);
16188   format %{ "addv  $dst,$src1,$src2\t# vector (4H)" %}
16189   ins_encode %{
16190     __ addv(as_FloatRegister($dst$$reg), __ T4H,
16191             as_FloatRegister($src1$$reg),
16192             as_FloatRegister($src2$$reg));
16193   %}
16194   ins_pipe(vdop64);
16195 %}
16196 
16197 instruct vadd8S(vecX dst, vecX src1, vecX src2)
16198 %{
16199   predicate(n->as_Vector()->length() == 8);
16200   match(Set dst (AddVS src1 src2));
16201   ins_cost(INSN_COST);
16202   format %{ "addv  $dst,$src1,$src2\t# vector (8H)" %}
16203   ins_encode %{
16204     __ addv(as_FloatRegister($dst$$reg), __ T8H,
16205             as_FloatRegister($src1$$reg),
16206             as_FloatRegister($src2$$reg));
16207   %}
16208   ins_pipe(vdop128);
16209 %}
16210 
16211 instruct vadd2I(vecD dst, vecD src1, vecD src2)
16212 %{
16213   predicate(n->as_Vector()->length() == 2);
16214   match(Set dst (AddVI src1 src2));
16215   ins_cost(INSN_COST);
16216   format %{ "addv  $dst,$src1,$src2\t# vector (2S)" %}
16217   ins_encode %{
16218     __ addv(as_FloatRegister($dst$$reg), __ T2S,
16219             as_FloatRegister($src1$$reg),
16220             as_FloatRegister($src2$$reg));
16221   %}
16222   ins_pipe(vdop64);
16223 %}
16224 
16225 instruct vadd4I(vecX dst, vecX src1, vecX src2)
16226 %{
16227   predicate(n->as_Vector()->length() == 4);
16228   match(Set dst (AddVI src1 src2));
16229   ins_cost(INSN_COST);
16230   format %{ "addv  $dst,$src1,$src2\t# vector (4S)" %}
16231   ins_encode %{
16232     __ addv(as_FloatRegister($dst$$reg), __ T4S,
16233             as_FloatRegister($src1$$reg),
16234             as_FloatRegister($src2$$reg));
16235   %}
16236   ins_pipe(vdop128);
16237 %}
16238 
16239 instruct vadd2L(vecX dst, vecX src1, vecX src2)
16240 %{
16241   predicate(n->as_Vector()->length() == 2);
16242   match(Set dst (AddVL src1 src2));
16243   ins_cost(INSN_COST);
16244   format %{ "addv  $dst,$src1,$src2\t# vector (2L)" %}
16245   ins_encode %{
16246     __ addv(as_FloatRegister($dst$$reg), __ T2D,
16247             as_FloatRegister($src1$$reg),
16248             as_FloatRegister($src2$$reg));
16249   %}
16250   ins_pipe(vdop128);
16251 %}
16252 
16253 instruct vadd2F(vecD dst, vecD src1, vecD src2)
16254 %{
16255   predicate(n->as_Vector()->length() == 2);
16256   match(Set dst (AddVF src1 src2));
16257   ins_cost(INSN_COST);
16258   format %{ "fadd  $dst,$src1,$src2\t# vector (2S)" %}
16259   ins_encode %{
16260     __ fadd(as_FloatRegister($dst$$reg), __ T2S,
16261             as_FloatRegister($src1$$reg),
16262             as_FloatRegister($src2$$reg));
16263   %}
16264   ins_pipe(vdop_fp64);
16265 %}
16266 
16267 instruct vadd4F(vecX dst, vecX src1, vecX src2)
16268 %{
16269   predicate(n->as_Vector()->length() == 4);
16270   match(Set dst (AddVF src1 src2));
16271   ins_cost(INSN_COST);
16272   format %{ "fadd  $dst,$src1,$src2\t# vector (4S)" %}
16273   ins_encode %{
16274     __ fadd(as_FloatRegister($dst$$reg), __ T4S,
16275             as_FloatRegister($src1$$reg),
16276             as_FloatRegister($src2$$reg));
16277   %}
16278   ins_pipe(vdop_fp128);
16279 %}
16280 
16281 instruct vadd2D(vecX dst, vecX src1, vecX src2)
16282 %{
16283   match(Set dst (AddVD src1 src2));
16284   ins_cost(INSN_COST);
16285   format %{ "fadd  $dst,$src1,$src2\t# vector (2D)" %}
16286   ins_encode %{
16287     __ fadd(as_FloatRegister($dst$$reg), __ T2D,
16288             as_FloatRegister($src1$$reg),
16289             as_FloatRegister($src2$$reg));
16290   %}
16291   ins_pipe(vdop_fp128);
16292 %}
16293 
16294 // --------------------------------- SUB --------------------------------------
16295 
16296 instruct vsub8B(vecD dst, vecD src1, vecD src2)
16297 %{
16298   predicate(n->as_Vector()->length() == 4 ||
16299             n->as_Vector()->length() == 8);
16300   match(Set dst (SubVB src1 src2));
16301   ins_cost(INSN_COST);
16302   format %{ "subv  $dst,$src1,$src2\t# vector (8B)" %}
16303   ins_encode %{
16304     __ subv(as_FloatRegister($dst$$reg), __ T8B,
16305             as_FloatRegister($src1$$reg),
16306             as_FloatRegister($src2$$reg));
16307   %}
16308   ins_pipe(vdop64);
16309 %}
16310 
16311 instruct vsub16B(vecX dst, vecX src1, vecX src2)
16312 %{
16313   predicate(n->as_Vector()->length() == 16);
16314   match(Set dst (SubVB src1 src2));
16315   ins_cost(INSN_COST);
16316   format %{ "subv  $dst,$src1,$src2\t# vector (16B)" %}
16317   ins_encode %{
16318     __ subv(as_FloatRegister($dst$$reg), __ T16B,
16319             as_FloatRegister($src1$$reg),
16320             as_FloatRegister($src2$$reg));
16321   %}
16322   ins_pipe(vdop128);
16323 %}
16324 
16325 instruct vsub4S(vecD dst, vecD src1, vecD src2)
16326 %{
16327   predicate(n->as_Vector()->length() == 2 ||
16328             n->as_Vector()->length() == 4);
16329   match(Set dst (SubVS src1 src2));
16330   ins_cost(INSN_COST);
16331   format %{ "subv  $dst,$src1,$src2\t# vector (4H)" %}
16332   ins_encode %{
16333     __ subv(as_FloatRegister($dst$$reg), __ T4H,
16334             as_FloatRegister($src1$$reg),
16335             as_FloatRegister($src2$$reg));
16336   %}
16337   ins_pipe(vdop64);
16338 %}
16339 
16340 instruct vsub8S(vecX dst, vecX src1, vecX src2)
16341 %{
16342   predicate(n->as_Vector()->length() == 8);
16343   match(Set dst (SubVS src1 src2));
16344   ins_cost(INSN_COST);
16345   format %{ "subv  $dst,$src1,$src2\t# vector (8H)" %}
16346   ins_encode %{
16347     __ subv(as_FloatRegister($dst$$reg), __ T8H,
16348             as_FloatRegister($src1$$reg),
16349             as_FloatRegister($src2$$reg));
16350   %}
16351   ins_pipe(vdop128);
16352 %}
16353 
16354 instruct vsub2I(vecD dst, vecD src1, vecD src2)
16355 %{
16356   predicate(n->as_Vector()->length() == 2);
16357   match(Set dst (SubVI src1 src2));
16358   ins_cost(INSN_COST);
16359   format %{ "subv  $dst,$src1,$src2\t# vector (2S)" %}
16360   ins_encode %{
16361     __ subv(as_FloatRegister($dst$$reg), __ T2S,
16362             as_FloatRegister($src1$$reg),
16363             as_FloatRegister($src2$$reg));
16364   %}
16365   ins_pipe(vdop64);
16366 %}
16367 
16368 instruct vsub4I(vecX dst, vecX src1, vecX src2)
16369 %{
16370   predicate(n->as_Vector()->length() == 4);
16371   match(Set dst (SubVI src1 src2));
16372   ins_cost(INSN_COST);
16373   format %{ "subv  $dst,$src1,$src2\t# vector (4S)" %}
16374   ins_encode %{
16375     __ subv(as_FloatRegister($dst$$reg), __ T4S,
16376             as_FloatRegister($src1$$reg),
16377             as_FloatRegister($src2$$reg));
16378   %}
16379   ins_pipe(vdop128);
16380 %}
16381 
16382 instruct vsub2L(vecX dst, vecX src1, vecX src2)
16383 %{
16384   predicate(n->as_Vector()->length() == 2);
16385   match(Set dst (SubVL src1 src2));
16386   ins_cost(INSN_COST);
16387   format %{ "subv  $dst,$src1,$src2\t# vector (2L)" %}
16388   ins_encode %{
16389     __ subv(as_FloatRegister($dst$$reg), __ T2D,
16390             as_FloatRegister($src1$$reg),
16391             as_FloatRegister($src2$$reg));
16392   %}
16393   ins_pipe(vdop128);
16394 %}
16395 
16396 instruct vsub2F(vecD dst, vecD src1, vecD src2)
16397 %{
16398   predicate(n->as_Vector()->length() == 2);
16399   match(Set dst (SubVF src1 src2));
16400   ins_cost(INSN_COST);
16401   format %{ "fsub  $dst,$src1,$src2\t# vector (2S)" %}
16402   ins_encode %{
16403     __ fsub(as_FloatRegister($dst$$reg), __ T2S,
16404             as_FloatRegister($src1$$reg),
16405             as_FloatRegister($src2$$reg));
16406   %}
16407   ins_pipe(vdop_fp64);
16408 %}
16409 
16410 instruct vsub4F(vecX dst, vecX src1, vecX src2)
16411 %{
16412   predicate(n->as_Vector()->length() == 4);
16413   match(Set dst (SubVF src1 src2));
16414   ins_cost(INSN_COST);
16415   format %{ "fsub  $dst,$src1,$src2\t# vector (4S)" %}
16416   ins_encode %{
16417     __ fsub(as_FloatRegister($dst$$reg), __ T4S,
16418             as_FloatRegister($src1$$reg),
16419             as_FloatRegister($src2$$reg));
16420   %}
16421   ins_pipe(vdop_fp128);
16422 %}
16423 
16424 instruct vsub2D(vecX dst, vecX src1, vecX src2)
16425 %{
16426   predicate(n->as_Vector()->length() == 2);
16427   match(Set dst (SubVD src1 src2));
16428   ins_cost(INSN_COST);
16429   format %{ "fsub  $dst,$src1,$src2\t# vector (2D)" %}
16430   ins_encode %{
16431     __ fsub(as_FloatRegister($dst$$reg), __ T2D,
16432             as_FloatRegister($src1$$reg),
16433             as_FloatRegister($src2$$reg));
16434   %}
16435   ins_pipe(vdop_fp128);
16436 %}
16437 
16438 // --------------------------------- MUL --------------------------------------
16439 
16440 instruct vmul4S(vecD dst, vecD src1, vecD src2)
16441 %{
16442   predicate(n->as_Vector()->length() == 2 ||
16443             n->as_Vector()->length() == 4);
16444   match(Set dst (MulVS src1 src2));
16445   ins_cost(INSN_COST);
16446   format %{ "mulv  $dst,$src1,$src2\t# vector (4H)" %}
16447   ins_encode %{
16448     __ mulv(as_FloatRegister($dst$$reg), __ T4H,
16449             as_FloatRegister($src1$$reg),
16450             as_FloatRegister($src2$$reg));
16451   %}
16452   ins_pipe(vmul64);
16453 %}
16454 
16455 instruct vmul8S(vecX dst, vecX src1, vecX src2)
16456 %{
16457   predicate(n->as_Vector()->length() == 8);
16458   match(Set dst (MulVS src1 src2));
16459   ins_cost(INSN_COST);
16460   format %{ "mulv  $dst,$src1,$src2\t# vector (8H)" %}
16461   ins_encode %{
16462     __ mulv(as_FloatRegister($dst$$reg), __ T8H,
16463             as_FloatRegister($src1$$reg),
16464             as_FloatRegister($src2$$reg));
16465   %}
16466   ins_pipe(vmul128);
16467 %}
16468 
16469 instruct vmul2I(vecD dst, vecD src1, vecD src2)
16470 %{
16471   predicate(n->as_Vector()->length() == 2);
16472   match(Set dst (MulVI src1 src2));
16473   ins_cost(INSN_COST);
16474   format %{ "mulv  $dst,$src1,$src2\t# vector (2S)" %}
16475   ins_encode %{
16476     __ mulv(as_FloatRegister($dst$$reg), __ T2S,
16477             as_FloatRegister($src1$$reg),
16478             as_FloatRegister($src2$$reg));
16479   %}
16480   ins_pipe(vmul64);
16481 %}
16482 
16483 instruct vmul4I(vecX dst, vecX src1, vecX src2)
16484 %{
16485   predicate(n->as_Vector()->length() == 4);
16486   match(Set dst (MulVI src1 src2));
16487   ins_cost(INSN_COST);
16488   format %{ "mulv  $dst,$src1,$src2\t# vector (4S)" %}
16489   ins_encode %{
16490     __ mulv(as_FloatRegister($dst$$reg), __ T4S,
16491             as_FloatRegister($src1$$reg),
16492             as_FloatRegister($src2$$reg));
16493   %}
16494   ins_pipe(vmul128);
16495 %}
16496 
16497 instruct vmul2F(vecD dst, vecD src1, vecD src2)
16498 %{
16499   predicate(n->as_Vector()->length() == 2);
16500   match(Set dst (MulVF src1 src2));
16501   ins_cost(INSN_COST);
16502   format %{ "fmul  $dst,$src1,$src2\t# vector (2S)" %}
16503   ins_encode %{
16504     __ fmul(as_FloatRegister($dst$$reg), __ T2S,
16505             as_FloatRegister($src1$$reg),
16506             as_FloatRegister($src2$$reg));
16507   %}
16508   ins_pipe(vmuldiv_fp64);
16509 %}
16510 
16511 instruct vmul4F(vecX dst, vecX src1, vecX src2)
16512 %{
16513   predicate(n->as_Vector()->length() == 4);
16514   match(Set dst (MulVF src1 src2));
16515   ins_cost(INSN_COST);
16516   format %{ "fmul  $dst,$src1,$src2\t# vector (4S)" %}
16517   ins_encode %{
16518     __ fmul(as_FloatRegister($dst$$reg), __ T4S,
16519             as_FloatRegister($src1$$reg),
16520             as_FloatRegister($src2$$reg));
16521   %}
16522   ins_pipe(vmuldiv_fp128);
16523 %}
16524 
16525 instruct vmul2D(vecX dst, vecX src1, vecX src2)
16526 %{
16527   predicate(n->as_Vector()->length() == 2);
16528   match(Set dst (MulVD src1 src2));
16529   ins_cost(INSN_COST);
16530   format %{ "fmul  $dst,$src1,$src2\t# vector (2D)" %}
16531   ins_encode %{
16532     __ fmul(as_FloatRegister($dst$$reg), __ T2D,
16533             as_FloatRegister($src1$$reg),
16534             as_FloatRegister($src2$$reg));
16535   %}
16536   ins_pipe(vmuldiv_fp128);
16537 %}
16538 
16539 // --------------------------------- MLA --------------------------------------
16540 
16541 instruct vmla4S(vecD dst, vecD src1, vecD src2)
16542 %{
16543   predicate(n->as_Vector()->length() == 2 ||
16544             n->as_Vector()->length() == 4);
16545   match(Set dst (AddVS dst (MulVS src1 src2)));
16546   ins_cost(INSN_COST);
16547   format %{ "mlav  $dst,$src1,$src2\t# vector (4H)" %}
16548   ins_encode %{
16549     __ mlav(as_FloatRegister($dst$$reg), __ T4H,
16550             as_FloatRegister($src1$$reg),
16551             as_FloatRegister($src2$$reg));
16552   %}
16553   ins_pipe(vmla64);
16554 %}
16555 
16556 instruct vmla8S(vecX dst, vecX src1, vecX src2)
16557 %{
16558   predicate(n->as_Vector()->length() == 8);
16559   match(Set dst (AddVS dst (MulVS src1 src2)));
16560   ins_cost(INSN_COST);
16561   format %{ "mlav  $dst,$src1,$src2\t# vector (8H)" %}
16562   ins_encode %{
16563     __ mlav(as_FloatRegister($dst$$reg), __ T8H,
16564             as_FloatRegister($src1$$reg),
16565             as_FloatRegister($src2$$reg));
16566   %}
16567   ins_pipe(vmla128);
16568 %}
16569 
16570 instruct vmla2I(vecD dst, vecD src1, vecD src2)
16571 %{
16572   predicate(n->as_Vector()->length() == 2);
16573   match(Set dst (AddVI dst (MulVI src1 src2)));
16574   ins_cost(INSN_COST);
16575   format %{ "mlav  $dst,$src1,$src2\t# vector (2S)" %}
16576   ins_encode %{
16577     __ mlav(as_FloatRegister($dst$$reg), __ T2S,
16578             as_FloatRegister($src1$$reg),
16579             as_FloatRegister($src2$$reg));
16580   %}
16581   ins_pipe(vmla64);
16582 %}
16583 
16584 instruct vmla4I(vecX dst, vecX src1, vecX src2)
16585 %{
16586   predicate(n->as_Vector()->length() == 4);
16587   match(Set dst (AddVI dst (MulVI src1 src2)));
16588   ins_cost(INSN_COST);
16589   format %{ "mlav  $dst,$src1,$src2\t# vector (4S)" %}
16590   ins_encode %{
16591     __ mlav(as_FloatRegister($dst$$reg), __ T4S,
16592             as_FloatRegister($src1$$reg),
16593             as_FloatRegister($src2$$reg));
16594   %}
16595   ins_pipe(vmla128);
16596 %}
16597 
16598 // dst + src1 * src2
16599 instruct vmla2F(vecD dst, vecD src1, vecD src2) %{
16600   predicate(UseFMA && n->as_Vector()->length() == 2);
16601   match(Set dst (FmaVF  dst (Binary src1 src2)));
16602   format %{ "fmla  $dst,$src1,$src2\t# vector (2S)" %}
16603   ins_cost(INSN_COST);
16604   ins_encode %{
16605     __ fmla(as_FloatRegister($dst$$reg), __ T2S,
16606             as_FloatRegister($src1$$reg),
16607             as_FloatRegister($src2$$reg));
16608   %}
16609   ins_pipe(vmuldiv_fp64);
16610 %}
16611 
16612 // dst + src1 * src2
16613 instruct vmla4F(vecX dst, vecX src1, vecX src2) %{
16614   predicate(UseFMA && n->as_Vector()->length() == 4);
16615   match(Set dst (FmaVF  dst (Binary src1 src2)));
16616   format %{ "fmla  $dst,$src1,$src2\t# vector (4S)" %}
16617   ins_cost(INSN_COST);
16618   ins_encode %{
16619     __ fmla(as_FloatRegister($dst$$reg), __ T4S,
16620             as_FloatRegister($src1$$reg),
16621             as_FloatRegister($src2$$reg));
16622   %}
16623   ins_pipe(vmuldiv_fp128);
16624 %}
16625 
16626 // dst + src1 * src2
16627 instruct vmla2D(vecX dst, vecX src1, vecX src2) %{
16628   predicate(UseFMA && n->as_Vector()->length() == 2);
16629   match(Set dst (FmaVD  dst (Binary src1 src2)));
16630   format %{ "fmla  $dst,$src1,$src2\t# vector (2D)" %}
16631   ins_cost(INSN_COST);
16632   ins_encode %{
16633     __ fmla(as_FloatRegister($dst$$reg), __ T2D,
16634             as_FloatRegister($src1$$reg),
16635             as_FloatRegister($src2$$reg));
16636   %}
16637   ins_pipe(vmuldiv_fp128);
16638 %}
16639 
16640 // --------------------------------- MLS --------------------------------------
16641 
16642 instruct vmls4S(vecD dst, vecD src1, vecD src2)
16643 %{
16644   predicate(n->as_Vector()->length() == 2 ||
16645             n->as_Vector()->length() == 4);
16646   match(Set dst (SubVS dst (MulVS src1 src2)));
16647   ins_cost(INSN_COST);
16648   format %{ "mlsv  $dst,$src1,$src2\t# vector (4H)" %}
16649   ins_encode %{
16650     __ mlsv(as_FloatRegister($dst$$reg), __ T4H,
16651             as_FloatRegister($src1$$reg),
16652             as_FloatRegister($src2$$reg));
16653   %}
16654   ins_pipe(vmla64);
16655 %}
16656 
16657 instruct vmls8S(vecX dst, vecX src1, vecX src2)
16658 %{
16659   predicate(n->as_Vector()->length() == 8);
16660   match(Set dst (SubVS dst (MulVS src1 src2)));
16661   ins_cost(INSN_COST);
16662   format %{ "mlsv  $dst,$src1,$src2\t# vector (8H)" %}
16663   ins_encode %{
16664     __ mlsv(as_FloatRegister($dst$$reg), __ T8H,
16665             as_FloatRegister($src1$$reg),
16666             as_FloatRegister($src2$$reg));
16667   %}
16668   ins_pipe(vmla128);
16669 %}
16670 
16671 instruct vmls2I(vecD dst, vecD src1, vecD src2)
16672 %{
16673   predicate(n->as_Vector()->length() == 2);
16674   match(Set dst (SubVI dst (MulVI src1 src2)));
16675   ins_cost(INSN_COST);
16676   format %{ "mlsv  $dst,$src1,$src2\t# vector (2S)" %}
16677   ins_encode %{
16678     __ mlsv(as_FloatRegister($dst$$reg), __ T2S,
16679             as_FloatRegister($src1$$reg),
16680             as_FloatRegister($src2$$reg));
16681   %}
16682   ins_pipe(vmla64);
16683 %}
16684 
16685 instruct vmls4I(vecX dst, vecX src1, vecX src2)
16686 %{
16687   predicate(n->as_Vector()->length() == 4);
16688   match(Set dst (SubVI dst (MulVI src1 src2)));
16689   ins_cost(INSN_COST);
16690   format %{ "mlsv  $dst,$src1,$src2\t# vector (4S)" %}
16691   ins_encode %{
16692     __ mlsv(as_FloatRegister($dst$$reg), __ T4S,
16693             as_FloatRegister($src1$$reg),
16694             as_FloatRegister($src2$$reg));
16695   %}
16696   ins_pipe(vmla128);
16697 %}
16698 
16699 // dst - src1 * src2
16700 instruct vmls2F(vecD dst, vecD src1, vecD src2) %{
16701   predicate(UseFMA && n->as_Vector()->length() == 2);
16702   match(Set dst (FmaVF  dst (Binary (NegVF src1) src2)));
16703   match(Set dst (FmaVF  dst (Binary src1 (NegVF src2))));
16704   format %{ "fmls  $dst,$src1,$src2\t# vector (2S)" %}
16705   ins_cost(INSN_COST);
16706   ins_encode %{
16707     __ fmls(as_FloatRegister($dst$$reg), __ T2S,
16708             as_FloatRegister($src1$$reg),
16709             as_FloatRegister($src2$$reg));
16710   %}
16711   ins_pipe(vmuldiv_fp64);
16712 %}
16713 
16714 // dst - src1 * src2
16715 instruct vmls4F(vecX dst, vecX src1, vecX src2) %{
16716   predicate(UseFMA && n->as_Vector()->length() == 4);
16717   match(Set dst (FmaVF  dst (Binary (NegVF src1) src2)));
16718   match(Set dst (FmaVF  dst (Binary src1 (NegVF src2))));
16719   format %{ "fmls  $dst,$src1,$src2\t# vector (4S)" %}
16720   ins_cost(INSN_COST);
16721   ins_encode %{
16722     __ fmls(as_FloatRegister($dst$$reg), __ T4S,
16723             as_FloatRegister($src1$$reg),
16724             as_FloatRegister($src2$$reg));
16725   %}
16726   ins_pipe(vmuldiv_fp128);
16727 %}
16728 
16729 // dst - src1 * src2
16730 instruct vmls2D(vecX dst, vecX src1, vecX src2) %{
16731   predicate(UseFMA && n->as_Vector()->length() == 2);
16732   match(Set dst (FmaVD  dst (Binary (NegVD src1) src2)));
16733   match(Set dst (FmaVD  dst (Binary src1 (NegVD src2))));
16734   format %{ "fmls  $dst,$src1,$src2\t# vector (2D)" %}
16735   ins_cost(INSN_COST);
16736   ins_encode %{
16737     __ fmls(as_FloatRegister($dst$$reg), __ T2D,
16738             as_FloatRegister($src1$$reg),
16739             as_FloatRegister($src2$$reg));
16740   %}
16741   ins_pipe(vmuldiv_fp128);
16742 %}
16743 
16744 // --------------------------------- DIV --------------------------------------
16745 
16746 instruct vdiv2F(vecD dst, vecD src1, vecD src2)
16747 %{
16748   predicate(n->as_Vector()->length() == 2);
16749   match(Set dst (DivVF src1 src2));
16750   ins_cost(INSN_COST);
16751   format %{ "fdiv  $dst,$src1,$src2\t# vector (2S)" %}
16752   ins_encode %{
16753     __ fdiv(as_FloatRegister($dst$$reg), __ T2S,
16754             as_FloatRegister($src1$$reg),
16755             as_FloatRegister($src2$$reg));
16756   %}
16757   ins_pipe(vmuldiv_fp64);
16758 %}
16759 
16760 instruct vdiv4F(vecX dst, vecX src1, vecX src2)
16761 %{
16762   predicate(n->as_Vector()->length() == 4);
16763   match(Set dst (DivVF src1 src2));
16764   ins_cost(INSN_COST);
16765   format %{ "fdiv  $dst,$src1,$src2\t# vector (4S)" %}
16766   ins_encode %{
16767     __ fdiv(as_FloatRegister($dst$$reg), __ T4S,
16768             as_FloatRegister($src1$$reg),
16769             as_FloatRegister($src2$$reg));
16770   %}
16771   ins_pipe(vmuldiv_fp128);
16772 %}
16773 
16774 instruct vdiv2D(vecX dst, vecX src1, vecX src2)
16775 %{
16776   predicate(n->as_Vector()->length() == 2);
16777   match(Set dst (DivVD src1 src2));
16778   ins_cost(INSN_COST);
16779   format %{ "fdiv  $dst,$src1,$src2\t# vector (2D)" %}
16780   ins_encode %{
16781     __ fdiv(as_FloatRegister($dst$$reg), __ T2D,
16782             as_FloatRegister($src1$$reg),
16783             as_FloatRegister($src2$$reg));
16784   %}
16785   ins_pipe(vmuldiv_fp128);
16786 %}
16787 
16788 // --------------------------------- SQRT -------------------------------------
16789 
16790 instruct vsqrt2D(vecX dst, vecX src)
16791 %{
16792   predicate(n->as_Vector()->length() == 2);
16793   match(Set dst (SqrtVD src));
16794   format %{ "fsqrt  $dst, $src\t# vector (2D)" %}
16795   ins_encode %{
16796     __ fsqrt(as_FloatRegister($dst$$reg), __ T2D,
16797              as_FloatRegister($src$$reg));
16798   %}
16799   ins_pipe(vsqrt_fp128);
16800 %}
16801 
16802 // --------------------------------- ABS --------------------------------------
16803 
16804 instruct vabs2F(vecD dst, vecD src)
16805 %{
16806   predicate(n->as_Vector()->length() == 2);
16807   match(Set dst (AbsVF src));
16808   ins_cost(INSN_COST * 3);
16809   format %{ "fabs  $dst,$src\t# vector (2S)" %}
16810   ins_encode %{
16811     __ fabs(as_FloatRegister($dst$$reg), __ T2S,
16812             as_FloatRegister($src$$reg));
16813   %}
16814   ins_pipe(vunop_fp64);
16815 %}
16816 
16817 instruct vabs4F(vecX dst, vecX src)
16818 %{
16819   predicate(n->as_Vector()->length() == 4);
16820   match(Set dst (AbsVF src));
16821   ins_cost(INSN_COST * 3);
16822   format %{ "fabs  $dst,$src\t# vector (4S)" %}
16823   ins_encode %{
16824     __ fabs(as_FloatRegister($dst$$reg), __ T4S,
16825             as_FloatRegister($src$$reg));
16826   %}
16827   ins_pipe(vunop_fp128);
16828 %}
16829 
16830 instruct vabs2D(vecX dst, vecX src)
16831 %{
16832   predicate(n->as_Vector()->length() == 2);
16833   match(Set dst (AbsVD src));
16834   ins_cost(INSN_COST * 3);
16835   format %{ "fabs  $dst,$src\t# vector (2D)" %}
16836   ins_encode %{
16837     __ fabs(as_FloatRegister($dst$$reg), __ T2D,
16838             as_FloatRegister($src$$reg));
16839   %}
16840   ins_pipe(vunop_fp128);
16841 %}
16842 
16843 // --------------------------------- NEG --------------------------------------
16844 
16845 instruct vneg2F(vecD dst, vecD src)
16846 %{
16847   predicate(n->as_Vector()->length() == 2);
16848   match(Set dst (NegVF src));
16849   ins_cost(INSN_COST * 3);
16850   format %{ "fneg  $dst,$src\t# vector (2S)" %}
16851   ins_encode %{
16852     __ fneg(as_FloatRegister($dst$$reg), __ T2S,
16853             as_FloatRegister($src$$reg));
16854   %}
16855   ins_pipe(vunop_fp64);
16856 %}
16857 
16858 instruct vneg4F(vecX dst, vecX src)
16859 %{
16860   predicate(n->as_Vector()->length() == 4);
16861   match(Set dst (NegVF src));
16862   ins_cost(INSN_COST * 3);
16863   format %{ "fneg  $dst,$src\t# vector (4S)" %}
16864   ins_encode %{
16865     __ fneg(as_FloatRegister($dst$$reg), __ T4S,
16866             as_FloatRegister($src$$reg));
16867   %}
16868   ins_pipe(vunop_fp128);
16869 %}
16870 
16871 instruct vneg2D(vecX dst, vecX src)
16872 %{
16873   predicate(n->as_Vector()->length() == 2);
16874   match(Set dst (NegVD src));
16875   ins_cost(INSN_COST * 3);
16876   format %{ "fneg  $dst,$src\t# vector (2D)" %}
16877   ins_encode %{
16878     __ fneg(as_FloatRegister($dst$$reg), __ T2D,
16879             as_FloatRegister($src$$reg));
16880   %}
16881   ins_pipe(vunop_fp128);
16882 %}
16883 
16884 // --------------------------------- AND --------------------------------------
16885 
16886 instruct vand8B(vecD dst, vecD src1, vecD src2)
16887 %{
16888   predicate(n->as_Vector()->length_in_bytes() == 4 ||
16889             n->as_Vector()->length_in_bytes() == 8);
16890   match(Set dst (AndV src1 src2));
16891   ins_cost(INSN_COST);
16892   format %{ "and  $dst,$src1,$src2\t# vector (8B)" %}
16893   ins_encode %{
16894     __ andr(as_FloatRegister($dst$$reg), __ T8B,
16895             as_FloatRegister($src1$$reg),
16896             as_FloatRegister($src2$$reg));
16897   %}
16898   ins_pipe(vlogical64);
16899 %}
16900 
16901 instruct vand16B(vecX dst, vecX src1, vecX src2)
16902 %{
16903   predicate(n->as_Vector()->length_in_bytes() == 16);
16904   match(Set dst (AndV src1 src2));
16905   ins_cost(INSN_COST);
16906   format %{ "and  $dst,$src1,$src2\t# vector (16B)" %}
16907   ins_encode %{
16908     __ andr(as_FloatRegister($dst$$reg), __ T16B,
16909             as_FloatRegister($src1$$reg),
16910             as_FloatRegister($src2$$reg));
16911   %}
16912   ins_pipe(vlogical128);
16913 %}
16914 
16915 // --------------------------------- OR ---------------------------------------
16916 
16917 instruct vor8B(vecD dst, vecD src1, vecD src2)
16918 %{
16919   predicate(n->as_Vector()->length_in_bytes() == 4 ||
16920             n->as_Vector()->length_in_bytes() == 8);
16921   match(Set dst (OrV src1 src2));
16922   ins_cost(INSN_COST);
16923   format %{ "and  $dst,$src1,$src2\t# vector (8B)" %}
16924   ins_encode %{
16925     __ orr(as_FloatRegister($dst$$reg), __ T8B,
16926             as_FloatRegister($src1$$reg),
16927             as_FloatRegister($src2$$reg));
16928   %}
16929   ins_pipe(vlogical64);
16930 %}
16931 
16932 instruct vor16B(vecX dst, vecX src1, vecX src2)
16933 %{
16934   predicate(n->as_Vector()->length_in_bytes() == 16);
16935   match(Set dst (OrV src1 src2));
16936   ins_cost(INSN_COST);
16937   format %{ "orr  $dst,$src1,$src2\t# vector (16B)" %}
16938   ins_encode %{
16939     __ orr(as_FloatRegister($dst$$reg), __ T16B,
16940             as_FloatRegister($src1$$reg),
16941             as_FloatRegister($src2$$reg));
16942   %}
16943   ins_pipe(vlogical128);
16944 %}
16945 
16946 // --------------------------------- XOR --------------------------------------
16947 
16948 instruct vxor8B(vecD dst, vecD src1, vecD src2)
16949 %{
16950   predicate(n->as_Vector()->length_in_bytes() == 4 ||
16951             n->as_Vector()->length_in_bytes() == 8);
16952   match(Set dst (XorV src1 src2));
16953   ins_cost(INSN_COST);
16954   format %{ "xor  $dst,$src1,$src2\t# vector (8B)" %}
16955   ins_encode %{
16956     __ eor(as_FloatRegister($dst$$reg), __ T8B,
16957             as_FloatRegister($src1$$reg),
16958             as_FloatRegister($src2$$reg));
16959   %}
16960   ins_pipe(vlogical64);
16961 %}
16962 
16963 instruct vxor16B(vecX dst, vecX src1, vecX src2)
16964 %{
16965   predicate(n->as_Vector()->length_in_bytes() == 16);
16966   match(Set dst (XorV src1 src2));
16967   ins_cost(INSN_COST);
16968   format %{ "xor  $dst,$src1,$src2\t# vector (16B)" %}
16969   ins_encode %{
16970     __ eor(as_FloatRegister($dst$$reg), __ T16B,
16971             as_FloatRegister($src1$$reg),
16972             as_FloatRegister($src2$$reg));
16973   %}
16974   ins_pipe(vlogical128);
16975 %}
16976 
16977 // ------------------------------ Shift ---------------------------------------
16978 instruct vshiftcnt8B(vecD dst, iRegIorL2I cnt) %{
16979   predicate(n->as_Vector()->length_in_bytes() == 8);
16980   match(Set dst (LShiftCntV cnt));
16981   match(Set dst (RShiftCntV cnt));
16982   format %{ "dup  $dst, $cnt\t# shift count vector (8B)" %}
16983   ins_encode %{
16984     __ dup(as_FloatRegister($dst$$reg), __ T8B, as_Register($cnt$$reg));
16985   %}
16986   ins_pipe(vdup_reg_reg64);
16987 %}
16988 
16989 instruct vshiftcnt16B(vecX dst, iRegIorL2I cnt) %{
16990   predicate(n->as_Vector()->length_in_bytes() == 16);
16991   match(Set dst (LShiftCntV cnt));
16992   match(Set dst (RShiftCntV cnt));
16993   format %{ "dup  $dst, $cnt\t# shift count vector (16B)" %}
16994   ins_encode %{
16995     __ dup(as_FloatRegister($dst$$reg), __ T16B, as_Register($cnt$$reg));
16996   %}
16997   ins_pipe(vdup_reg_reg128);
16998 %}
16999 
17000 instruct vsll8B(vecD dst, vecD src, vecD shift) %{
17001   predicate(n->as_Vector()->length() == 4 ||
17002             n->as_Vector()->length() == 8);
17003   match(Set dst (LShiftVB src shift));
17004   ins_cost(INSN_COST);
17005   format %{ "sshl  $dst,$src,$shift\t# vector (8B)" %}
17006   ins_encode %{
17007     __ sshl(as_FloatRegister($dst$$reg), __ T8B,
17008             as_FloatRegister($src$$reg),
17009             as_FloatRegister($shift$$reg));
17010   %}
17011   ins_pipe(vshift64);
17012 %}
17013 
17014 instruct vsll16B(vecX dst, vecX src, vecX shift) %{
17015   predicate(n->as_Vector()->length() == 16);
17016   match(Set dst (LShiftVB src shift));
17017   ins_cost(INSN_COST);
17018   format %{ "sshl  $dst,$src,$shift\t# vector (16B)" %}
17019   ins_encode %{
17020     __ sshl(as_FloatRegister($dst$$reg), __ T16B,
17021             as_FloatRegister($src$$reg),
17022             as_FloatRegister($shift$$reg));
17023   %}
17024   ins_pipe(vshift128);
17025 %}
17026 
17027 // Right shifts with vector shift count on aarch64 SIMD are implemented
17028 // as left shift by negative shift count.
17029 // There are two cases for vector shift count.
17030 //
17031 // Case 1: The vector shift count is from replication.
17032 //        |            |
17033 //    LoadVector  RShiftCntV
17034 //        |       /
17035 //     RShiftVI
17036 // Note: In inner loop, multiple neg instructions are used, which can be
17037 // moved to outer loop and merge into one neg instruction.
17038 //
17039 // Case 2: The vector shift count is from loading.
17040 // This case isn't supported by middle-end now. But it's supported by
17041 // panama/vectorIntrinsics(JEP 338: Vector API).
17042 //        |            |
17043 //    LoadVector  LoadVector
17044 //        |       /
17045 //     RShiftVI
17046 //
17047 
17048 instruct vsra8B(vecD dst, vecD src, vecD shift, vecD tmp) %{
17049   predicate(n->as_Vector()->length() == 4 ||
17050             n->as_Vector()->length() == 8);
17051   match(Set dst (RShiftVB src shift));
17052   ins_cost(INSN_COST);
17053   effect(TEMP tmp);
17054   format %{ "negr  $tmp,$shift\t"
17055             "sshl  $dst,$src,$tmp\t# vector (8B)" %}
17056   ins_encode %{
17057     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
17058             as_FloatRegister($shift$$reg));
17059     __ sshl(as_FloatRegister($dst$$reg), __ T8B,
17060             as_FloatRegister($src$$reg),
17061             as_FloatRegister($tmp$$reg));
17062   %}
17063   ins_pipe(vshift64);
17064 %}
17065 
17066 instruct vsra16B(vecX dst, vecX src, vecX shift, vecX tmp) %{
17067   predicate(n->as_Vector()->length() == 16);
17068   match(Set dst (RShiftVB src shift));
17069   ins_cost(INSN_COST);
17070   effect(TEMP tmp);
17071   format %{ "negr  $tmp,$shift\t"
17072             "sshl  $dst,$src,$tmp\t# vector (16B)" %}
17073   ins_encode %{
17074     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17075             as_FloatRegister($shift$$reg));
17076     __ sshl(as_FloatRegister($dst$$reg), __ T16B,
17077             as_FloatRegister($src$$reg),
17078             as_FloatRegister($tmp$$reg));
17079   %}
17080   ins_pipe(vshift128);
17081 %}
17082 
17083 instruct vsrl8B(vecD dst, vecD src, vecD shift, vecD tmp) %{
17084   predicate(n->as_Vector()->length() == 4 ||
17085             n->as_Vector()->length() == 8);
17086   match(Set dst (URShiftVB src shift));
17087   ins_cost(INSN_COST);
17088   effect(TEMP tmp);
17089   format %{ "negr  $tmp,$shift\t"
17090             "ushl  $dst,$src,$tmp\t# vector (8B)" %}
17091   ins_encode %{
17092     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
17093             as_FloatRegister($shift$$reg));
17094     __ ushl(as_FloatRegister($dst$$reg), __ T8B,
17095             as_FloatRegister($src$$reg),
17096             as_FloatRegister($tmp$$reg));
17097   %}
17098   ins_pipe(vshift64);
17099 %}
17100 
17101 instruct vsrl16B(vecX dst, vecX src, vecX shift, vecX tmp) %{
17102   predicate(n->as_Vector()->length() == 16);
17103   match(Set dst (URShiftVB src shift));
17104   ins_cost(INSN_COST);
17105   effect(TEMP tmp);
17106   format %{ "negr  $tmp,$shift\t"
17107             "ushl  $dst,$src,$tmp\t# vector (16B)" %}
17108   ins_encode %{
17109     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17110             as_FloatRegister($shift$$reg));
17111     __ ushl(as_FloatRegister($dst$$reg), __ T16B,
17112             as_FloatRegister($src$$reg),
17113             as_FloatRegister($tmp$$reg));
17114   %}
17115   ins_pipe(vshift128);
17116 %}
17117 
17118 instruct vsll8B_imm(vecD dst, vecD src, immI shift) %{
17119   predicate(n->as_Vector()->length() == 4 ||
17120             n->as_Vector()->length() == 8);
17121   match(Set dst (LShiftVB src shift));
17122   ins_cost(INSN_COST);
17123   format %{ "shl    $dst, $src, $shift\t# vector (8B)" %}
17124   ins_encode %{
17125     int sh = (int)$shift$$constant;
17126     if (sh >= 8) {
17127       __ eor(as_FloatRegister($dst$$reg), __ T8B,
17128              as_FloatRegister($src$$reg),
17129              as_FloatRegister($src$$reg));
17130     } else {
17131       __ shl(as_FloatRegister($dst$$reg), __ T8B,
17132              as_FloatRegister($src$$reg), sh);
17133     }
17134   %}
17135   ins_pipe(vshift64_imm);
17136 %}
17137 
17138 instruct vsll16B_imm(vecX dst, vecX src, immI shift) %{
17139   predicate(n->as_Vector()->length() == 16);
17140   match(Set dst (LShiftVB src shift));
17141   ins_cost(INSN_COST);
17142   format %{ "shl    $dst, $src, $shift\t# vector (16B)" %}
17143   ins_encode %{
17144     int sh = (int)$shift$$constant;
17145     if (sh >= 8) {
17146       __ eor(as_FloatRegister($dst$$reg), __ T16B,
17147              as_FloatRegister($src$$reg),
17148              as_FloatRegister($src$$reg));
17149     } else {
17150       __ shl(as_FloatRegister($dst$$reg), __ T16B,
17151              as_FloatRegister($src$$reg), sh);
17152     }
17153   %}
17154   ins_pipe(vshift128_imm);
17155 %}
17156 
17157 instruct vsra8B_imm(vecD dst, vecD src, immI shift) %{
17158   predicate(n->as_Vector()->length() == 4 ||
17159             n->as_Vector()->length() == 8);
17160   match(Set dst (RShiftVB src shift));
17161   ins_cost(INSN_COST);
17162   format %{ "sshr    $dst, $src, $shift\t# vector (8B)" %}
17163   ins_encode %{
17164     int sh = (int)$shift$$constant;
17165     if (sh >= 8) sh = 7;
17166     __ sshr(as_FloatRegister($dst$$reg), __ T8B,
17167            as_FloatRegister($src$$reg), sh);
17168   %}
17169   ins_pipe(vshift64_imm);
17170 %}
17171 
17172 instruct vsra16B_imm(vecX dst, vecX src, immI shift) %{
17173   predicate(n->as_Vector()->length() == 16);
17174   match(Set dst (RShiftVB src shift));
17175   ins_cost(INSN_COST);
17176   format %{ "sshr    $dst, $src, $shift\t# vector (16B)" %}
17177   ins_encode %{
17178     int sh = (int)$shift$$constant;
17179     if (sh >= 8) sh = 7;
17180     __ sshr(as_FloatRegister($dst$$reg), __ T16B,
17181            as_FloatRegister($src$$reg), sh);
17182   %}
17183   ins_pipe(vshift128_imm);
17184 %}
17185 
17186 instruct vsrl8B_imm(vecD dst, vecD src, immI shift) %{
17187   predicate(n->as_Vector()->length() == 4 ||
17188             n->as_Vector()->length() == 8);
17189   match(Set dst (URShiftVB src shift));
17190   ins_cost(INSN_COST);
17191   format %{ "ushr    $dst, $src, $shift\t# vector (8B)" %}
17192   ins_encode %{
17193     int sh = (int)$shift$$constant;
17194     if (sh >= 8) {
17195       __ eor(as_FloatRegister($dst$$reg), __ T8B,
17196              as_FloatRegister($src$$reg),
17197              as_FloatRegister($src$$reg));
17198     } else {
17199       __ ushr(as_FloatRegister($dst$$reg), __ T8B,
17200              as_FloatRegister($src$$reg), sh);
17201     }
17202   %}
17203   ins_pipe(vshift64_imm);
17204 %}
17205 
17206 instruct vsrl16B_imm(vecX dst, vecX src, immI shift) %{
17207   predicate(n->as_Vector()->length() == 16);
17208   match(Set dst (URShiftVB src shift));
17209   ins_cost(INSN_COST);
17210   format %{ "ushr    $dst, $src, $shift\t# vector (16B)" %}
17211   ins_encode %{
17212     int sh = (int)$shift$$constant;
17213     if (sh >= 8) {
17214       __ eor(as_FloatRegister($dst$$reg), __ T16B,
17215              as_FloatRegister($src$$reg),
17216              as_FloatRegister($src$$reg));
17217     } else {
17218       __ ushr(as_FloatRegister($dst$$reg), __ T16B,
17219              as_FloatRegister($src$$reg), sh);
17220     }
17221   %}
17222   ins_pipe(vshift128_imm);
17223 %}
17224 
17225 instruct vsll4S(vecD dst, vecD src, vecD shift) %{
17226   predicate(n->as_Vector()->length() == 2 ||
17227             n->as_Vector()->length() == 4);
17228   match(Set dst (LShiftVS src shift));
17229   ins_cost(INSN_COST);
17230   format %{ "sshl  $dst,$src,$shift\t# vector (4H)" %}
17231   ins_encode %{
17232     __ sshl(as_FloatRegister($dst$$reg), __ T4H,
17233             as_FloatRegister($src$$reg),
17234             as_FloatRegister($shift$$reg));
17235   %}
17236   ins_pipe(vshift64);
17237 %}
17238 
17239 instruct vsll8S(vecX dst, vecX src, vecX shift) %{
17240   predicate(n->as_Vector()->length() == 8);
17241   match(Set dst (LShiftVS src shift));
17242   ins_cost(INSN_COST);
17243   format %{ "sshl  $dst,$src,$shift\t# vector (8H)" %}
17244   ins_encode %{
17245     __ sshl(as_FloatRegister($dst$$reg), __ T8H,
17246             as_FloatRegister($src$$reg),
17247             as_FloatRegister($shift$$reg));
17248   %}
17249   ins_pipe(vshift128);
17250 %}
17251 
17252 instruct vsra4S(vecD dst, vecD src, vecD shift, vecD tmp) %{
17253   predicate(n->as_Vector()->length() == 2 ||
17254             n->as_Vector()->length() == 4);
17255   match(Set dst (RShiftVS src shift));
17256   ins_cost(INSN_COST);
17257   effect(TEMP tmp);
17258   format %{ "negr  $tmp,$shift\t"
17259             "sshl  $dst,$src,$tmp\t# vector (4H)" %}
17260   ins_encode %{
17261     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
17262             as_FloatRegister($shift$$reg));
17263     __ sshl(as_FloatRegister($dst$$reg), __ T4H,
17264             as_FloatRegister($src$$reg),
17265             as_FloatRegister($tmp$$reg));
17266   %}
17267   ins_pipe(vshift64);
17268 %}
17269 
17270 instruct vsra8S(vecX dst, vecX src, vecX shift, vecX tmp) %{
17271   predicate(n->as_Vector()->length() == 8);
17272   match(Set dst (RShiftVS src shift));
17273   ins_cost(INSN_COST);
17274   effect(TEMP tmp);
17275   format %{ "negr  $tmp,$shift\t"
17276             "sshl  $dst,$src,$tmp\t# vector (8H)" %}
17277   ins_encode %{
17278     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17279             as_FloatRegister($shift$$reg));
17280     __ sshl(as_FloatRegister($dst$$reg), __ T8H,
17281             as_FloatRegister($src$$reg),
17282             as_FloatRegister($tmp$$reg));
17283   %}
17284   ins_pipe(vshift128);
17285 %}
17286 
17287 instruct vsrl4S(vecD dst, vecD src, vecD shift, vecD tmp) %{
17288   predicate(n->as_Vector()->length() == 2 ||
17289             n->as_Vector()->length() == 4);
17290   match(Set dst (URShiftVS src shift));
17291   ins_cost(INSN_COST);
17292   effect(TEMP tmp);
17293   format %{ "negr  $tmp,$shift\t"
17294             "ushl  $dst,$src,$tmp\t# vector (4H)" %}
17295   ins_encode %{
17296     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
17297             as_FloatRegister($shift$$reg));
17298     __ ushl(as_FloatRegister($dst$$reg), __ T4H,
17299             as_FloatRegister($src$$reg),
17300             as_FloatRegister($tmp$$reg));
17301   %}
17302   ins_pipe(vshift64);
17303 %}
17304 
17305 instruct vsrl8S(vecX dst, vecX src, vecX shift, vecX tmp) %{
17306   predicate(n->as_Vector()->length() == 8);
17307   match(Set dst (URShiftVS src shift));
17308   ins_cost(INSN_COST);
17309   effect(TEMP tmp);
17310   format %{ "negr  $tmp,$shift\t"
17311             "ushl  $dst,$src,$tmp\t# vector (8H)" %}
17312   ins_encode %{
17313     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17314             as_FloatRegister($shift$$reg));
17315     __ ushl(as_FloatRegister($dst$$reg), __ T8H,
17316             as_FloatRegister($src$$reg),
17317             as_FloatRegister($tmp$$reg));
17318   %}
17319   ins_pipe(vshift128);
17320 %}
17321 
17322 instruct vsll4S_imm(vecD dst, vecD src, immI shift) %{
17323   predicate(n->as_Vector()->length() == 2 ||
17324             n->as_Vector()->length() == 4);
17325   match(Set dst (LShiftVS src shift));
17326   ins_cost(INSN_COST);
17327   format %{ "shl    $dst, $src, $shift\t# vector (4H)" %}
17328   ins_encode %{
17329     int sh = (int)$shift$$constant;
17330     if (sh >= 16) {
17331       __ eor(as_FloatRegister($dst$$reg), __ T8B,
17332              as_FloatRegister($src$$reg),
17333              as_FloatRegister($src$$reg));
17334     } else {
17335       __ shl(as_FloatRegister($dst$$reg), __ T4H,
17336              as_FloatRegister($src$$reg), sh);
17337     }
17338   %}
17339   ins_pipe(vshift64_imm);
17340 %}
17341 
17342 instruct vsll8S_imm(vecX dst, vecX src, immI shift) %{
17343   predicate(n->as_Vector()->length() == 8);
17344   match(Set dst (LShiftVS src shift));
17345   ins_cost(INSN_COST);
17346   format %{ "shl    $dst, $src, $shift\t# vector (8H)" %}
17347   ins_encode %{
17348     int sh = (int)$shift$$constant;
17349     if (sh >= 16) {
17350       __ eor(as_FloatRegister($dst$$reg), __ T16B,
17351              as_FloatRegister($src$$reg),
17352              as_FloatRegister($src$$reg));
17353     } else {
17354       __ shl(as_FloatRegister($dst$$reg), __ T8H,
17355              as_FloatRegister($src$$reg), sh);
17356     }
17357   %}
17358   ins_pipe(vshift128_imm);
17359 %}
17360 
17361 instruct vsra4S_imm(vecD dst, vecD src, immI shift) %{
17362   predicate(n->as_Vector()->length() == 2 ||
17363             n->as_Vector()->length() == 4);
17364   match(Set dst (RShiftVS src shift));
17365   ins_cost(INSN_COST);
17366   format %{ "sshr    $dst, $src, $shift\t# vector (4H)" %}
17367   ins_encode %{
17368     int sh = (int)$shift$$constant;
17369     if (sh >= 16) sh = 15;
17370     __ sshr(as_FloatRegister($dst$$reg), __ T4H,
17371            as_FloatRegister($src$$reg), sh);
17372   %}
17373   ins_pipe(vshift64_imm);
17374 %}
17375 
17376 instruct vsra8S_imm(vecX dst, vecX src, immI shift) %{
17377   predicate(n->as_Vector()->length() == 8);
17378   match(Set dst (RShiftVS src shift));
17379   ins_cost(INSN_COST);
17380   format %{ "sshr    $dst, $src, $shift\t# vector (8H)" %}
17381   ins_encode %{
17382     int sh = (int)$shift$$constant;
17383     if (sh >= 16) sh = 15;
17384     __ sshr(as_FloatRegister($dst$$reg), __ T8H,
17385            as_FloatRegister($src$$reg), sh);
17386   %}
17387   ins_pipe(vshift128_imm);
17388 %}
17389 
17390 instruct vsrl4S_imm(vecD dst, vecD src, immI shift) %{
17391   predicate(n->as_Vector()->length() == 2 ||
17392             n->as_Vector()->length() == 4);
17393   match(Set dst (URShiftVS src shift));
17394   ins_cost(INSN_COST);
17395   format %{ "ushr    $dst, $src, $shift\t# vector (4H)" %}
17396   ins_encode %{
17397     int sh = (int)$shift$$constant;
17398     if (sh >= 16) {
17399       __ eor(as_FloatRegister($dst$$reg), __ T8B,
17400              as_FloatRegister($src$$reg),
17401              as_FloatRegister($src$$reg));
17402     } else {
17403       __ ushr(as_FloatRegister($dst$$reg), __ T4H,
17404              as_FloatRegister($src$$reg), sh);
17405     }
17406   %}
17407   ins_pipe(vshift64_imm);
17408 %}
17409 
17410 instruct vsrl8S_imm(vecX dst, vecX src, immI shift) %{
17411   predicate(n->as_Vector()->length() == 8);
17412   match(Set dst (URShiftVS src shift));
17413   ins_cost(INSN_COST);
17414   format %{ "ushr    $dst, $src, $shift\t# vector (8H)" %}
17415   ins_encode %{
17416     int sh = (int)$shift$$constant;
17417     if (sh >= 16) {
17418       __ eor(as_FloatRegister($dst$$reg), __ T16B,
17419              as_FloatRegister($src$$reg),
17420              as_FloatRegister($src$$reg));
17421     } else {
17422       __ ushr(as_FloatRegister($dst$$reg), __ T8H,
17423              as_FloatRegister($src$$reg), sh);
17424     }
17425   %}
17426   ins_pipe(vshift128_imm);
17427 %}
17428 
17429 instruct vsll2I(vecD dst, vecD src, vecD shift) %{
17430   predicate(n->as_Vector()->length() == 2);
17431   match(Set dst (LShiftVI src shift));
17432   ins_cost(INSN_COST);
17433   format %{ "sshl  $dst,$src,$shift\t# vector (2S)" %}
17434   ins_encode %{
17435     __ sshl(as_FloatRegister($dst$$reg), __ T2S,
17436             as_FloatRegister($src$$reg),
17437             as_FloatRegister($shift$$reg));
17438   %}
17439   ins_pipe(vshift64);
17440 %}
17441 
17442 instruct vsll4I(vecX dst, vecX src, vecX shift) %{
17443   predicate(n->as_Vector()->length() == 4);
17444   match(Set dst (LShiftVI src shift));
17445   ins_cost(INSN_COST);
17446   format %{ "sshl  $dst,$src,$shift\t# vector (4S)" %}
17447   ins_encode %{
17448     __ sshl(as_FloatRegister($dst$$reg), __ T4S,
17449             as_FloatRegister($src$$reg),
17450             as_FloatRegister($shift$$reg));
17451   %}
17452   ins_pipe(vshift128);
17453 %}
17454 
17455 instruct vsra2I(vecD dst, vecD src, vecD shift, vecD tmp) %{
17456   predicate(n->as_Vector()->length() == 2);
17457   match(Set dst (RShiftVI src shift));
17458   ins_cost(INSN_COST);
17459   effect(TEMP tmp);
17460   format %{ "negr  $tmp,$shift\t"
17461             "sshl  $dst,$src,$tmp\t# vector (2S)" %}
17462   ins_encode %{
17463     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
17464             as_FloatRegister($shift$$reg));
17465     __ sshl(as_FloatRegister($dst$$reg), __ T2S,
17466             as_FloatRegister($src$$reg),
17467             as_FloatRegister($tmp$$reg));
17468   %}
17469   ins_pipe(vshift64);
17470 %}
17471 
17472 instruct vsra4I(vecX dst, vecX src, vecX shift, vecX tmp) %{
17473   predicate(n->as_Vector()->length() == 4);
17474   match(Set dst (RShiftVI src shift));
17475   ins_cost(INSN_COST);
17476   effect(TEMP tmp);
17477   format %{ "negr  $tmp,$shift\t"
17478             "sshl  $dst,$src,$tmp\t# vector (4S)" %}
17479   ins_encode %{
17480     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17481             as_FloatRegister($shift$$reg));
17482     __ sshl(as_FloatRegister($dst$$reg), __ T4S,
17483             as_FloatRegister($src$$reg),
17484             as_FloatRegister($tmp$$reg));
17485   %}
17486   ins_pipe(vshift128);
17487 %}
17488 
17489 instruct vsrl2I(vecD dst, vecD src, vecD shift, vecD tmp) %{
17490   predicate(n->as_Vector()->length() == 2);
17491   match(Set dst (URShiftVI src shift));
17492   ins_cost(INSN_COST);
17493   effect(TEMP tmp);
17494   format %{ "negr  $tmp,$shift\t"
17495             "ushl  $dst,$src,$tmp\t# vector (2S)" %}
17496   ins_encode %{
17497     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
17498             as_FloatRegister($shift$$reg));
17499     __ ushl(as_FloatRegister($dst$$reg), __ T2S,
17500             as_FloatRegister($src$$reg),
17501             as_FloatRegister($tmp$$reg));
17502   %}
17503   ins_pipe(vshift64);
17504 %}
17505 
17506 instruct vsrl4I(vecX dst, vecX src, vecX shift, vecX tmp) %{
17507   predicate(n->as_Vector()->length() == 4);
17508   match(Set dst (URShiftVI src shift));
17509   ins_cost(INSN_COST);
17510   effect(TEMP tmp);
17511   format %{ "negr  $tmp,$shift\t"
17512             "ushl  $dst,$src,$tmp\t# vector (4S)" %}
17513   ins_encode %{
17514     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17515             as_FloatRegister($shift$$reg));
17516     __ ushl(as_FloatRegister($dst$$reg), __ T4S,
17517             as_FloatRegister($src$$reg),
17518             as_FloatRegister($tmp$$reg));
17519   %}
17520   ins_pipe(vshift128);
17521 %}
17522 
17523 instruct vsll2I_imm(vecD dst, vecD src, immI shift) %{
17524   predicate(n->as_Vector()->length() == 2);
17525   match(Set dst (LShiftVI src shift));
17526   ins_cost(INSN_COST);
17527   format %{ "shl    $dst, $src, $shift\t# vector (2S)" %}
17528   ins_encode %{
17529     __ shl(as_FloatRegister($dst$$reg), __ T2S,
17530            as_FloatRegister($src$$reg),
17531            (int)$shift$$constant);
17532   %}
17533   ins_pipe(vshift64_imm);
17534 %}
17535 
17536 instruct vsll4I_imm(vecX dst, vecX src, immI shift) %{
17537   predicate(n->as_Vector()->length() == 4);
17538   match(Set dst (LShiftVI src shift));
17539   ins_cost(INSN_COST);
17540   format %{ "shl    $dst, $src, $shift\t# vector (4S)" %}
17541   ins_encode %{
17542     __ shl(as_FloatRegister($dst$$reg), __ T4S,
17543            as_FloatRegister($src$$reg),
17544            (int)$shift$$constant);
17545   %}
17546   ins_pipe(vshift128_imm);
17547 %}
17548 
17549 instruct vsra2I_imm(vecD dst, vecD src, immI shift) %{
17550   predicate(n->as_Vector()->length() == 2);
17551   match(Set dst (RShiftVI src shift));
17552   ins_cost(INSN_COST);
17553   format %{ "sshr    $dst, $src, $shift\t# vector (2S)" %}
17554   ins_encode %{
17555     __ sshr(as_FloatRegister($dst$$reg), __ T2S,
17556             as_FloatRegister($src$$reg),
17557             (int)$shift$$constant);
17558   %}
17559   ins_pipe(vshift64_imm);
17560 %}
17561 
17562 instruct vsra4I_imm(vecX dst, vecX src, immI shift) %{
17563   predicate(n->as_Vector()->length() == 4);
17564   match(Set dst (RShiftVI src shift));
17565   ins_cost(INSN_COST);
17566   format %{ "sshr    $dst, $src, $shift\t# vector (4S)" %}
17567   ins_encode %{
17568     __ sshr(as_FloatRegister($dst$$reg), __ T4S,
17569             as_FloatRegister($src$$reg),
17570             (int)$shift$$constant);
17571   %}
17572   ins_pipe(vshift128_imm);
17573 %}
17574 
17575 instruct vsrl2I_imm(vecD dst, vecD src, immI shift) %{
17576   predicate(n->as_Vector()->length() == 2);
17577   match(Set dst (URShiftVI src shift));
17578   ins_cost(INSN_COST);
17579   format %{ "ushr    $dst, $src, $shift\t# vector (2S)" %}
17580   ins_encode %{
17581     __ ushr(as_FloatRegister($dst$$reg), __ T2S,
17582             as_FloatRegister($src$$reg),
17583             (int)$shift$$constant);
17584   %}
17585   ins_pipe(vshift64_imm);
17586 %}
17587 
17588 instruct vsrl4I_imm(vecX dst, vecX src, immI shift) %{
17589   predicate(n->as_Vector()->length() == 4);
17590   match(Set dst (URShiftVI src shift));
17591   ins_cost(INSN_COST);
17592   format %{ "ushr    $dst, $src, $shift\t# vector (4S)" %}
17593   ins_encode %{
17594     __ ushr(as_FloatRegister($dst$$reg), __ T4S,
17595             as_FloatRegister($src$$reg),
17596             (int)$shift$$constant);
17597   %}
17598   ins_pipe(vshift128_imm);
17599 %}
17600 
17601 instruct vsll2L(vecX dst, vecX src, vecX shift) %{
17602   predicate(n->as_Vector()->length() == 2);
17603   match(Set dst (LShiftVL src shift));
17604   ins_cost(INSN_COST);
17605   format %{ "sshl  $dst,$src,$shift\t# vector (2D)" %}
17606   ins_encode %{
17607     __ sshl(as_FloatRegister($dst$$reg), __ T2D,
17608             as_FloatRegister($src$$reg),
17609             as_FloatRegister($shift$$reg));
17610   %}
17611   ins_pipe(vshift128);
17612 %}
17613 
17614 instruct vsra2L(vecX dst, vecX src, vecX shift, vecX tmp) %{
17615   predicate(n->as_Vector()->length() == 2);
17616   match(Set dst (RShiftVL src shift));
17617   ins_cost(INSN_COST);
17618   effect(TEMP tmp);
17619   format %{ "negr  $tmp,$shift\t"
17620             "sshl  $dst,$src,$tmp\t# vector (2D)" %}
17621   ins_encode %{
17622     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17623             as_FloatRegister($shift$$reg));
17624     __ sshl(as_FloatRegister($dst$$reg), __ T2D,
17625             as_FloatRegister($src$$reg),
17626             as_FloatRegister($tmp$$reg));
17627   %}
17628   ins_pipe(vshift128);
17629 %}
17630 
17631 instruct vsrl2L(vecX dst, vecX src, vecX shift, vecX tmp) %{
17632   predicate(n->as_Vector()->length() == 2);
17633   match(Set dst (URShiftVL src shift));
17634   ins_cost(INSN_COST);
17635   effect(TEMP tmp);
17636   format %{ "negr  $tmp,$shift\t"
17637             "ushl  $dst,$src,$tmp\t# vector (2D)" %}
17638   ins_encode %{
17639     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17640             as_FloatRegister($shift$$reg));
17641     __ ushl(as_FloatRegister($dst$$reg), __ T2D,
17642             as_FloatRegister($src$$reg),
17643             as_FloatRegister($tmp$$reg));
17644   %}
17645   ins_pipe(vshift128);
17646 %}
17647 
17648 instruct vsll2L_imm(vecX dst, vecX src, immI shift) %{
17649   predicate(n->as_Vector()->length() == 2);
17650   match(Set dst (LShiftVL src shift));
17651   ins_cost(INSN_COST);
17652   format %{ "shl    $dst, $src, $shift\t# vector (2D)" %}
17653   ins_encode %{
17654     __ shl(as_FloatRegister($dst$$reg), __ T2D,
17655            as_FloatRegister($src$$reg),
17656            (int)$shift$$constant);
17657   %}
17658   ins_pipe(vshift128_imm);
17659 %}
17660 
17661 instruct vsra2L_imm(vecX dst, vecX src, immI shift) %{
17662   predicate(n->as_Vector()->length() == 2);
17663   match(Set dst (RShiftVL src shift));
17664   ins_cost(INSN_COST);
17665   format %{ "sshr    $dst, $src, $shift\t# vector (2D)" %}
17666   ins_encode %{
17667     __ sshr(as_FloatRegister($dst$$reg), __ T2D,
17668             as_FloatRegister($src$$reg),
17669             (int)$shift$$constant);
17670   %}
17671   ins_pipe(vshift128_imm);
17672 %}
17673 
17674 instruct vsrl2L_imm(vecX dst, vecX src, immI shift) %{
17675   predicate(n->as_Vector()->length() == 2);
17676   match(Set dst (URShiftVL src shift));
17677   ins_cost(INSN_COST);
17678   format %{ "ushr    $dst, $src, $shift\t# vector (2D)" %}
17679   ins_encode %{
17680     __ ushr(as_FloatRegister($dst$$reg), __ T2D,
17681             as_FloatRegister($src$$reg),
17682             (int)$shift$$constant);
17683   %}
17684   ins_pipe(vshift128_imm);
17685 %}
17686 
17687 instruct vmax2F(vecD dst, vecD src1, vecD src2)
17688 %{
17689   predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
17690   match(Set dst (MaxV src1 src2));
17691   ins_cost(INSN_COST);
17692   format %{ "fmax  $dst,$src1,$src2\t# vector (2F)" %}
17693   ins_encode %{
17694     __ fmax(as_FloatRegister($dst$$reg), __ T2S,
17695             as_FloatRegister($src1$$reg),
17696             as_FloatRegister($src2$$reg));
17697   %}
17698   ins_pipe(vdop_fp64);
17699 %}
17700 
17701 instruct vmax4F(vecX dst, vecX src1, vecX src2)
17702 %{
17703   predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
17704   match(Set dst (MaxV src1 src2));
17705   ins_cost(INSN_COST);
17706   format %{ "fmax  $dst,$src1,$src2\t# vector (4S)" %}
17707   ins_encode %{
17708     __ fmax(as_FloatRegister($dst$$reg), __ T4S,
17709             as_FloatRegister($src1$$reg),
17710             as_FloatRegister($src2$$reg));
17711   %}
17712   ins_pipe(vdop_fp128);
17713 %}
17714 
17715 instruct vmax2D(vecX dst, vecX src1, vecX src2)
17716 %{
17717   predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
17718   match(Set dst (MaxV src1 src2));
17719   ins_cost(INSN_COST);
17720   format %{ "fmax  $dst,$src1,$src2\t# vector (2D)" %}
17721   ins_encode %{
17722     __ fmax(as_FloatRegister($dst$$reg), __ T2D,
17723             as_FloatRegister($src1$$reg),
17724             as_FloatRegister($src2$$reg));
17725   %}
17726   ins_pipe(vdop_fp128);
17727 %}
17728 
17729 instruct vmin2F(vecD dst, vecD src1, vecD src2)
17730 %{
17731   predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
17732   match(Set dst (MinV src1 src2));
17733   ins_cost(INSN_COST);
17734   format %{ "fmin  $dst,$src1,$src2\t# vector (2F)" %}
17735   ins_encode %{
17736     __ fmin(as_FloatRegister($dst$$reg), __ T2S,
17737             as_FloatRegister($src1$$reg),
17738             as_FloatRegister($src2$$reg));
17739   %}
17740   ins_pipe(vdop_fp64);
17741 %}
17742 
17743 instruct vmin4F(vecX dst, vecX src1, vecX src2)
17744 %{
17745   predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
17746   match(Set dst (MinV src1 src2));
17747   ins_cost(INSN_COST);
17748   format %{ "fmin  $dst,$src1,$src2\t# vector (4S)" %}
17749   ins_encode %{
17750     __ fmin(as_FloatRegister($dst$$reg), __ T4S,
17751             as_FloatRegister($src1$$reg),
17752             as_FloatRegister($src2$$reg));
17753   %}
17754   ins_pipe(vdop_fp128);
17755 %}
17756 
17757 instruct vmin2D(vecX dst, vecX src1, vecX src2)
17758 %{
17759   predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
17760   match(Set dst (MinV src1 src2));
17761   ins_cost(INSN_COST);
17762   format %{ "fmin  $dst,$src1,$src2\t# vector (2D)" %}
17763   ins_encode %{
17764     __ fmin(as_FloatRegister($dst$$reg), __ T2D,
17765             as_FloatRegister($src1$$reg),
17766             as_FloatRegister($src2$$reg));
17767   %}
17768   ins_pipe(vdop_fp128);
17769 %}
17770 
17771 //----------PEEPHOLE RULES-----------------------------------------------------
17772 // These must follow all instruction definitions as they use the names
17773 // defined in the instructions definitions.
17774 //
17775 // peepmatch ( root_instr_name [preceding_instruction]* );
17776 //
17777 // peepconstraint %{
17778 // (instruction_number.operand_name relational_op instruction_number.operand_name
17779 //  [, ...] );
17780 // // instruction numbers are zero-based using left to right order in peepmatch
17781 //
17782 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17783 // // provide an instruction_number.operand_name for each operand that appears
17784 // // in the replacement instruction's match rule
17785 //
17786 // ---------VM FLAGS---------------------------------------------------------
17787 //
17788 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17789 //
17790 // Each peephole rule is given an identifying number starting with zero and
17791 // increasing by one in the order seen by the parser.  An individual peephole
17792 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17793 // on the command-line.
17794 //
17795 // ---------CURRENT LIMITATIONS----------------------------------------------
17796 //
17797 // Only match adjacent instructions in same basic block
17798 // Only equality constraints
17799 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17800 // Only one replacement instruction
17801 //
17802 // ---------EXAMPLE----------------------------------------------------------
17803 //
17804 // // pertinent parts of existing instructions in architecture description
17805 // instruct movI(iRegINoSp dst, iRegI src)
17806 // %{
17807 //   match(Set dst (CopyI src));
17808 // %}
17809 //
17810 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17811 // %{
17812 //   match(Set dst (AddI dst src));
17813 //   effect(KILL cr);
17814 // %}
17815 //
17816 // // Change (inc mov) to lea
17817 // peephole %{
17818 //   // increment preceeded by register-register move
17819 //   peepmatch ( incI_iReg movI );
17820 //   // require that the destination register of the increment
17821 //   // match the destination register of the move
17822 //   peepconstraint ( 0.dst == 1.dst );
17823 //   // construct a replacement instruction that sets
17824 //   // the destination to ( move's source register + one )
17825 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17826 // %}
17827 //
17828 
17829 // Implementation no longer uses movX instructions since
17830 // machine-independent system no longer uses CopyX nodes.
17831 //
17832 // peephole
17833 // %{
17834 //   peepmatch (incI_iReg movI);
17835 //   peepconstraint (0.dst == 1.dst);
17836 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17837 // %}
17838 
17839 // peephole
17840 // %{
17841 //   peepmatch (decI_iReg movI);
17842 //   peepconstraint (0.dst == 1.dst);
17843 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17844 // %}
17845 
17846 // peephole
17847 // %{
17848 //   peepmatch (addI_iReg_imm movI);
17849 //   peepconstraint (0.dst == 1.dst);
17850 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17851 // %}
17852 
17853 // peephole
17854 // %{
17855 //   peepmatch (incL_iReg movL);
17856 //   peepconstraint (0.dst == 1.dst);
17857 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17858 // %}
17859 
17860 // peephole
17861 // %{
17862 //   peepmatch (decL_iReg movL);
17863 //   peepconstraint (0.dst == 1.dst);
17864 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17865 // %}
17866 
17867 // peephole
17868 // %{
17869 //   peepmatch (addL_iReg_imm movL);
17870 //   peepconstraint (0.dst == 1.dst);
17871 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17872 // %}
17873 
17874 // peephole
17875 // %{
17876 //   peepmatch (addP_iReg_imm movP);
17877 //   peepconstraint (0.dst == 1.dst);
17878 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17879 // %}
17880 
17881 // // Change load of spilled value to only a spill
17882 // instruct storeI(memory mem, iRegI src)
17883 // %{
17884 //   match(Set mem (StoreI mem src));
17885 // %}
17886 //
17887 // instruct loadI(iRegINoSp dst, memory mem)
17888 // %{
17889 //   match(Set dst (LoadI mem));
17890 // %}
17891 //
17892 
17893 //----------SMARTSPILL RULES---------------------------------------------------
17894 // These must follow all instruction definitions as they use the names
17895 // defined in the instructions definitions.
17896 
17897 // Local Variables:
17898 // mode: c++
17899 // End: