1 //
   2 // Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
   3 // Copyright (c) 2014, 2019, Red Hat, Inc. All rights reserved.
   4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5 //
   6 // This code is free software; you can redistribute it and/or modify it
   7 // under the terms of the GNU General Public License version 2 only, as
   8 // published by the Free Software Foundation.
   9 //
  10 // This code is distributed in the hope that it will be useful, but WITHOUT
  11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 // version 2 for more details (a copy is included in the LICENSE file that
  14 // accompanied this code).
  15 //
  16 // You should have received a copy of the GNU General Public License version
  17 // 2 along with this work; if not, write to the Free Software Foundation,
  18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19 //
  20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21 // or visit www.oracle.com if you need additional information or have any
  22 // questions.
  23 //
  24 //
  25 
  26 // AArch64 Architecture Description File
  27 
  28 //----------REGISTER DEFINITION BLOCK------------------------------------------
  29 // This information is used by the matcher and the register allocator to
  30 // describe individual registers and classes of registers within the target
  31 // archtecture.
  32 
  33 register %{
  34 //----------Architecture Description Register Definitions----------------------
  35 // General Registers
  36 // "reg_def"  name ( register save type, C convention save type,
  37 //                   ideal register type, encoding );
  38 // Register Save Types:
  39 //
  40 // NS  = No-Save:       The register allocator assumes that these registers
  41 //                      can be used without saving upon entry to the method, &
  42 //                      that they do not need to be saved at call sites.
  43 //
  44 // SOC = Save-On-Call:  The register allocator assumes that these registers
  45 //                      can be used without saving upon entry to the method,
  46 //                      but that they must be saved at call sites.
  47 //
  48 // SOE = Save-On-Entry: The register allocator assumes that these registers
  49 //                      must be saved before using them upon entry to the
  50 //                      method, but they do not need to be saved at call
  51 //                      sites.
  52 //
  53 // AS  = Always-Save:   The register allocator assumes that these registers
  54 //                      must be saved before using them upon entry to the
  55 //                      method, & that they must be saved at call sites.
  56 //
  57 // Ideal Register Type is used to determine how to save & restore a
  58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  60 //
  61 // The encoding number is the actual bit-pattern placed into the opcodes.
  62 
  63 // We must define the 64 bit int registers in two 32 bit halves, the
  64 // real lower register and a virtual upper half register. upper halves
  65 // are used by the register allocator but are not actually supplied as
  66 // operands to memory ops.
  67 //
  68 // follow the C1 compiler in making registers
  69 //
  70 //   r0-r7,r10-r26 volatile (caller save)
  71 //   r27-r32 system (no save, no allocate)
  72 //   r8-r9 invisible to the allocator (so we can use them as scratch regs)
  73 //
  74 // as regards Java usage. we don't use any callee save registers
  75 // because this makes it difficult to de-optimise a frame (see comment
  76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
  77 //
  78 
  79 // General Registers
  80 
  81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
  82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
  83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
  84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
  85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
  86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
  87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
  88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
  89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
  90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
  91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
  92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
  93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
  94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
  95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
  96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
  97 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  98 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  99 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
 100 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 101 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
 102 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
 103 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
 104 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
 105 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
 106 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
 107 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
 108 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
 109 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
 110 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
 111 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
 112 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
 113 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18->as_VMReg()        );
 114 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
 115 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
 116 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
 117 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
 118 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
 119 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
 120 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
 121 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
 122 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
 123 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
 124 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
 125 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
 126 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
 127 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
 128 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
 129 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
 130 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
 131 reg_def R27     (  NS, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
 132 reg_def R27_H   (  NS, SOE, Op_RegI, 27, r27->as_VMReg()->next());
 133 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
 134 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
 135 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
 136 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
 137 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
 138 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
 139 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
 140 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
 141 
 142 // ----------------------------
 143 // Float/Double Registers
 144 // ----------------------------
 145 
 146 // Double Registers
 147 
 148 // The rules of ADL require that double registers be defined in pairs.
 149 // Each pair must be two 32-bit values, but not necessarily a pair of
 150 // single float registers. In each pair, ADLC-assigned register numbers
 151 // must be adjacent, with the lower number even. Finally, when the
 152 // CPU stores such a register pair to memory, the word associated with
 153 // the lower ADLC-assigned number must be stored to the lower address.
 154 
 155 // AArch64 has 32 floating-point registers. Each can store a vector of
 156 // single or double precision floating-point values up to 8 * 32
 157 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
 158 // use the first float or double element of the vector.
 159 
 160 // for Java use float registers v0-v15 are always save on call whereas
 161 // the platform ABI treats v8-v15 as callee save). float registers
 162 // v16-v31 are SOC as per the platform spec
 163 
 164   reg_def V0   ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()          );
 165   reg_def V0_H ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()->next()  );
 166   reg_def V0_J ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()->next(2) );
 167   reg_def V0_K ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()->next(3) );
 168 
 169   reg_def V1   ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()          );
 170   reg_def V1_H ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()->next()  );
 171   reg_def V1_J ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()->next(2) );
 172   reg_def V1_K ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()->next(3) );
 173 
 174   reg_def V2   ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()          );
 175   reg_def V2_H ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()->next()  );
 176   reg_def V2_J ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()->next(2) );
 177   reg_def V2_K ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()->next(3) );
 178 
 179   reg_def V3   ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()          );
 180   reg_def V3_H ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()->next()  );
 181   reg_def V3_J ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()->next(2) );
 182   reg_def V3_K ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()->next(3) );
 183 
 184   reg_def V4   ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()          );
 185   reg_def V4_H ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()->next()  );
 186   reg_def V4_J ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()->next(2) );
 187   reg_def V4_K ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()->next(3) );
 188 
 189   reg_def V5   ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()          );
 190   reg_def V5_H ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()->next()  );
 191   reg_def V5_J ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()->next(2) );
 192   reg_def V5_K ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()->next(3) );
 193 
 194   reg_def V6   ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()          );
 195   reg_def V6_H ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()->next()  );
 196   reg_def V6_J ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()->next(2) );
 197   reg_def V6_K ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()->next(3) );
 198 
 199   reg_def V7   ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()          );
 200   reg_def V7_H ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()->next()  );
 201   reg_def V7_J ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()->next(2) );
 202   reg_def V7_K ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()->next(3) );
 203 
 204   reg_def V8   ( SOC, SOC, Op_RegF,  8, v8->as_VMReg()          );
 205   reg_def V8_H ( SOC, SOC, Op_RegF,  8, v8->as_VMReg()->next()  );
 206   reg_def V8_J ( SOC, SOC, Op_RegF,  8, v8->as_VMReg()->next(2) );
 207   reg_def V8_K ( SOC, SOC, Op_RegF,  8, v8->as_VMReg()->next(3) );
 208 
 209   reg_def V9   ( SOC, SOC, Op_RegF,  9, v9->as_VMReg()          );
 210   reg_def V9_H ( SOC, SOC, Op_RegF,  9, v9->as_VMReg()->next()  );
 211   reg_def V9_J ( SOC, SOC, Op_RegF,  9, v9->as_VMReg()->next(2) );
 212   reg_def V9_K ( SOC, SOC, Op_RegF,  9, v9->as_VMReg()->next(3) );
 213 
 214   reg_def V10  ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()         );
 215   reg_def V10_H( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next() );
 216   reg_def V10_J( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2));
 217   reg_def V10_K( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3));
 218 
 219   reg_def V11  ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()         );
 220   reg_def V11_H( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next() );
 221   reg_def V11_J( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2));
 222   reg_def V11_K( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3));
 223 
 224   reg_def V12  ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()         );
 225   reg_def V12_H( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next() );
 226   reg_def V12_J( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2));
 227   reg_def V12_K( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3));
 228 
 229   reg_def V13  ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()         );
 230   reg_def V13_H( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next() );
 231   reg_def V13_J( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2));
 232   reg_def V13_K( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3));
 233 
 234   reg_def V14  ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()         );
 235   reg_def V14_H( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next() );
 236   reg_def V14_J( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2));
 237   reg_def V14_K( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3));
 238 
 239   reg_def V15  ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()         );
 240   reg_def V15_H( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next() );
 241   reg_def V15_J( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2));
 242   reg_def V15_K( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3));
 243 
 244   reg_def V16  ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()         );
 245   reg_def V16_H( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
 246   reg_def V16_J( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2));
 247   reg_def V16_K( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3));
 248 
 249   reg_def V17  ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()         );
 250   reg_def V17_H( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
 251   reg_def V17_J( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2));
 252   reg_def V17_K( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3));
 253 
 254   reg_def V18  ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()         );
 255   reg_def V18_H( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
 256   reg_def V18_J( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2));
 257   reg_def V18_K( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3));
 258 
 259   reg_def V19  ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()         );
 260   reg_def V19_H( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
 261   reg_def V19_J( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2));
 262   reg_def V19_K( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3));
 263 
 264   reg_def V20  ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()         );
 265   reg_def V20_H( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
 266   reg_def V20_J( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2));
 267   reg_def V20_K( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3));
 268 
 269   reg_def V21  ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()         );
 270   reg_def V21_H( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
 271   reg_def V21_J( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2));
 272   reg_def V21_K( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3));
 273 
 274   reg_def V22  ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()         );
 275   reg_def V22_H( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
 276   reg_def V22_J( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2));
 277   reg_def V22_K( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3));
 278 
 279   reg_def V23  ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()         );
 280   reg_def V23_H( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
 281   reg_def V23_J( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2));
 282   reg_def V23_K( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3));
 283 
 284   reg_def V24  ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()         );
 285   reg_def V24_H( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
 286   reg_def V24_J( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2));
 287   reg_def V24_K( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3));
 288 
 289   reg_def V25  ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()         );
 290   reg_def V25_H( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
 291   reg_def V25_J( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2));
 292   reg_def V25_K( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3));
 293 
 294   reg_def V26  ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()         );
 295   reg_def V26_H( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
 296   reg_def V26_J( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2));
 297   reg_def V26_K( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3));
 298 
 299   reg_def V27  ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()         );
 300   reg_def V27_H( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
 301   reg_def V27_J( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2));
 302   reg_def V27_K( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3));
 303 
 304   reg_def V28  ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()         );
 305   reg_def V28_H( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
 306   reg_def V28_J( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2));
 307   reg_def V28_K( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3));
 308 
 309   reg_def V29  ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()         );
 310   reg_def V29_H( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
 311   reg_def V29_J( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2));
 312   reg_def V29_K( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3));
 313 
 314   reg_def V30  ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()         );
 315   reg_def V30_H( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
 316   reg_def V30_J( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2));
 317   reg_def V30_K( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3));
 318 
 319   reg_def V31  ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()         );
 320   reg_def V31_H( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
 321   reg_def V31_J( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2));
 322   reg_def V31_K( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3));
 323 
 324 // ----------------------------
 325 // Special Registers
 326 // ----------------------------
 327 
 328 // the AArch64 CSPR status flag register is not directly acessible as
 329 // instruction operand. the FPSR status flag register is a system
 330 // register which can be written/read using MSR/MRS but again does not
 331 // appear as an operand (a code identifying the FSPR occurs as an
 332 // immediate value in the instruction).
 333 
 334 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
 335 
 336 
 337 // Specify priority of register selection within phases of register
 338 // allocation.  Highest priority is first.  A useful heuristic is to
 339 // give registers a low priority when they are required by machine
 340 // instructions, like EAX and EDX on I486, and choose no-save registers
 341 // before save-on-call, & save-on-call before save-on-entry.  Registers
 342 // which participate in fixed calling sequences should come last.
 343 // Registers which are used as pairs must fall on an even boundary.
 344 
 345 alloc_class chunk0(
 346     // volatiles
 347     R10, R10_H,
 348     R11, R11_H,
 349     R12, R12_H,
 350     R13, R13_H,
 351     R14, R14_H,
 352     R15, R15_H,
 353     R16, R16_H,
 354     R17, R17_H,
 355     R18, R18_H,
 356 
 357     // arg registers
 358     R0, R0_H,
 359     R1, R1_H,
 360     R2, R2_H,
 361     R3, R3_H,
 362     R4, R4_H,
 363     R5, R5_H,
 364     R6, R6_H,
 365     R7, R7_H,
 366 
 367     // non-volatiles
 368     R19, R19_H,
 369     R20, R20_H,
 370     R21, R21_H,
 371     R22, R22_H,
 372     R23, R23_H,
 373     R24, R24_H,
 374     R25, R25_H,
 375     R26, R26_H,
 376 
 377     // non-allocatable registers
 378 
 379     R27, R27_H, // heapbase
 380     R28, R28_H, // thread
 381     R29, R29_H, // fp
 382     R30, R30_H, // lr
 383     R31, R31_H, // sp
 384 );
 385 
 386 alloc_class chunk1(
 387 
 388     // no save
 389     V16, V16_H, V16_J, V16_K,
 390     V17, V17_H, V17_J, V17_K,
 391     V18, V18_H, V18_J, V18_K,
 392     V19, V19_H, V19_J, V19_K,
 393     V20, V20_H, V20_J, V20_K,
 394     V21, V21_H, V21_J, V21_K,
 395     V22, V22_H, V22_J, V22_K,
 396     V23, V23_H, V23_J, V23_K,
 397     V24, V24_H, V24_J, V24_K,
 398     V25, V25_H, V25_J, V25_K,
 399     V26, V26_H, V26_J, V26_K,
 400     V27, V27_H, V27_J, V27_K,
 401     V28, V28_H, V28_J, V28_K,
 402     V29, V29_H, V29_J, V29_K,
 403     V30, V30_H, V30_J, V30_K,
 404     V31, V31_H, V31_J, V31_K,
 405 
 406     // arg registers
 407     V0, V0_H, V0_J, V0_K,
 408     V1, V1_H, V1_J, V1_K,
 409     V2, V2_H, V2_J, V2_K,
 410     V3, V3_H, V3_J, V3_K,
 411     V4, V4_H, V4_J, V4_K,
 412     V5, V5_H, V5_J, V5_K,
 413     V6, V6_H, V6_J, V6_K,
 414     V7, V7_H, V7_J, V7_K,
 415 
 416     // non-volatiles
 417     V8, V8_H, V8_J, V8_K,
 418     V9, V9_H, V9_J, V9_K,
 419     V10, V10_H, V10_J, V10_K,
 420     V11, V11_H, V11_J, V11_K,
 421     V12, V12_H, V12_J, V12_K,
 422     V13, V13_H, V13_J, V13_K,
 423     V14, V14_H, V14_J, V14_K,
 424     V15, V15_H, V15_J, V15_K,
 425 );
 426 
 427 alloc_class chunk2(RFLAGS);
 428 
 429 //----------Architecture Description Register Classes--------------------------
 430 // Several register classes are automatically defined based upon information in
 431 // this architecture description.
 432 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 433 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 434 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 435 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 436 //
 437 
 438 // Class for all 32 bit integer registers -- excludes SP which will
 439 // never be used as an integer register
 440 reg_class any_reg32(
 441     R0,
 442     R1,
 443     R2,
 444     R3,
 445     R4,
 446     R5,
 447     R6,
 448     R7,
 449     R10,
 450     R11,
 451     R12,
 452     R13,
 453     R14,
 454     R15,
 455     R16,
 456     R17,
 457     R18,
 458     R19,
 459     R20,
 460     R21,
 461     R22,
 462     R23,
 463     R24,
 464     R25,
 465     R26,
 466     R27,
 467     R28,
 468     R29,
 469     R30
 470 );
 471 
 472 // Singleton class for R0 int register
 473 reg_class int_r0_reg(R0);
 474 
 475 // Singleton class for R2 int register
 476 reg_class int_r2_reg(R2);
 477 
 478 // Singleton class for R3 int register
 479 reg_class int_r3_reg(R3);
 480 
 481 // Singleton class for R4 int register
 482 reg_class int_r4_reg(R4);
 483 
 484 // Class for all long integer registers (including RSP)
 485 reg_class any_reg(
 486     R0, R0_H,
 487     R1, R1_H,
 488     R2, R2_H,
 489     R3, R3_H,
 490     R4, R4_H,
 491     R5, R5_H,
 492     R6, R6_H,
 493     R7, R7_H,
 494     R10, R10_H,
 495     R11, R11_H,
 496     R12, R12_H,
 497     R13, R13_H,
 498     R14, R14_H,
 499     R15, R15_H,
 500     R16, R16_H,
 501     R17, R17_H,
 502     R18, R18_H,
 503     R19, R19_H,
 504     R20, R20_H,
 505     R21, R21_H,
 506     R22, R22_H,
 507     R23, R23_H,
 508     R24, R24_H,
 509     R25, R25_H,
 510     R26, R26_H,
 511     R27, R27_H,
 512     R28, R28_H,
 513     R29, R29_H,
 514     R30, R30_H,
 515     R31, R31_H
 516 );
 517 
 518 // Class for all non-special integer registers
 519 reg_class no_special_reg32_no_fp(
 520     R0,
 521     R1,
 522     R2,
 523     R3,
 524     R4,
 525     R5,
 526     R6,
 527     R7,
 528     R10,
 529     R11,
 530     R12,                        // rmethod
 531     R13,
 532     R14,
 533     R15,
 534     R16,
 535     R17,
 536     R18,
 537     R19,
 538     R20,
 539     R21,
 540     R22,
 541     R23,
 542     R24,
 543     R25,
 544     R26
 545  /* R27, */                     // heapbase
 546  /* R28, */                     // thread
 547  /* R29, */                     // fp
 548  /* R30, */                     // lr
 549  /* R31 */                      // sp
 550 );
 551 
 552 reg_class no_special_reg32_with_fp(
 553     R0,
 554     R1,
 555     R2,
 556     R3,
 557     R4,
 558     R5,
 559     R6,
 560     R7,
 561     R10,
 562     R11,
 563     R12,                        // rmethod
 564     R13,
 565     R14,
 566     R15,
 567     R16,
 568     R17,
 569     R18,
 570     R19,
 571     R20,
 572     R21,
 573     R22,
 574     R23,
 575     R24,
 576     R25,
 577     R26
 578  /* R27, */                     // heapbase
 579  /* R28, */                     // thread
 580     R29,                        // fp
 581  /* R30, */                     // lr
 582  /* R31 */                      // sp
 583 );
 584 
 585 reg_class_dynamic no_special_reg32(no_special_reg32_no_fp, no_special_reg32_with_fp, %{ PreserveFramePointer %});
 586 
 587 // Class for all non-special long integer registers
 588 reg_class no_special_reg_no_fp(
 589     R0, R0_H,
 590     R1, R1_H,
 591     R2, R2_H,
 592     R3, R3_H,
 593     R4, R4_H,
 594     R5, R5_H,
 595     R6, R6_H,
 596     R7, R7_H,
 597     R10, R10_H,
 598     R11, R11_H,
 599     R12, R12_H,                 // rmethod
 600     R13, R13_H,
 601     R14, R14_H,
 602     R15, R15_H,
 603     R16, R16_H,
 604     R17, R17_H,
 605     R18, R18_H,
 606     R19, R19_H,
 607     R20, R20_H,
 608     R21, R21_H,
 609     R22, R22_H,
 610     R23, R23_H,
 611     R24, R24_H,
 612     R25, R25_H,
 613     R26, R26_H,
 614  /* R27, R27_H, */              // heapbase
 615  /* R28, R28_H, */              // thread
 616  /* R29, R29_H, */              // fp
 617  /* R30, R30_H, */              // lr
 618  /* R31, R31_H */               // sp
 619 );
 620 
 621 reg_class no_special_reg_with_fp(
 622     R0, R0_H,
 623     R1, R1_H,
 624     R2, R2_H,
 625     R3, R3_H,
 626     R4, R4_H,
 627     R5, R5_H,
 628     R6, R6_H,
 629     R7, R7_H,
 630     R10, R10_H,
 631     R11, R11_H,
 632     R12, R12_H,                 // rmethod
 633     R13, R13_H,
 634     R14, R14_H,
 635     R15, R15_H,
 636     R16, R16_H,
 637     R17, R17_H,
 638     R18, R18_H,
 639     R19, R19_H,
 640     R20, R20_H,
 641     R21, R21_H,
 642     R22, R22_H,
 643     R23, R23_H,
 644     R24, R24_H,
 645     R25, R25_H,
 646     R26, R26_H,
 647  /* R27, R27_H, */              // heapbase
 648  /* R28, R28_H, */              // thread
 649     R29, R29_H,                 // fp
 650  /* R30, R30_H, */              // lr
 651  /* R31, R31_H */               // sp
 652 );
 653 
 654 reg_class_dynamic no_special_reg(no_special_reg_no_fp, no_special_reg_with_fp, %{ PreserveFramePointer %});
 655 
 656 // Class for 64 bit register r0
 657 reg_class r0_reg(
 658     R0, R0_H
 659 );
 660 
 661 // Class for 64 bit register r1
 662 reg_class r1_reg(
 663     R1, R1_H
 664 );
 665 
 666 // Class for 64 bit register r2
 667 reg_class r2_reg(
 668     R2, R2_H
 669 );
 670 
 671 // Class for 64 bit register r3
 672 reg_class r3_reg(
 673     R3, R3_H
 674 );
 675 
 676 // Class for 64 bit register r4
 677 reg_class r4_reg(
 678     R4, R4_H
 679 );
 680 
 681 // Class for 64 bit register r5
 682 reg_class r5_reg(
 683     R5, R5_H
 684 );
 685 
 686 // Class for 64 bit register r10
 687 reg_class r10_reg(
 688     R10, R10_H
 689 );
 690 
 691 // Class for 64 bit register r11
 692 reg_class r11_reg(
 693     R11, R11_H
 694 );
 695 
 696 // Class for method register
 697 reg_class method_reg(
 698     R12, R12_H
 699 );
 700 
 701 // Class for heapbase register
 702 reg_class heapbase_reg(
 703     R27, R27_H
 704 );
 705 
 706 // Class for thread register
 707 reg_class thread_reg(
 708     R28, R28_H
 709 );
 710 
 711 // Class for frame pointer register
 712 reg_class fp_reg(
 713     R29, R29_H
 714 );
 715 
 716 // Class for link register
 717 reg_class lr_reg(
 718     R30, R30_H
 719 );
 720 
 721 // Class for long sp register
 722 reg_class sp_reg(
 723   R31, R31_H
 724 );
 725 
 726 // Class for all pointer registers
 727 reg_class ptr_reg(
 728     R0, R0_H,
 729     R1, R1_H,
 730     R2, R2_H,
 731     R3, R3_H,
 732     R4, R4_H,
 733     R5, R5_H,
 734     R6, R6_H,
 735     R7, R7_H,
 736     R10, R10_H,
 737     R11, R11_H,
 738     R12, R12_H,
 739     R13, R13_H,
 740     R14, R14_H,
 741     R15, R15_H,
 742     R16, R16_H,
 743     R17, R17_H,
 744     R18, R18_H,
 745     R19, R19_H,
 746     R20, R20_H,
 747     R21, R21_H,
 748     R22, R22_H,
 749     R23, R23_H,
 750     R24, R24_H,
 751     R25, R25_H,
 752     R26, R26_H,
 753     R27, R27_H,
 754     R28, R28_H,
 755     R29, R29_H,
 756     R30, R30_H,
 757     R31, R31_H
 758 );
 759 
 760 // Class for all non_special pointer registers
 761 reg_class no_special_ptr_reg(
 762     R0, R0_H,
 763     R1, R1_H,
 764     R2, R2_H,
 765     R3, R3_H,
 766     R4, R4_H,
 767     R5, R5_H,
 768     R6, R6_H,
 769     R7, R7_H,
 770     R10, R10_H,
 771     R11, R11_H,
 772     R12, R12_H,
 773     R13, R13_H,
 774     R14, R14_H,
 775     R15, R15_H,
 776     R16, R16_H,
 777     R17, R17_H,
 778     R18, R18_H,
 779     R19, R19_H,
 780     R20, R20_H,
 781     R21, R21_H,
 782     R22, R22_H,
 783     R23, R23_H,
 784     R24, R24_H,
 785     R25, R25_H,
 786     R26, R26_H,
 787  /* R27, R27_H, */              // heapbase
 788  /* R28, R28_H, */              // thread
 789  /* R29, R29_H, */              // fp
 790  /* R30, R30_H, */              // lr
 791  /* R31, R31_H */               // sp
 792 );
 793 
 794 // Class for all float registers
 795 reg_class float_reg(
 796     V0,
 797     V1,
 798     V2,
 799     V3,
 800     V4,
 801     V5,
 802     V6,
 803     V7,
 804     V8,
 805     V9,
 806     V10,
 807     V11,
 808     V12,
 809     V13,
 810     V14,
 811     V15,
 812     V16,
 813     V17,
 814     V18,
 815     V19,
 816     V20,
 817     V21,
 818     V22,
 819     V23,
 820     V24,
 821     V25,
 822     V26,
 823     V27,
 824     V28,
 825     V29,
 826     V30,
 827     V31
 828 );
 829 
 830 // Double precision float registers have virtual `high halves' that
 831 // are needed by the allocator.
 832 // Class for all double registers
 833 reg_class double_reg(
 834     V0, V0_H,
 835     V1, V1_H,
 836     V2, V2_H,
 837     V3, V3_H,
 838     V4, V4_H,
 839     V5, V5_H,
 840     V6, V6_H,
 841     V7, V7_H,
 842     V8, V8_H,
 843     V9, V9_H,
 844     V10, V10_H,
 845     V11, V11_H,
 846     V12, V12_H,
 847     V13, V13_H,
 848     V14, V14_H,
 849     V15, V15_H,
 850     V16, V16_H,
 851     V17, V17_H,
 852     V18, V18_H,
 853     V19, V19_H,
 854     V20, V20_H,
 855     V21, V21_H,
 856     V22, V22_H,
 857     V23, V23_H,
 858     V24, V24_H,
 859     V25, V25_H,
 860     V26, V26_H,
 861     V27, V27_H,
 862     V28, V28_H,
 863     V29, V29_H,
 864     V30, V30_H,
 865     V31, V31_H
 866 );
 867 
 868 // Class for all 64bit vector registers
 869 reg_class vectord_reg(
 870     V0, V0_H,
 871     V1, V1_H,
 872     V2, V2_H,
 873     V3, V3_H,
 874     V4, V4_H,
 875     V5, V5_H,
 876     V6, V6_H,
 877     V7, V7_H,
 878     V8, V8_H,
 879     V9, V9_H,
 880     V10, V10_H,
 881     V11, V11_H,
 882     V12, V12_H,
 883     V13, V13_H,
 884     V14, V14_H,
 885     V15, V15_H,
 886     V16, V16_H,
 887     V17, V17_H,
 888     V18, V18_H,
 889     V19, V19_H,
 890     V20, V20_H,
 891     V21, V21_H,
 892     V22, V22_H,
 893     V23, V23_H,
 894     V24, V24_H,
 895     V25, V25_H,
 896     V26, V26_H,
 897     V27, V27_H,
 898     V28, V28_H,
 899     V29, V29_H,
 900     V30, V30_H,
 901     V31, V31_H
 902 );
 903 
 904 // Class for all 128bit vector registers
 905 reg_class vectorx_reg(
 906     V0, V0_H, V0_J, V0_K,
 907     V1, V1_H, V1_J, V1_K,
 908     V2, V2_H, V2_J, V2_K,
 909     V3, V3_H, V3_J, V3_K,
 910     V4, V4_H, V4_J, V4_K,
 911     V5, V5_H, V5_J, V5_K,
 912     V6, V6_H, V6_J, V6_K,
 913     V7, V7_H, V7_J, V7_K,
 914     V8, V8_H, V8_J, V8_K,
 915     V9, V9_H, V9_J, V9_K,
 916     V10, V10_H, V10_J, V10_K,
 917     V11, V11_H, V11_J, V11_K,
 918     V12, V12_H, V12_J, V12_K,
 919     V13, V13_H, V13_J, V13_K,
 920     V14, V14_H, V14_J, V14_K,
 921     V15, V15_H, V15_J, V15_K,
 922     V16, V16_H, V16_J, V16_K,
 923     V17, V17_H, V17_J, V17_K,
 924     V18, V18_H, V18_J, V18_K,
 925     V19, V19_H, V19_J, V19_K,
 926     V20, V20_H, V20_J, V20_K,
 927     V21, V21_H, V21_J, V21_K,
 928     V22, V22_H, V22_J, V22_K,
 929     V23, V23_H, V23_J, V23_K,
 930     V24, V24_H, V24_J, V24_K,
 931     V25, V25_H, V25_J, V25_K,
 932     V26, V26_H, V26_J, V26_K,
 933     V27, V27_H, V27_J, V27_K,
 934     V28, V28_H, V28_J, V28_K,
 935     V29, V29_H, V29_J, V29_K,
 936     V30, V30_H, V30_J, V30_K,
 937     V31, V31_H, V31_J, V31_K
 938 );
 939 
 940 // Class for 128 bit register v0
 941 reg_class v0_reg(
 942     V0, V0_H
 943 );
 944 
 945 // Class for 128 bit register v1
 946 reg_class v1_reg(
 947     V1, V1_H
 948 );
 949 
 950 // Class for 128 bit register v2
 951 reg_class v2_reg(
 952     V2, V2_H
 953 );
 954 
 955 // Class for 128 bit register v3
 956 reg_class v3_reg(
 957     V3, V3_H
 958 );
 959 
 960 // Singleton class for condition codes
 961 reg_class int_flags(RFLAGS);
 962 
 963 %}
 964 
 965 //----------DEFINITION BLOCK---------------------------------------------------
 966 // Define name --> value mappings to inform the ADLC of an integer valued name
 967 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 968 // Format:
 969 //        int_def  <name>         ( <int_value>, <expression>);
 970 // Generated Code in ad_<arch>.hpp
 971 //        #define  <name>   (<expression>)
 972 //        // value == <int_value>
 973 // Generated code in ad_<arch>.cpp adlc_verification()
 974 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 975 //
 976 
 977 // we follow the ppc-aix port in using a simple cost model which ranks
 978 // register operations as cheap, memory ops as more expensive and
 979 // branches as most expensive. the first two have a low as well as a
 980 // normal cost. huge cost appears to be a way of saying don't do
 981 // something
 982 
 983 definitions %{
 984   // The default cost (of a register move instruction).
 985   int_def INSN_COST            (    100,     100);
 986   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 987   int_def CALL_COST            (    200,     2 * INSN_COST);
 988   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 989 %}
 990 
 991 
 992 //----------SOURCE BLOCK-------------------------------------------------------
 993 // This is a block of C++ code which provides values, functions, and
 994 // definitions necessary in the rest of the architecture description
 995 
 996 source_hpp %{
 997 
 998 #include "asm/macroAssembler.hpp"
 999 #include "gc/shared/cardTable.hpp"
1000 #include "gc/shared/cardTableBarrierSet.hpp"
1001 #include "gc/shared/collectedHeap.hpp"
1002 #include "opto/addnode.hpp"
1003 
1004 class CallStubImpl {
1005 
1006   //--------------------------------------------------------------
1007   //---<  Used for optimization in Compile::shorten_branches  >---
1008   //--------------------------------------------------------------
1009 
1010  public:
1011   // Size of call trampoline stub.
1012   static uint size_call_trampoline() {
1013     return 0; // no call trampolines on this platform
1014   }
1015 
1016   // number of relocations needed by a call trampoline stub
1017   static uint reloc_call_trampoline() {
1018     return 0; // no call trampolines on this platform
1019   }
1020 };
1021 
1022 class HandlerImpl {
1023 
1024  public:
1025 
1026   static int emit_exception_handler(CodeBuffer &cbuf);
1027   static int emit_deopt_handler(CodeBuffer& cbuf);
1028 
1029   static uint size_exception_handler() {
1030     return MacroAssembler::far_branch_size();
1031   }
1032 
1033   static uint size_deopt_handler() {
1034     // count one adr and one far branch instruction
1035     return 4 * NativeInstruction::instruction_size;
1036   }
1037 };
1038 
1039  bool is_CAS(int opcode, bool maybe_volatile);
1040 
1041   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1042 
1043   bool unnecessary_acquire(const Node *barrier);
1044   bool needs_acquiring_load(const Node *load);
1045 
1046   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1047 
1048   bool unnecessary_release(const Node *barrier);
1049   bool unnecessary_volatile(const Node *barrier);
1050   bool needs_releasing_store(const Node *store);
1051 
1052   // predicate controlling translation of CompareAndSwapX
1053   bool needs_acquiring_load_exclusive(const Node *load);
1054 
1055   // predicate controlling translation of StoreCM
1056   bool unnecessary_storestore(const Node *storecm);
1057 
1058   // predicate controlling addressing modes
1059   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1060 %}
1061 
1062 source %{
1063 
1064   // Optimizaton of volatile gets and puts
1065   // -------------------------------------
1066   //
1067   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1068   // use to implement volatile reads and writes. For a volatile read
1069   // we simply need
1070   //
1071   //   ldar<x>
1072   //
1073   // and for a volatile write we need
1074   //
1075   //   stlr<x>
1076   //
1077   // Alternatively, we can implement them by pairing a normal
1078   // load/store with a memory barrier. For a volatile read we need
1079   //
1080   //   ldr<x>
1081   //   dmb ishld
1082   //
1083   // for a volatile write
1084   //
1085   //   dmb ish
1086   //   str<x>
1087   //   dmb ish
1088   //
1089   // We can also use ldaxr and stlxr to implement compare and swap CAS
1090   // sequences. These are normally translated to an instruction
1091   // sequence like the following
1092   //
1093   //   dmb      ish
1094   // retry:
1095   //   ldxr<x>   rval raddr
1096   //   cmp       rval rold
1097   //   b.ne done
1098   //   stlxr<x>  rval, rnew, rold
1099   //   cbnz      rval retry
1100   // done:
1101   //   cset      r0, eq
1102   //   dmb ishld
1103   //
1104   // Note that the exclusive store is already using an stlxr
1105   // instruction. That is required to ensure visibility to other
1106   // threads of the exclusive write (assuming it succeeds) before that
1107   // of any subsequent writes.
1108   //
1109   // The following instruction sequence is an improvement on the above
1110   //
1111   // retry:
1112   //   ldaxr<x>  rval raddr
1113   //   cmp       rval rold
1114   //   b.ne done
1115   //   stlxr<x>  rval, rnew, rold
1116   //   cbnz      rval retry
1117   // done:
1118   //   cset      r0, eq
1119   //
1120   // We don't need the leading dmb ish since the stlxr guarantees
1121   // visibility of prior writes in the case that the swap is
1122   // successful. Crucially we don't have to worry about the case where
1123   // the swap is not successful since no valid program should be
1124   // relying on visibility of prior changes by the attempting thread
1125   // in the case where the CAS fails.
1126   //
1127   // Similarly, we don't need the trailing dmb ishld if we substitute
1128   // an ldaxr instruction since that will provide all the guarantees we
1129   // require regarding observation of changes made by other threads
1130   // before any change to the CAS address observed by the load.
1131   //
1132   // In order to generate the desired instruction sequence we need to
1133   // be able to identify specific 'signature' ideal graph node
1134   // sequences which i) occur as a translation of a volatile reads or
1135   // writes or CAS operations and ii) do not occur through any other
1136   // translation or graph transformation. We can then provide
1137   // alternative aldc matching rules which translate these node
1138   // sequences to the desired machine code sequences. Selection of the
1139   // alternative rules can be implemented by predicates which identify
1140   // the relevant node sequences.
1141   //
1142   // The ideal graph generator translates a volatile read to the node
1143   // sequence
1144   //
1145   //   LoadX[mo_acquire]
1146   //   MemBarAcquire
1147   //
1148   // As a special case when using the compressed oops optimization we
1149   // may also see this variant
1150   //
1151   //   LoadN[mo_acquire]
1152   //   DecodeN
1153   //   MemBarAcquire
1154   //
1155   // A volatile write is translated to the node sequence
1156   //
1157   //   MemBarRelease
1158   //   StoreX[mo_release] {CardMark}-optional
1159   //   MemBarVolatile
1160   //
1161   // n.b. the above node patterns are generated with a strict
1162   // 'signature' configuration of input and output dependencies (see
1163   // the predicates below for exact details). The card mark may be as
1164   // simple as a few extra nodes or, in a few GC configurations, may
1165   // include more complex control flow between the leading and
1166   // trailing memory barriers. However, whatever the card mark
1167   // configuration these signatures are unique to translated volatile
1168   // reads/stores -- they will not appear as a result of any other
1169   // bytecode translation or inlining nor as a consequence of
1170   // optimizing transforms.
1171   //
1172   // We also want to catch inlined unsafe volatile gets and puts and
1173   // be able to implement them using either ldar<x>/stlr<x> or some
1174   // combination of ldr<x>/stlr<x> and dmb instructions.
1175   //
1176   // Inlined unsafe volatiles puts manifest as a minor variant of the
1177   // normal volatile put node sequence containing an extra cpuorder
1178   // membar
1179   //
1180   //   MemBarRelease
1181   //   MemBarCPUOrder
1182   //   StoreX[mo_release] {CardMark}-optional
1183   //   MemBarCPUOrder
1184   //   MemBarVolatile
1185   //
1186   // n.b. as an aside, a cpuorder membar is not itself subject to
1187   // matching and translation by adlc rules.  However, the rule
1188   // predicates need to detect its presence in order to correctly
1189   // select the desired adlc rules.
1190   //
1191   // Inlined unsafe volatile gets manifest as a slightly different
1192   // node sequence to a normal volatile get because of the
1193   // introduction of some CPUOrder memory barriers to bracket the
1194   // Load. However, but the same basic skeleton of a LoadX feeding a
1195   // MemBarAcquire, possibly thorugh an optional DecodeN, is still
1196   // present
1197   //
1198   //   MemBarCPUOrder
1199   //        ||       \\
1200   //   MemBarCPUOrder LoadX[mo_acquire]
1201   //        ||            |
1202   //        ||       {DecodeN} optional
1203   //        ||       /
1204   //     MemBarAcquire
1205   //
1206   // In this case the acquire membar does not directly depend on the
1207   // load. However, we can be sure that the load is generated from an
1208   // inlined unsafe volatile get if we see it dependent on this unique
1209   // sequence of membar nodes. Similarly, given an acquire membar we
1210   // can know that it was added because of an inlined unsafe volatile
1211   // get if it is fed and feeds a cpuorder membar and if its feed
1212   // membar also feeds an acquiring load.
1213   //
1214   // Finally an inlined (Unsafe) CAS operation is translated to the
1215   // following ideal graph
1216   //
1217   //   MemBarRelease
1218   //   MemBarCPUOrder
1219   //   CompareAndSwapX {CardMark}-optional
1220   //   MemBarCPUOrder
1221   //   MemBarAcquire
1222   //
1223   // So, where we can identify these volatile read and write
1224   // signatures we can choose to plant either of the above two code
1225   // sequences. For a volatile read we can simply plant a normal
1226   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1227   // also choose to inhibit translation of the MemBarAcquire and
1228   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1229   //
1230   // When we recognise a volatile store signature we can choose to
1231   // plant at a dmb ish as a translation for the MemBarRelease, a
1232   // normal str<x> and then a dmb ish for the MemBarVolatile.
1233   // Alternatively, we can inhibit translation of the MemBarRelease
1234   // and MemBarVolatile and instead plant a simple stlr<x>
1235   // instruction.
1236   //
1237   // when we recognise a CAS signature we can choose to plant a dmb
1238   // ish as a translation for the MemBarRelease, the conventional
1239   // macro-instruction sequence for the CompareAndSwap node (which
1240   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1241   // Alternatively, we can elide generation of the dmb instructions
1242   // and plant the alternative CompareAndSwap macro-instruction
1243   // sequence (which uses ldaxr<x>).
1244   //
1245   // Of course, the above only applies when we see these signature
1246   // configurations. We still want to plant dmb instructions in any
1247   // other cases where we may see a MemBarAcquire, MemBarRelease or
1248   // MemBarVolatile. For example, at the end of a constructor which
1249   // writes final/volatile fields we will see a MemBarRelease
1250   // instruction and this needs a 'dmb ish' lest we risk the
1251   // constructed object being visible without making the
1252   // final/volatile field writes visible.
1253   //
1254   // n.b. the translation rules below which rely on detection of the
1255   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1256   // If we see anything other than the signature configurations we
1257   // always just translate the loads and stores to ldr<x> and str<x>
1258   // and translate acquire, release and volatile membars to the
1259   // relevant dmb instructions.
1260   //
1261 
1262   // is_CAS(int opcode, bool maybe_volatile)
1263   //
1264   // return true if opcode is one of the possible CompareAndSwapX
1265   // values otherwise false.
1266 
1267   bool is_CAS(int opcode, bool maybe_volatile)
1268   {
1269     switch(opcode) {
1270       // We handle these
1271     case Op_CompareAndSwapI:
1272     case Op_CompareAndSwapL:
1273     case Op_CompareAndSwapP:
1274     case Op_CompareAndSwapN:
1275     case Op_ShenandoahCompareAndSwapP:
1276     case Op_ShenandoahCompareAndSwapN:
1277     case Op_CompareAndSwapB:
1278     case Op_CompareAndSwapS:
1279     case Op_GetAndSetI:
1280     case Op_GetAndSetL:
1281     case Op_GetAndSetP:
1282     case Op_GetAndSetN:
1283     case Op_GetAndAddI:
1284     case Op_GetAndAddL:
1285       return true;
1286     case Op_CompareAndExchangeI:
1287     case Op_CompareAndExchangeN:
1288     case Op_CompareAndExchangeB:
1289     case Op_CompareAndExchangeS:
1290     case Op_CompareAndExchangeL:
1291     case Op_CompareAndExchangeP:
1292     case Op_WeakCompareAndSwapB:
1293     case Op_WeakCompareAndSwapS:
1294     case Op_WeakCompareAndSwapI:
1295     case Op_WeakCompareAndSwapL:
1296     case Op_WeakCompareAndSwapP:
1297     case Op_WeakCompareAndSwapN:
1298     case Op_ShenandoahWeakCompareAndSwapP:
1299     case Op_ShenandoahWeakCompareAndSwapN:
1300     case Op_ShenandoahCompareAndExchangeP:
1301     case Op_ShenandoahCompareAndExchangeN:
1302       return maybe_volatile;
1303     default:
1304       return false;
1305     }
1306   }
1307 
1308   // helper to determine the maximum number of Phi nodes we may need to
1309   // traverse when searching from a card mark membar for the merge mem
1310   // feeding a trailing membar or vice versa
1311 
1312 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1313 
1314 bool unnecessary_acquire(const Node *barrier)
1315 {
1316   assert(barrier->is_MemBar(), "expecting a membar");
1317 
1318   if (UseBarriersForVolatile) {
1319     // we need to plant a dmb
1320     return false;
1321   }
1322 
1323   MemBarNode* mb = barrier->as_MemBar();
1324 
1325   if (mb->trailing_load()) {
1326     return true;
1327   }
1328 
1329   if (mb->trailing_load_store()) {
1330     Node* load_store = mb->in(MemBarNode::Precedent);
1331     assert(load_store->is_LoadStore(), "unexpected graph shape");
1332     return is_CAS(load_store->Opcode(), true);
1333   }
1334 
1335   return false;
1336 }
1337 
1338 bool needs_acquiring_load(const Node *n)
1339 {
1340   assert(n->is_Load(), "expecting a load");
1341   if (UseBarriersForVolatile) {
1342     // we use a normal load and a dmb
1343     return false;
1344   }
1345 
1346   LoadNode *ld = n->as_Load();
1347 
1348   return ld->is_acquire();
1349 }
1350 
1351 bool unnecessary_release(const Node *n)
1352 {
1353   assert((n->is_MemBar() &&
1354           n->Opcode() == Op_MemBarRelease),
1355          "expecting a release membar");
1356 
1357   if (UseBarriersForVolatile) {
1358     // we need to plant a dmb
1359     return false;
1360   }
1361 
1362   MemBarNode *barrier = n->as_MemBar();
1363   if (!barrier->leading()) {
1364     return false;
1365   } else {
1366     Node* trailing = barrier->trailing_membar();
1367     MemBarNode* trailing_mb = trailing->as_MemBar();
1368     assert(trailing_mb->trailing(), "Not a trailing membar?");
1369     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1370 
1371     Node* mem = trailing_mb->in(MemBarNode::Precedent);
1372     if (mem->is_Store()) {
1373       assert(mem->as_Store()->is_release(), "");
1374       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1375       return true;
1376     } else {
1377       assert(mem->is_LoadStore(), "");
1378       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1379       return is_CAS(mem->Opcode(), true);
1380     }
1381   }
1382   return false;
1383 }
1384 
1385 bool unnecessary_volatile(const Node *n)
1386 {
1387   // assert n->is_MemBar();
1388   if (UseBarriersForVolatile) {
1389     // we need to plant a dmb
1390     return false;
1391   }
1392 
1393   MemBarNode *mbvol = n->as_MemBar();
1394 
1395   bool release = mbvol->trailing_store();
1396   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1397 #ifdef ASSERT
1398   if (release) {
1399     Node* leading = mbvol->leading_membar();
1400     assert(leading->Opcode() == Op_MemBarRelease, "");
1401     assert(leading->as_MemBar()->leading_store(), "");
1402     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1403   }
1404 #endif
1405 
1406   return release;
1407 }
1408 
1409 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1410 
1411 bool needs_releasing_store(const Node *n)
1412 {
1413   // assert n->is_Store();
1414   if (UseBarriersForVolatile) {
1415     // we use a normal store and dmb combination
1416     return false;
1417   }
1418 
1419   StoreNode *st = n->as_Store();
1420 
1421   return st->trailing_membar() != NULL;
1422 }
1423 
1424 // predicate controlling translation of CAS
1425 //
1426 // returns true if CAS needs to use an acquiring load otherwise false
1427 
1428 bool needs_acquiring_load_exclusive(const Node *n)
1429 {
1430   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1431   if (UseBarriersForVolatile) {
1432     return false;
1433   }
1434 
1435   LoadStoreNode* ldst = n->as_LoadStore();
1436   if (is_CAS(n->Opcode(), false)) {
1437     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
1438   } else {
1439     return ldst->trailing_membar() != NULL;
1440   }
1441 
1442   // so we can just return true here
1443   return true;
1444 }
1445 
1446 // predicate controlling translation of StoreCM
1447 //
1448 // returns true if a StoreStore must precede the card write otherwise
1449 // false
1450 
1451 bool unnecessary_storestore(const Node *storecm)
1452 {
1453   assert(storecm->Opcode()  == Op_StoreCM, "expecting a StoreCM");
1454 
1455   // we need to generate a dmb ishst between an object put and the
1456   // associated card mark when we are using CMS without conditional
1457   // card marking
1458 
1459   if (UseConcMarkSweepGC && !UseCondCardMark) {
1460     return false;
1461   }
1462 
1463   // a storestore is unnecesary in all other cases
1464 
1465   return true;
1466 }
1467 
1468 
1469 #define __ _masm.
1470 
1471 // advance declarations for helper functions to convert register
1472 // indices to register objects
1473 
1474 // the ad file has to provide implementations of certain methods
1475 // expected by the generic code
1476 //
1477 // REQUIRED FUNCTIONALITY
1478 
1479 //=============================================================================
1480 
1481 // !!!!! Special hack to get all types of calls to specify the byte offset
1482 //       from the start of the call to the point where the return address
1483 //       will point.
1484 
1485 int MachCallStaticJavaNode::ret_addr_offset()
1486 {
1487   // call should be a simple bl
1488   int off = 4;
1489   return off;
1490 }
1491 
1492 int MachCallDynamicJavaNode::ret_addr_offset()
1493 {
1494   return 16; // movz, movk, movk, bl
1495 }
1496 
1497 int MachCallRuntimeNode::ret_addr_offset() {
1498   // for generated stubs the call will be
1499   //   far_call(addr)
1500   // for real runtime callouts it will be six instructions
1501   // see aarch64_enc_java_to_runtime
1502   //   adr(rscratch2, retaddr)
1503   //   lea(rscratch1, RuntimeAddress(addr)
1504   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1505   //   blrt rscratch1
1506   CodeBlob *cb = CodeCache::find_blob(_entry_point);
1507   if (cb) {
1508     return MacroAssembler::far_branch_size();
1509   } else {
1510     return 6 * NativeInstruction::instruction_size;
1511   }
1512 }
1513 
1514 // Indicate if the safepoint node needs the polling page as an input
1515 
1516 // the shared code plants the oop data at the start of the generated
1517 // code for the safepoint node and that needs ot be at the load
1518 // instruction itself. so we cannot plant a mov of the safepoint poll
1519 // address followed by a load. setting this to true means the mov is
1520 // scheduled as a prior instruction. that's better for scheduling
1521 // anyway.
1522 
1523 bool SafePointNode::needs_polling_address_input()
1524 {
1525   return true;
1526 }
1527 
1528 //=============================================================================
1529 
1530 #ifndef PRODUCT
1531 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1532   st->print("BREAKPOINT");
1533 }
1534 #endif
1535 
1536 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1537   MacroAssembler _masm(&cbuf);
1538   __ brk(0);
1539 }
1540 
1541 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1542   return MachNode::size(ra_);
1543 }
1544 
1545 //=============================================================================
1546 
1547 #ifndef PRODUCT
1548   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1549     st->print("nop \t# %d bytes pad for loops and calls", _count);
1550   }
1551 #endif
1552 
1553   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1554     MacroAssembler _masm(&cbuf);
1555     for (int i = 0; i < _count; i++) {
1556       __ nop();
1557     }
1558   }
1559 
1560   uint MachNopNode::size(PhaseRegAlloc*) const {
1561     return _count * NativeInstruction::instruction_size;
1562   }
1563 
1564 //=============================================================================
1565 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1566 
1567 int Compile::ConstantTable::calculate_table_base_offset() const {
1568   return 0;  // absolute addressing, no offset
1569 }
1570 
1571 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1572 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1573   ShouldNotReachHere();
1574 }
1575 
1576 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1577   // Empty encoding
1578 }
1579 
1580 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1581   return 0;
1582 }
1583 
1584 #ifndef PRODUCT
1585 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1586   st->print("-- \t// MachConstantBaseNode (empty encoding)");
1587 }
1588 #endif
1589 
1590 #ifndef PRODUCT
1591 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1592   Compile* C = ra_->C;
1593 
1594   int framesize = C->frame_slots() << LogBytesPerInt;
1595 
1596   if (C->need_stack_bang(framesize))
1597     st->print("# stack bang size=%d\n\t", framesize);
1598 
1599   if (framesize < ((1 << 9) + 2 * wordSize)) {
1600     st->print("sub  sp, sp, #%d\n\t", framesize);
1601     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1602     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
1603   } else {
1604     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1605     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
1606     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
1607     st->print("sub  sp, sp, rscratch1");
1608   }
1609 }
1610 #endif
1611 
1612 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1613   Compile* C = ra_->C;
1614   MacroAssembler _masm(&cbuf);
1615 
1616   // n.b. frame size includes space for return pc and rfp
1617   const long framesize = C->frame_size_in_bytes();
1618   assert(framesize%(2*wordSize) == 0, "must preserve 2*wordSize alignment");
1619 
1620   // insert a nop at the start of the prolog so we can patch in a
1621   // branch if we need to invalidate the method later
1622   __ nop();
1623 
1624   int bangsize = C->bang_size_in_bytes();
1625   if (C->need_stack_bang(bangsize) && UseStackBanging)
1626     __ generate_stack_overflow_check(bangsize);
1627 
1628   __ build_frame(framesize);
1629 
1630   if (NotifySimulator) {
1631     __ notify(Assembler::method_entry);
1632   }
1633 
1634   if (VerifyStackAtCalls) {
1635     Unimplemented();
1636   }
1637 
1638   C->set_frame_complete(cbuf.insts_size());
1639 
1640   if (C->has_mach_constant_base_node()) {
1641     // NOTE: We set the table base offset here because users might be
1642     // emitted before MachConstantBaseNode.
1643     Compile::ConstantTable& constant_table = C->constant_table();
1644     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1645   }
1646 }
1647 
1648 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1649 {
1650   return MachNode::size(ra_); // too many variables; just compute it
1651                               // the hard way
1652 }
1653 
1654 int MachPrologNode::reloc() const
1655 {
1656   return 0;
1657 }
1658 
1659 //=============================================================================
1660 
1661 #ifndef PRODUCT
1662 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1663   Compile* C = ra_->C;
1664   int framesize = C->frame_slots() << LogBytesPerInt;
1665 
1666   st->print("# pop frame %d\n\t",framesize);
1667 
1668   if (framesize == 0) {
1669     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1670   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1671     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1672     st->print("add  sp, sp, #%d\n\t", framesize);
1673   } else {
1674     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
1675     st->print("add  sp, sp, rscratch1\n\t");
1676     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1677   }
1678 
1679   if (do_polling() && C->is_method_compilation()) {
1680     st->print("# touch polling page\n\t");
1681     st->print("mov  rscratch1, #0x%lx\n\t", p2i(os::get_polling_page()));
1682     st->print("ldr zr, [rscratch1]");
1683   }
1684 }
1685 #endif
1686 
1687 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1688   Compile* C = ra_->C;
1689   MacroAssembler _masm(&cbuf);
1690   int framesize = C->frame_slots() << LogBytesPerInt;
1691 
1692   __ remove_frame(framesize);
1693 
1694   if (NotifySimulator) {
1695     __ notify(Assembler::method_reentry);
1696   }
1697 
1698   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1699     __ reserved_stack_check();
1700   }
1701 
1702   if (do_polling() && C->is_method_compilation()) {
1703     __ read_polling_page(rscratch1, os::get_polling_page(), relocInfo::poll_return_type);
1704   }
1705 }
1706 
1707 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1708   // Variable size. Determine dynamically.
1709   return MachNode::size(ra_);
1710 }
1711 
1712 int MachEpilogNode::reloc() const {
1713   // Return number of relocatable values contained in this instruction.
1714   return 1; // 1 for polling page.
1715 }
1716 
1717 const Pipeline * MachEpilogNode::pipeline() const {
1718   return MachNode::pipeline_class();
1719 }
1720 
1721 // This method seems to be obsolete. It is declared in machnode.hpp
1722 // and defined in all *.ad files, but it is never called. Should we
1723 // get rid of it?
1724 int MachEpilogNode::safepoint_offset() const {
1725   assert(do_polling(), "no return for this epilog node");
1726   return 4;
1727 }
1728 
1729 //=============================================================================
1730 
1731 // Figure out which register class each belongs in: rc_int, rc_float or
1732 // rc_stack.
1733 enum RC { rc_bad, rc_int, rc_float, rc_stack };
1734 
1735 static enum RC rc_class(OptoReg::Name reg) {
1736 
1737   if (reg == OptoReg::Bad) {
1738     return rc_bad;
1739   }
1740 
1741   // we have 30 int registers * 2 halves
1742   // (rscratch1 and rscratch2 are omitted)
1743 
1744   if (reg < 60) {
1745     return rc_int;
1746   }
1747 
1748   // we have 32 float register * 2 halves
1749   if (reg < 60 + 128) {
1750     return rc_float;
1751   }
1752 
1753   // Between float regs & stack is the flags regs.
1754   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1755 
1756   return rc_stack;
1757 }
1758 
1759 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1760   Compile* C = ra_->C;
1761 
1762   // Get registers to move.
1763   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1764   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1765   OptoReg::Name dst_hi = ra_->get_reg_second(this);
1766   OptoReg::Name dst_lo = ra_->get_reg_first(this);
1767 
1768   enum RC src_hi_rc = rc_class(src_hi);
1769   enum RC src_lo_rc = rc_class(src_lo);
1770   enum RC dst_hi_rc = rc_class(dst_hi);
1771   enum RC dst_lo_rc = rc_class(dst_lo);
1772 
1773   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1774 
1775   if (src_hi != OptoReg::Bad) {
1776     assert((src_lo&1)==0 && src_lo+1==src_hi &&
1777            (dst_lo&1)==0 && dst_lo+1==dst_hi,
1778            "expected aligned-adjacent pairs");
1779   }
1780 
1781   if (src_lo == dst_lo && src_hi == dst_hi) {
1782     return 0;            // Self copy, no move.
1783   }
1784 
1785   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1786               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1787   int src_offset = ra_->reg2offset(src_lo);
1788   int dst_offset = ra_->reg2offset(dst_lo);
1789 
1790   if (bottom_type()->isa_vect() != NULL) {
1791     uint ireg = ideal_reg();
1792     assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1793     if (cbuf) {
1794       MacroAssembler _masm(cbuf);
1795       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1796       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1797         // stack->stack
1798         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1799         if (ireg == Op_VecD) {
1800           __ unspill(rscratch1, true, src_offset);
1801           __ spill(rscratch1, true, dst_offset);
1802         } else {
1803           __ spill_copy128(src_offset, dst_offset);
1804         }
1805       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1806         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1807                ireg == Op_VecD ? __ T8B : __ T16B,
1808                as_FloatRegister(Matcher::_regEncode[src_lo]));
1809       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1810         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1811                        ireg == Op_VecD ? __ D : __ Q,
1812                        ra_->reg2offset(dst_lo));
1813       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1814         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1815                        ireg == Op_VecD ? __ D : __ Q,
1816                        ra_->reg2offset(src_lo));
1817       } else {
1818         ShouldNotReachHere();
1819       }
1820     }
1821   } else if (cbuf) {
1822     MacroAssembler _masm(cbuf);
1823     switch (src_lo_rc) {
1824     case rc_int:
1825       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
1826         if (is64) {
1827             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
1828                    as_Register(Matcher::_regEncode[src_lo]));
1829         } else {
1830             MacroAssembler _masm(cbuf);
1831             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
1832                     as_Register(Matcher::_regEncode[src_lo]));
1833         }
1834       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
1835         if (is64) {
1836             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1837                      as_Register(Matcher::_regEncode[src_lo]));
1838         } else {
1839             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1840                      as_Register(Matcher::_regEncode[src_lo]));
1841         }
1842       } else {                    // gpr --> stack spill
1843         assert(dst_lo_rc == rc_stack, "spill to bad register class");
1844         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
1845       }
1846       break;
1847     case rc_float:
1848       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
1849         if (is64) {
1850             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
1851                      as_FloatRegister(Matcher::_regEncode[src_lo]));
1852         } else {
1853             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
1854                      as_FloatRegister(Matcher::_regEncode[src_lo]));
1855         }
1856       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
1857           if (cbuf) {
1858             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1859                      as_FloatRegister(Matcher::_regEncode[src_lo]));
1860         } else {
1861             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1862                      as_FloatRegister(Matcher::_regEncode[src_lo]));
1863         }
1864       } else {                    // fpr --> stack spill
1865         assert(dst_lo_rc == rc_stack, "spill to bad register class");
1866         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1867                  is64 ? __ D : __ S, dst_offset);
1868       }
1869       break;
1870     case rc_stack:
1871       if (dst_lo_rc == rc_int) {  // stack --> gpr load
1872         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
1873       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
1874         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1875                    is64 ? __ D : __ S, src_offset);
1876       } else {                    // stack --> stack copy
1877         assert(dst_lo_rc == rc_stack, "spill to bad register class");
1878         __ unspill(rscratch1, is64, src_offset);
1879         __ spill(rscratch1, is64, dst_offset);
1880       }
1881       break;
1882     default:
1883       assert(false, "bad rc_class for spill");
1884       ShouldNotReachHere();
1885     }
1886   }
1887 
1888   if (st) {
1889     st->print("spill ");
1890     if (src_lo_rc == rc_stack) {
1891       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
1892     } else {
1893       st->print("%s -> ", Matcher::regName[src_lo]);
1894     }
1895     if (dst_lo_rc == rc_stack) {
1896       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
1897     } else {
1898       st->print("%s", Matcher::regName[dst_lo]);
1899     }
1900     if (bottom_type()->isa_vect() != NULL) {
1901       st->print("\t# vector spill size = %d", ideal_reg()==Op_VecD ? 64:128);
1902     } else {
1903       st->print("\t# spill size = %d", is64 ? 64:32);
1904     }
1905   }
1906 
1907   return 0;
1908 
1909 }
1910 
1911 #ifndef PRODUCT
1912 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1913   if (!ra_)
1914     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
1915   else
1916     implementation(NULL, ra_, false, st);
1917 }
1918 #endif
1919 
1920 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1921   implementation(&cbuf, ra_, false, NULL);
1922 }
1923 
1924 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1925   return MachNode::size(ra_);
1926 }
1927 
1928 //=============================================================================
1929 
1930 #ifndef PRODUCT
1931 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1932   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1933   int reg = ra_->get_reg_first(this);
1934   st->print("add %s, rsp, #%d]\t# box lock",
1935             Matcher::regName[reg], offset);
1936 }
1937 #endif
1938 
1939 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1940   MacroAssembler _masm(&cbuf);
1941 
1942   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1943   int reg    = ra_->get_encode(this);
1944 
1945   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
1946     __ add(as_Register(reg), sp, offset);
1947   } else {
1948     ShouldNotReachHere();
1949   }
1950 }
1951 
1952 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
1953   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
1954   return 4;
1955 }
1956 
1957 //=============================================================================
1958 
1959 #ifndef PRODUCT
1960 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1961 {
1962   st->print_cr("# MachUEPNode");
1963   if (UseCompressedClassPointers) {
1964     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1965     if (Universe::narrow_klass_shift() != 0) {
1966       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1967     }
1968   } else {
1969    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1970   }
1971   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
1972   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
1973 }
1974 #endif
1975 
1976 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1977 {
1978   // This is the unverified entry point.
1979   MacroAssembler _masm(&cbuf);
1980 
1981   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
1982   Label skip;
1983   // TODO
1984   // can we avoid this skip and still use a reloc?
1985   __ br(Assembler::EQ, skip);
1986   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1987   __ bind(skip);
1988 }
1989 
1990 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1991 {
1992   return MachNode::size(ra_);
1993 }
1994 
1995 // REQUIRED EMIT CODE
1996 
1997 //=============================================================================
1998 
1999 // Emit exception handler code.
2000 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2001 {
2002   // mov rscratch1 #exception_blob_entry_point
2003   // br rscratch1
2004   // Note that the code buffer's insts_mark is always relative to insts.
2005   // That's why we must use the macroassembler to generate a handler.
2006   MacroAssembler _masm(&cbuf);
2007   address base = __ start_a_stub(size_exception_handler());
2008   if (base == NULL) {
2009     ciEnv::current()->record_failure("CodeCache is full");
2010     return 0;  // CodeBuffer::expand failed
2011   }
2012   int offset = __ offset();
2013   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2014   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2015   __ end_a_stub();
2016   return offset;
2017 }
2018 
2019 // Emit deopt handler code.
2020 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2021 {
2022   // Note that the code buffer's insts_mark is always relative to insts.
2023   // That's why we must use the macroassembler to generate a handler.
2024   MacroAssembler _masm(&cbuf);
2025   address base = __ start_a_stub(size_deopt_handler());
2026   if (base == NULL) {
2027     ciEnv::current()->record_failure("CodeCache is full");
2028     return 0;  // CodeBuffer::expand failed
2029   }
2030   int offset = __ offset();
2031 
2032   __ adr(lr, __ pc());
2033   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2034 
2035   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
2036   __ end_a_stub();
2037   return offset;
2038 }
2039 
2040 // REQUIRED MATCHER CODE
2041 
2042 //=============================================================================
2043 
2044 const bool Matcher::match_rule_supported(int opcode) {
2045 
2046   switch (opcode) {
2047   default:
2048     break;
2049   }
2050 
2051   if (!has_match_rule(opcode)) {
2052     return false;
2053   }
2054 
2055   return true;  // Per default match rules are supported.
2056 }
2057 
2058 const bool Matcher::match_rule_supported_vector(int opcode, int vlen) {
2059 
2060   // TODO
2061   // identify extra cases that we might want to provide match rules for
2062   // e.g. Op_ vector nodes and other intrinsics while guarding with vlen
2063   bool ret_value = match_rule_supported(opcode);
2064   // Add rules here.
2065 
2066   return ret_value;  // Per default match rules are supported.
2067 }
2068 
2069 const bool Matcher::has_predicated_vectors(void) {
2070   return false;
2071 }
2072 
2073 const int Matcher::float_pressure(int default_pressure_threshold) {
2074   return default_pressure_threshold;
2075 }
2076 
2077 int Matcher::regnum_to_fpu_offset(int regnum)
2078 {
2079   Unimplemented();
2080   return 0;
2081 }
2082 
2083 // Is this branch offset short enough that a short branch can be used?
2084 //
2085 // NOTE: If the platform does not provide any short branch variants, then
2086 //       this method should return false for offset 0.
2087 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2088   // The passed offset is relative to address of the branch.
2089 
2090   return (-32768 <= offset && offset < 32768);
2091 }
2092 
2093 const bool Matcher::isSimpleConstant64(jlong value) {
2094   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
2095   // Probably always true, even if a temp register is required.
2096   return true;
2097 }
2098 
2099 // true just means we have fast l2f conversion
2100 const bool Matcher::convL2FSupported(void) {
2101   return true;
2102 }
2103 
2104 // Vector width in bytes.
2105 const int Matcher::vector_width_in_bytes(BasicType bt) {
2106   int size = MIN2(16,(int)MaxVectorSize);
2107   // Minimum 2 values in vector
2108   if (size < 2*type2aelembytes(bt)) size = 0;
2109   // But never < 4
2110   if (size < 4) size = 0;
2111   return size;
2112 }
2113 
2114 // Limits on vector size (number of elements) loaded into vector.
2115 const int Matcher::max_vector_size(const BasicType bt) {
2116   return vector_width_in_bytes(bt)/type2aelembytes(bt);
2117 }
2118 const int Matcher::min_vector_size(const BasicType bt) {
2119 //  For the moment limit the vector size to 8 bytes
2120     int size = 8 / type2aelembytes(bt);
2121     if (size < 2) size = 2;
2122     return size;
2123 }
2124 
2125 // Vector ideal reg.
2126 const uint Matcher::vector_ideal_reg(int len) {
2127   switch(len) {
2128     case  8: return Op_VecD;
2129     case 16: return Op_VecX;
2130   }
2131   ShouldNotReachHere();
2132   return 0;
2133 }
2134 
2135 const uint Matcher::vector_shift_count_ideal_reg(int size) {
2136   switch(size) {
2137     case  8: return Op_VecD;
2138     case 16: return Op_VecX;
2139   }
2140   ShouldNotReachHere();
2141   return 0;
2142 }
2143 
2144 // AES support not yet implemented
2145 const bool Matcher::pass_original_key_for_aes() {
2146   return false;
2147 }
2148 
2149 // aarch64 supports misaligned vectors store/load.
2150 const bool Matcher::misaligned_vectors_ok() {
2151   return true;
2152 }
2153 
2154 // false => size gets scaled to BytesPerLong, ok.
2155 const bool Matcher::init_array_count_is_in_bytes = false;
2156 
2157 // Use conditional move (CMOVL)
2158 const int Matcher::long_cmove_cost() {
2159   // long cmoves are no more expensive than int cmoves
2160   return 0;
2161 }
2162 
2163 const int Matcher::float_cmove_cost() {
2164   // float cmoves are no more expensive than int cmoves
2165   return 0;
2166 }
2167 
2168 // Does the CPU require late expand (see block.cpp for description of late expand)?
2169 const bool Matcher::require_postalloc_expand = false;
2170 
2171 // Do we need to mask the count passed to shift instructions or does
2172 // the cpu only look at the lower 5/6 bits anyway?
2173 const bool Matcher::need_masked_shift_count = false;
2174 
2175 // This affects two different things:
2176 //  - how Decode nodes are matched
2177 //  - how ImplicitNullCheck opportunities are recognized
2178 // If true, the matcher will try to remove all Decodes and match them
2179 // (as operands) into nodes. NullChecks are not prepared to deal with
2180 // Decodes by final_graph_reshaping().
2181 // If false, final_graph_reshaping() forces the decode behind the Cmp
2182 // for a NullCheck. The matcher matches the Decode node into a register.
2183 // Implicit_null_check optimization moves the Decode along with the
2184 // memory operation back up before the NullCheck.
2185 bool Matcher::narrow_oop_use_complex_address() {
2186   return Universe::narrow_oop_shift() == 0;
2187 }
2188 
2189 bool Matcher::narrow_klass_use_complex_address() {
2190 // TODO
2191 // decide whether we need to set this to true
2192   return false;
2193 }
2194 
2195 bool Matcher::const_oop_prefer_decode() {
2196   // Prefer ConN+DecodeN over ConP in simple compressed oops mode.
2197   return Universe::narrow_oop_base() == NULL;
2198 }
2199 
2200 bool Matcher::const_klass_prefer_decode() {
2201   // Prefer ConNKlass+DecodeNKlass over ConP in simple compressed klass mode.
2202   return Universe::narrow_klass_base() == NULL;
2203 }
2204 
2205 // Is it better to copy float constants, or load them directly from
2206 // memory?  Intel can load a float constant from a direct address,
2207 // requiring no extra registers.  Most RISCs will have to materialize
2208 // an address into a register first, so they would do better to copy
2209 // the constant from stack.
2210 const bool Matcher::rematerialize_float_constants = false;
2211 
2212 // If CPU can load and store mis-aligned doubles directly then no
2213 // fixup is needed.  Else we split the double into 2 integer pieces
2214 // and move it piece-by-piece.  Only happens when passing doubles into
2215 // C code as the Java calling convention forces doubles to be aligned.
2216 const bool Matcher::misaligned_doubles_ok = true;
2217 
2218 // No-op on amd64
2219 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {
2220   Unimplemented();
2221 }
2222 
2223 // Advertise here if the CPU requires explicit rounding operations to
2224 // implement the UseStrictFP mode.
2225 const bool Matcher::strict_fp_requires_explicit_rounding = false;
2226 
2227 // Are floats converted to double when stored to stack during
2228 // deoptimization?
2229 bool Matcher::float_in_double() { return false; }
2230 
2231 // Do ints take an entire long register or just half?
2232 // The relevant question is how the int is callee-saved:
2233 // the whole long is written but de-opt'ing will have to extract
2234 // the relevant 32 bits.
2235 const bool Matcher::int_in_long = true;
2236 
2237 // Return whether or not this register is ever used as an argument.
2238 // This function is used on startup to build the trampoline stubs in
2239 // generateOptoStub.  Registers not mentioned will be killed by the VM
2240 // call in the trampoline, and arguments in those registers not be
2241 // available to the callee.
2242 bool Matcher::can_be_java_arg(int reg)
2243 {
2244   return
2245     reg ==  R0_num || reg == R0_H_num ||
2246     reg ==  R1_num || reg == R1_H_num ||
2247     reg ==  R2_num || reg == R2_H_num ||
2248     reg ==  R3_num || reg == R3_H_num ||
2249     reg ==  R4_num || reg == R4_H_num ||
2250     reg ==  R5_num || reg == R5_H_num ||
2251     reg ==  R6_num || reg == R6_H_num ||
2252     reg ==  R7_num || reg == R7_H_num ||
2253     reg ==  V0_num || reg == V0_H_num ||
2254     reg ==  V1_num || reg == V1_H_num ||
2255     reg ==  V2_num || reg == V2_H_num ||
2256     reg ==  V3_num || reg == V3_H_num ||
2257     reg ==  V4_num || reg == V4_H_num ||
2258     reg ==  V5_num || reg == V5_H_num ||
2259     reg ==  V6_num || reg == V6_H_num ||
2260     reg ==  V7_num || reg == V7_H_num;
2261 }
2262 
2263 bool Matcher::is_spillable_arg(int reg)
2264 {
2265   return can_be_java_arg(reg);
2266 }
2267 
2268 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2269   return false;
2270 }
2271 
2272 RegMask Matcher::divI_proj_mask() {
2273   ShouldNotReachHere();
2274   return RegMask();
2275 }
2276 
2277 // Register for MODI projection of divmodI.
2278 RegMask Matcher::modI_proj_mask() {
2279   ShouldNotReachHere();
2280   return RegMask();
2281 }
2282 
2283 // Register for DIVL projection of divmodL.
2284 RegMask Matcher::divL_proj_mask() {
2285   ShouldNotReachHere();
2286   return RegMask();
2287 }
2288 
2289 // Register for MODL projection of divmodL.
2290 RegMask Matcher::modL_proj_mask() {
2291   ShouldNotReachHere();
2292   return RegMask();
2293 }
2294 
2295 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2296   return FP_REG_mask();
2297 }
2298 
2299 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2300   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2301     Node* u = addp->fast_out(i);
2302     if (u->is_Mem()) {
2303       int opsize = u->as_Mem()->memory_size();
2304       assert(opsize > 0, "unexpected memory operand size");
2305       if (u->as_Mem()->memory_size() != (1<<shift)) {
2306         return false;
2307       }
2308     }
2309   }
2310   return true;
2311 }
2312 
2313 const bool Matcher::convi2l_type_required = false;
2314 
2315 // Should the Matcher clone shifts on addressing modes, expecting them
2316 // to be subsumed into complex addressing expressions or compute them
2317 // into registers?
2318 bool Matcher::clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2319   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2320     return true;
2321   }
2322 
2323   Node *off = m->in(AddPNode::Offset);
2324   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2325       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2326       // Are there other uses besides address expressions?
2327       !is_visited(off)) {
2328     address_visited.set(off->_idx); // Flag as address_visited
2329     mstack.push(off->in(2), Visit);
2330     Node *conv = off->in(1);
2331     if (conv->Opcode() == Op_ConvI2L &&
2332         // Are there other uses besides address expressions?
2333         !is_visited(conv)) {
2334       address_visited.set(conv->_idx); // Flag as address_visited
2335       mstack.push(conv->in(1), Pre_Visit);
2336     } else {
2337       mstack.push(conv, Pre_Visit);
2338     }
2339     address_visited.test_set(m->_idx); // Flag as address_visited
2340     mstack.push(m->in(AddPNode::Address), Pre_Visit);
2341     mstack.push(m->in(AddPNode::Base), Pre_Visit);
2342     return true;
2343   } else if (off->Opcode() == Op_ConvI2L &&
2344              // Are there other uses besides address expressions?
2345              !is_visited(off)) {
2346     address_visited.test_set(m->_idx); // Flag as address_visited
2347     address_visited.set(off->_idx); // Flag as address_visited
2348     mstack.push(off->in(1), Pre_Visit);
2349     mstack.push(m->in(AddPNode::Address), Pre_Visit);
2350     mstack.push(m->in(AddPNode::Base), Pre_Visit);
2351     return true;
2352   }
2353   return false;
2354 }
2355 
2356 void Compile::reshape_address(AddPNode* addp) {
2357 }
2358 
2359 // helper for encoding java_to_runtime calls on sim
2360 //
2361 // this is needed to compute the extra arguments required when
2362 // planting a call to the simulator blrt instruction. the TypeFunc
2363 // can be queried to identify the counts for integral, and floating
2364 // arguments and the return type
2365 
2366 static void getCallInfo(const TypeFunc *tf, int &gpcnt, int &fpcnt, int &rtype)
2367 {
2368   int gps = 0;
2369   int fps = 0;
2370   const TypeTuple *domain = tf->domain();
2371   int max = domain->cnt();
2372   for (int i = TypeFunc::Parms; i < max; i++) {
2373     const Type *t = domain->field_at(i);
2374     switch(t->basic_type()) {
2375     case T_FLOAT:
2376     case T_DOUBLE:
2377       fps++;
2378     default:
2379       gps++;
2380     }
2381   }
2382   gpcnt = gps;
2383   fpcnt = fps;
2384   BasicType rt = tf->return_type();
2385   switch (rt) {
2386   case T_VOID:
2387     rtype = MacroAssembler::ret_type_void;
2388     break;
2389   default:
2390     rtype = MacroAssembler::ret_type_integral;
2391     break;
2392   case T_FLOAT:
2393     rtype = MacroAssembler::ret_type_float;
2394     break;
2395   case T_DOUBLE:
2396     rtype = MacroAssembler::ret_type_double;
2397     break;
2398   }
2399 }
2400 
2401 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
2402   MacroAssembler _masm(&cbuf);                                          \
2403   {                                                                     \
2404     guarantee(INDEX == -1, "mode not permitted for volatile");          \
2405     guarantee(DISP == 0, "mode not permitted for volatile");            \
2406     guarantee(SCALE == 0, "mode not permitted for volatile");           \
2407     __ INSN(REG, as_Register(BASE));                                    \
2408   }
2409 
2410 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2411 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2412 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2413                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
2414 
2415   // Used for all non-volatile memory accesses.  The use of
2416   // $mem->opcode() to discover whether this pattern uses sign-extended
2417   // offsets is something of a kludge.
2418   static void loadStore(MacroAssembler masm, mem_insn insn,
2419                          Register reg, int opcode,
2420                          Register base, int index, int size, int disp)
2421   {
2422     Address::extend scale;
2423 
2424     // Hooboy, this is fugly.  We need a way to communicate to the
2425     // encoder that the index needs to be sign extended, so we have to
2426     // enumerate all the cases.
2427     switch (opcode) {
2428     case INDINDEXSCALEDI2L:
2429     case INDINDEXSCALEDI2LN:
2430     case INDINDEXI2L:
2431     case INDINDEXI2LN:
2432       scale = Address::sxtw(size);
2433       break;
2434     default:
2435       scale = Address::lsl(size);
2436     }
2437 
2438     if (index == -1) {
2439       (masm.*insn)(reg, Address(base, disp));
2440     } else {
2441       assert(disp == 0, "unsupported address mode: disp = %d", disp);
2442       (masm.*insn)(reg, Address(base, as_Register(index), scale));
2443     }
2444   }
2445 
2446   static void loadStore(MacroAssembler masm, mem_float_insn insn,
2447                          FloatRegister reg, int opcode,
2448                          Register base, int index, int size, int disp)
2449   {
2450     Address::extend scale;
2451 
2452     switch (opcode) {
2453     case INDINDEXSCALEDI2L:
2454     case INDINDEXSCALEDI2LN:
2455       scale = Address::sxtw(size);
2456       break;
2457     default:
2458       scale = Address::lsl(size);
2459     }
2460 
2461      if (index == -1) {
2462       (masm.*insn)(reg, Address(base, disp));
2463     } else {
2464       assert(disp == 0, "unsupported address mode: disp = %d", disp);
2465       (masm.*insn)(reg, Address(base, as_Register(index), scale));
2466     }
2467   }
2468 
2469   static void loadStore(MacroAssembler masm, mem_vector_insn insn,
2470                          FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2471                          int opcode, Register base, int index, int size, int disp)
2472   {
2473     if (index == -1) {
2474       (masm.*insn)(reg, T, Address(base, disp));
2475     } else {
2476       assert(disp == 0, "unsupported address mode");
2477       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2478     }
2479   }
2480 
2481 %}
2482 
2483 
2484 
2485 //----------ENCODING BLOCK-----------------------------------------------------
2486 // This block specifies the encoding classes used by the compiler to
2487 // output byte streams.  Encoding classes are parameterized macros
2488 // used by Machine Instruction Nodes in order to generate the bit
2489 // encoding of the instruction.  Operands specify their base encoding
2490 // interface with the interface keyword.  There are currently
2491 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2492 // COND_INTER.  REG_INTER causes an operand to generate a function
2493 // which returns its register number when queried.  CONST_INTER causes
2494 // an operand to generate a function which returns the value of the
2495 // constant when queried.  MEMORY_INTER causes an operand to generate
2496 // four functions which return the Base Register, the Index Register,
2497 // the Scale Value, and the Offset Value of the operand when queried.
2498 // COND_INTER causes an operand to generate six functions which return
2499 // the encoding code (ie - encoding bits for the instruction)
2500 // associated with each basic boolean condition for a conditional
2501 // instruction.
2502 //
2503 // Instructions specify two basic values for encoding.  Again, a
2504 // function is available to check if the constant displacement is an
2505 // oop. They use the ins_encode keyword to specify their encoding
2506 // classes (which must be a sequence of enc_class names, and their
2507 // parameters, specified in the encoding block), and they use the
2508 // opcode keyword to specify, in order, their primary, secondary, and
2509 // tertiary opcode.  Only the opcode sections which a particular
2510 // instruction needs for encoding need to be specified.
2511 encode %{
2512   // Build emit functions for each basic byte or larger field in the
2513   // intel encoding scheme (opcode, rm, sib, immediate), and call them
2514   // from C++ code in the enc_class source block.  Emit functions will
2515   // live in the main source block for now.  In future, we can
2516   // generalize this by adding a syntax that specifies the sizes of
2517   // fields in an order, so that the adlc can build the emit functions
2518   // automagically
2519 
2520   // catch all for unimplemented encodings
2521   enc_class enc_unimplemented %{
2522     MacroAssembler _masm(&cbuf);
2523     __ unimplemented("C2 catch all");
2524   %}
2525 
2526   // BEGIN Non-volatile memory access
2527 
2528   enc_class aarch64_enc_ldrsbw(iRegI dst, memory mem) %{
2529     Register dst_reg = as_Register($dst$$reg);
2530     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2531                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2532   %}
2533 
2534   enc_class aarch64_enc_ldrsb(iRegI dst, memory mem) %{
2535     Register dst_reg = as_Register($dst$$reg);
2536     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2537                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2538   %}
2539 
2540   enc_class aarch64_enc_ldrb(iRegI dst, memory mem) %{
2541     Register dst_reg = as_Register($dst$$reg);
2542     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2543                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2544   %}
2545 
2546   enc_class aarch64_enc_ldrb(iRegL dst, memory mem) %{
2547     Register dst_reg = as_Register($dst$$reg);
2548     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2549                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2550   %}
2551 
2552   enc_class aarch64_enc_ldrshw(iRegI dst, memory mem) %{
2553     Register dst_reg = as_Register($dst$$reg);
2554     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2555                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2556   %}
2557 
2558   enc_class aarch64_enc_ldrsh(iRegI dst, memory mem) %{
2559     Register dst_reg = as_Register($dst$$reg);
2560     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2561                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2562   %}
2563 
2564   enc_class aarch64_enc_ldrh(iRegI dst, memory mem) %{
2565     Register dst_reg = as_Register($dst$$reg);
2566     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2567                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2568   %}
2569 
2570   enc_class aarch64_enc_ldrh(iRegL dst, memory mem) %{
2571     Register dst_reg = as_Register($dst$$reg);
2572     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2573                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2574   %}
2575 
2576   enc_class aarch64_enc_ldrw(iRegI dst, memory mem) %{
2577     Register dst_reg = as_Register($dst$$reg);
2578     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2579                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2580   %}
2581 
2582   enc_class aarch64_enc_ldrw(iRegL dst, memory mem) %{
2583     Register dst_reg = as_Register($dst$$reg);
2584     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2585                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2586   %}
2587 
2588   enc_class aarch64_enc_ldrsw(iRegL dst, memory mem) %{
2589     Register dst_reg = as_Register($dst$$reg);
2590     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2591                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2592   %}
2593 
2594   enc_class aarch64_enc_ldr(iRegL dst, memory mem) %{
2595     Register dst_reg = as_Register($dst$$reg);
2596     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2597                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2598   %}
2599 
2600   enc_class aarch64_enc_ldrs(vRegF dst, memory mem) %{
2601     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2602     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2603                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2604   %}
2605 
2606   enc_class aarch64_enc_ldrd(vRegD dst, memory mem) %{
2607     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2608     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2609                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2610   %}
2611 
2612   enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
2613     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2614     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
2615        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2616   %}
2617 
2618   enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
2619     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2620     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
2621        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2622   %}
2623 
2624   enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
2625     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2626     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
2627        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2628   %}
2629 
2630   enc_class aarch64_enc_strb(iRegI src, memory mem) %{
2631     Register src_reg = as_Register($src$$reg);
2632     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2633                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2634   %}
2635 
2636   enc_class aarch64_enc_strb0(memory mem) %{
2637     MacroAssembler _masm(&cbuf);
2638     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2639                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2640   %}
2641 
2642   enc_class aarch64_enc_strb0_ordered(memory mem) %{
2643     MacroAssembler _masm(&cbuf);
2644     __ membar(Assembler::StoreStore);
2645     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2646                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2647   %}
2648 
2649   enc_class aarch64_enc_strh(iRegI src, memory mem) %{
2650     Register src_reg = as_Register($src$$reg);
2651     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2652                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2653   %}
2654 
2655   enc_class aarch64_enc_strh0(memory mem) %{
2656     MacroAssembler _masm(&cbuf);
2657     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2658                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2659   %}
2660 
2661   enc_class aarch64_enc_strw(iRegI src, memory mem) %{
2662     Register src_reg = as_Register($src$$reg);
2663     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2664                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2665   %}
2666 
2667   enc_class aarch64_enc_strw0(memory mem) %{
2668     MacroAssembler _masm(&cbuf);
2669     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2670                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2671   %}
2672 
2673   enc_class aarch64_enc_str(iRegL src, memory mem) %{
2674     Register src_reg = as_Register($src$$reg);
2675     // we sometimes get asked to store the stack pointer into the
2676     // current thread -- we cannot do that directly on AArch64
2677     if (src_reg == r31_sp) {
2678       MacroAssembler _masm(&cbuf);
2679       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2680       __ mov(rscratch2, sp);
2681       src_reg = rscratch2;
2682     }
2683     loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
2684                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2685   %}
2686 
2687   enc_class aarch64_enc_str0(memory mem) %{
2688     MacroAssembler _masm(&cbuf);
2689     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
2690                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2691   %}
2692 
2693   enc_class aarch64_enc_strs(vRegF src, memory mem) %{
2694     FloatRegister src_reg = as_FloatRegister($src$$reg);
2695     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
2696                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2697   %}
2698 
2699   enc_class aarch64_enc_strd(vRegD src, memory mem) %{
2700     FloatRegister src_reg = as_FloatRegister($src$$reg);
2701     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
2702                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2703   %}
2704 
2705   enc_class aarch64_enc_strvS(vecD src, memory mem) %{
2706     FloatRegister src_reg = as_FloatRegister($src$$reg);
2707     loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
2708        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2709   %}
2710 
2711   enc_class aarch64_enc_strvD(vecD src, memory mem) %{
2712     FloatRegister src_reg = as_FloatRegister($src$$reg);
2713     loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
2714        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2715   %}
2716 
2717   enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
2718     FloatRegister src_reg = as_FloatRegister($src$$reg);
2719     loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
2720        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2721   %}
2722 
2723   // END Non-volatile memory access
2724 
2725   // volatile loads and stores
2726 
2727   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
2728     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2729                  rscratch1, stlrb);
2730   %}
2731 
2732   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
2733     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2734                  rscratch1, stlrh);
2735   %}
2736 
2737   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
2738     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2739                  rscratch1, stlrw);
2740   %}
2741 
2742 
2743   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
2744     Register dst_reg = as_Register($dst$$reg);
2745     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2746              rscratch1, ldarb);
2747     __ sxtbw(dst_reg, dst_reg);
2748   %}
2749 
2750   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
2751     Register dst_reg = as_Register($dst$$reg);
2752     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2753              rscratch1, ldarb);
2754     __ sxtb(dst_reg, dst_reg);
2755   %}
2756 
2757   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
2758     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2759              rscratch1, ldarb);
2760   %}
2761 
2762   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
2763     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2764              rscratch1, ldarb);
2765   %}
2766 
2767   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
2768     Register dst_reg = as_Register($dst$$reg);
2769     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2770              rscratch1, ldarh);
2771     __ sxthw(dst_reg, dst_reg);
2772   %}
2773 
2774   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
2775     Register dst_reg = as_Register($dst$$reg);
2776     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2777              rscratch1, ldarh);
2778     __ sxth(dst_reg, dst_reg);
2779   %}
2780 
2781   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
2782     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2783              rscratch1, ldarh);
2784   %}
2785 
2786   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
2787     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2788              rscratch1, ldarh);
2789   %}
2790 
2791   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
2792     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2793              rscratch1, ldarw);
2794   %}
2795 
2796   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
2797     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2798              rscratch1, ldarw);
2799   %}
2800 
2801   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
2802     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2803              rscratch1, ldar);
2804   %}
2805 
2806   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
2807     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2808              rscratch1, ldarw);
2809     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
2810   %}
2811 
2812   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
2813     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2814              rscratch1, ldar);
2815     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
2816   %}
2817 
2818   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
2819     Register src_reg = as_Register($src$$reg);
2820     // we sometimes get asked to store the stack pointer into the
2821     // current thread -- we cannot do that directly on AArch64
2822     if (src_reg == r31_sp) {
2823         MacroAssembler _masm(&cbuf);
2824       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2825       __ mov(rscratch2, sp);
2826       src_reg = rscratch2;
2827     }
2828     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2829                  rscratch1, stlr);
2830   %}
2831 
2832   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
2833     {
2834       MacroAssembler _masm(&cbuf);
2835       FloatRegister src_reg = as_FloatRegister($src$$reg);
2836       __ fmovs(rscratch2, src_reg);
2837     }
2838     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2839                  rscratch1, stlrw);
2840   %}
2841 
2842   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
2843     {
2844       MacroAssembler _masm(&cbuf);
2845       FloatRegister src_reg = as_FloatRegister($src$$reg);
2846       __ fmovd(rscratch2, src_reg);
2847     }
2848     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2849                  rscratch1, stlr);
2850   %}
2851 
2852   // synchronized read/update encodings
2853 
2854   enc_class aarch64_enc_ldaxr(iRegL dst, memory mem) %{
2855     MacroAssembler _masm(&cbuf);
2856     Register dst_reg = as_Register($dst$$reg);
2857     Register base = as_Register($mem$$base);
2858     int index = $mem$$index;
2859     int scale = $mem$$scale;
2860     int disp = $mem$$disp;
2861     if (index == -1) {
2862        if (disp != 0) {
2863         __ lea(rscratch1, Address(base, disp));
2864         __ ldaxr(dst_reg, rscratch1);
2865       } else {
2866         // TODO
2867         // should we ever get anything other than this case?
2868         __ ldaxr(dst_reg, base);
2869       }
2870     } else {
2871       Register index_reg = as_Register(index);
2872       if (disp == 0) {
2873         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
2874         __ ldaxr(dst_reg, rscratch1);
2875       } else {
2876         __ lea(rscratch1, Address(base, disp));
2877         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
2878         __ ldaxr(dst_reg, rscratch1);
2879       }
2880     }
2881   %}
2882 
2883   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory mem) %{
2884     MacroAssembler _masm(&cbuf);
2885     Register src_reg = as_Register($src$$reg);
2886     Register base = as_Register($mem$$base);
2887     int index = $mem$$index;
2888     int scale = $mem$$scale;
2889     int disp = $mem$$disp;
2890     if (index == -1) {
2891        if (disp != 0) {
2892         __ lea(rscratch2, Address(base, disp));
2893         __ stlxr(rscratch1, src_reg, rscratch2);
2894       } else {
2895         // TODO
2896         // should we ever get anything other than this case?
2897         __ stlxr(rscratch1, src_reg, base);
2898       }
2899     } else {
2900       Register index_reg = as_Register(index);
2901       if (disp == 0) {
2902         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
2903         __ stlxr(rscratch1, src_reg, rscratch2);
2904       } else {
2905         __ lea(rscratch2, Address(base, disp));
2906         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
2907         __ stlxr(rscratch1, src_reg, rscratch2);
2908       }
2909     }
2910     __ cmpw(rscratch1, zr);
2911   %}
2912 
2913   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
2914     MacroAssembler _masm(&cbuf);
2915     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2916     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2917                Assembler::xword, /*acquire*/ false, /*release*/ true,
2918                /*weak*/ false, noreg);
2919   %}
2920 
2921   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2922     MacroAssembler _masm(&cbuf);
2923     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2924     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2925                Assembler::word, /*acquire*/ false, /*release*/ true,
2926                /*weak*/ false, noreg);
2927   %}
2928 
2929   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2930     MacroAssembler _masm(&cbuf);
2931     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2932     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2933                Assembler::halfword, /*acquire*/ false, /*release*/ true,
2934                /*weak*/ false, noreg);
2935   %}
2936 
2937   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2938     MacroAssembler _masm(&cbuf);
2939     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2940     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2941                Assembler::byte, /*acquire*/ false, /*release*/ true,
2942                /*weak*/ false, noreg);
2943   %}
2944 
2945 
2946   // The only difference between aarch64_enc_cmpxchg and
2947   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
2948   // CompareAndSwap sequence to serve as a barrier on acquiring a
2949   // lock.
2950   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
2951     MacroAssembler _masm(&cbuf);
2952     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2953     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2954                Assembler::xword, /*acquire*/ true, /*release*/ true,
2955                /*weak*/ false, noreg);
2956   %}
2957 
2958   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2959     MacroAssembler _masm(&cbuf);
2960     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2961     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2962                Assembler::word, /*acquire*/ true, /*release*/ true,
2963                /*weak*/ false, noreg);
2964   %}
2965 
2966   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2967     MacroAssembler _masm(&cbuf);
2968     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2969     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2970                Assembler::halfword, /*acquire*/ true, /*release*/ true,
2971                /*weak*/ false, noreg);
2972   %}
2973 
2974   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2975     MacroAssembler _masm(&cbuf);
2976     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2977     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2978                Assembler::byte, /*acquire*/ true, /*release*/ true,
2979                /*weak*/ false, noreg);
2980   %}
2981 
2982   // auxiliary used for CompareAndSwapX to set result register
2983   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
2984     MacroAssembler _masm(&cbuf);
2985     Register res_reg = as_Register($res$$reg);
2986     __ cset(res_reg, Assembler::EQ);
2987   %}
2988 
2989   // prefetch encodings
2990 
2991   enc_class aarch64_enc_prefetchw(memory mem) %{
2992     MacroAssembler _masm(&cbuf);
2993     Register base = as_Register($mem$$base);
2994     int index = $mem$$index;
2995     int scale = $mem$$scale;
2996     int disp = $mem$$disp;
2997     if (index == -1) {
2998       __ prfm(Address(base, disp), PSTL1KEEP);
2999     } else {
3000       Register index_reg = as_Register(index);
3001       if (disp == 0) {
3002         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3003       } else {
3004         __ lea(rscratch1, Address(base, disp));
3005         __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3006       }
3007     }
3008   %}
3009 
3010   /// mov envcodings
3011 
3012   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3013     MacroAssembler _masm(&cbuf);
3014     u_int32_t con = (u_int32_t)$src$$constant;
3015     Register dst_reg = as_Register($dst$$reg);
3016     if (con == 0) {
3017       __ movw(dst_reg, zr);
3018     } else {
3019       __ movw(dst_reg, con);
3020     }
3021   %}
3022 
3023   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3024     MacroAssembler _masm(&cbuf);
3025     Register dst_reg = as_Register($dst$$reg);
3026     u_int64_t con = (u_int64_t)$src$$constant;
3027     if (con == 0) {
3028       __ mov(dst_reg, zr);
3029     } else {
3030       __ mov(dst_reg, con);
3031     }
3032   %}
3033 
3034   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3035     MacroAssembler _masm(&cbuf);
3036     Register dst_reg = as_Register($dst$$reg);
3037     address con = (address)$src$$constant;
3038     if (con == NULL || con == (address)1) {
3039       ShouldNotReachHere();
3040     } else {
3041       relocInfo::relocType rtype = $src->constant_reloc();
3042       if (rtype == relocInfo::oop_type) {
3043         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
3044       } else if (rtype == relocInfo::metadata_type) {
3045         __ mov_metadata(dst_reg, (Metadata*)con);
3046       } else {
3047         assert(rtype == relocInfo::none, "unexpected reloc type");
3048         if (con < (address)(uintptr_t)os::vm_page_size()) {
3049           __ mov(dst_reg, con);
3050         } else {
3051           unsigned long offset;
3052           __ adrp(dst_reg, con, offset);
3053           __ add(dst_reg, dst_reg, offset);
3054         }
3055       }
3056     }
3057   %}
3058 
3059   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3060     MacroAssembler _masm(&cbuf);
3061     Register dst_reg = as_Register($dst$$reg);
3062     __ mov(dst_reg, zr);
3063   %}
3064 
3065   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3066     MacroAssembler _masm(&cbuf);
3067     Register dst_reg = as_Register($dst$$reg);
3068     __ mov(dst_reg, (u_int64_t)1);
3069   %}
3070 
3071   enc_class aarch64_enc_mov_poll_page(iRegP dst, immPollPage src) %{
3072     MacroAssembler _masm(&cbuf);
3073     address page = (address)$src$$constant;
3074     Register dst_reg = as_Register($dst$$reg);
3075     unsigned long off;
3076     __ adrp(dst_reg, Address(page, relocInfo::poll_type), off);
3077     assert(off == 0, "assumed offset == 0");
3078   %}
3079 
3080   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3081     MacroAssembler _masm(&cbuf);
3082     __ load_byte_map_base($dst$$Register);
3083   %}
3084 
3085   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3086     MacroAssembler _masm(&cbuf);
3087     Register dst_reg = as_Register($dst$$reg);
3088     address con = (address)$src$$constant;
3089     if (con == NULL) {
3090       ShouldNotReachHere();
3091     } else {
3092       relocInfo::relocType rtype = $src->constant_reloc();
3093       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3094       __ set_narrow_oop(dst_reg, (jobject)con);
3095     }
3096   %}
3097 
3098   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3099     MacroAssembler _masm(&cbuf);
3100     Register dst_reg = as_Register($dst$$reg);
3101     __ mov(dst_reg, zr);
3102   %}
3103 
3104   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3105     MacroAssembler _masm(&cbuf);
3106     Register dst_reg = as_Register($dst$$reg);
3107     address con = (address)$src$$constant;
3108     if (con == NULL) {
3109       ShouldNotReachHere();
3110     } else {
3111       relocInfo::relocType rtype = $src->constant_reloc();
3112       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3113       __ set_narrow_klass(dst_reg, (Klass *)con);
3114     }
3115   %}
3116 
3117   // arithmetic encodings
3118 
3119   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3120     MacroAssembler _masm(&cbuf);
3121     Register dst_reg = as_Register($dst$$reg);
3122     Register src_reg = as_Register($src1$$reg);
3123     int32_t con = (int32_t)$src2$$constant;
3124     // add has primary == 0, subtract has primary == 1
3125     if ($primary) { con = -con; }
3126     if (con < 0) {
3127       __ subw(dst_reg, src_reg, -con);
3128     } else {
3129       __ addw(dst_reg, src_reg, con);
3130     }
3131   %}
3132 
3133   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3134     MacroAssembler _masm(&cbuf);
3135     Register dst_reg = as_Register($dst$$reg);
3136     Register src_reg = as_Register($src1$$reg);
3137     int32_t con = (int32_t)$src2$$constant;
3138     // add has primary == 0, subtract has primary == 1
3139     if ($primary) { con = -con; }
3140     if (con < 0) {
3141       __ sub(dst_reg, src_reg, -con);
3142     } else {
3143       __ add(dst_reg, src_reg, con);
3144     }
3145   %}
3146 
3147   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3148     MacroAssembler _masm(&cbuf);
3149    Register dst_reg = as_Register($dst$$reg);
3150    Register src1_reg = as_Register($src1$$reg);
3151    Register src2_reg = as_Register($src2$$reg);
3152     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3153   %}
3154 
3155   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3156     MacroAssembler _masm(&cbuf);
3157    Register dst_reg = as_Register($dst$$reg);
3158    Register src1_reg = as_Register($src1$$reg);
3159    Register src2_reg = as_Register($src2$$reg);
3160     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3161   %}
3162 
3163   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3164     MacroAssembler _masm(&cbuf);
3165    Register dst_reg = as_Register($dst$$reg);
3166    Register src1_reg = as_Register($src1$$reg);
3167    Register src2_reg = as_Register($src2$$reg);
3168     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3169   %}
3170 
3171   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3172     MacroAssembler _masm(&cbuf);
3173    Register dst_reg = as_Register($dst$$reg);
3174    Register src1_reg = as_Register($src1$$reg);
3175    Register src2_reg = as_Register($src2$$reg);
3176     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3177   %}
3178 
3179   // compare instruction encodings
3180 
3181   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3182     MacroAssembler _masm(&cbuf);
3183     Register reg1 = as_Register($src1$$reg);
3184     Register reg2 = as_Register($src2$$reg);
3185     __ cmpw(reg1, reg2);
3186   %}
3187 
3188   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3189     MacroAssembler _masm(&cbuf);
3190     Register reg = as_Register($src1$$reg);
3191     int32_t val = $src2$$constant;
3192     if (val >= 0) {
3193       __ subsw(zr, reg, val);
3194     } else {
3195       __ addsw(zr, reg, -val);
3196     }
3197   %}
3198 
3199   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3200     MacroAssembler _masm(&cbuf);
3201     Register reg1 = as_Register($src1$$reg);
3202     u_int32_t val = (u_int32_t)$src2$$constant;
3203     __ movw(rscratch1, val);
3204     __ cmpw(reg1, rscratch1);
3205   %}
3206 
3207   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3208     MacroAssembler _masm(&cbuf);
3209     Register reg1 = as_Register($src1$$reg);
3210     Register reg2 = as_Register($src2$$reg);
3211     __ cmp(reg1, reg2);
3212   %}
3213 
3214   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3215     MacroAssembler _masm(&cbuf);
3216     Register reg = as_Register($src1$$reg);
3217     int64_t val = $src2$$constant;
3218     if (val >= 0) {
3219       __ subs(zr, reg, val);
3220     } else if (val != -val) {
3221       __ adds(zr, reg, -val);
3222     } else {
3223     // aargh, Long.MIN_VALUE is a special case
3224       __ orr(rscratch1, zr, (u_int64_t)val);
3225       __ subs(zr, reg, rscratch1);
3226     }
3227   %}
3228 
3229   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3230     MacroAssembler _masm(&cbuf);
3231     Register reg1 = as_Register($src1$$reg);
3232     u_int64_t val = (u_int64_t)$src2$$constant;
3233     __ mov(rscratch1, val);
3234     __ cmp(reg1, rscratch1);
3235   %}
3236 
3237   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3238     MacroAssembler _masm(&cbuf);
3239     Register reg1 = as_Register($src1$$reg);
3240     Register reg2 = as_Register($src2$$reg);
3241     __ cmp(reg1, reg2);
3242   %}
3243 
3244   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3245     MacroAssembler _masm(&cbuf);
3246     Register reg1 = as_Register($src1$$reg);
3247     Register reg2 = as_Register($src2$$reg);
3248     __ cmpw(reg1, reg2);
3249   %}
3250 
3251   enc_class aarch64_enc_testp(iRegP src) %{
3252     MacroAssembler _masm(&cbuf);
3253     Register reg = as_Register($src$$reg);
3254     __ cmp(reg, zr);
3255   %}
3256 
3257   enc_class aarch64_enc_testn(iRegN src) %{
3258     MacroAssembler _masm(&cbuf);
3259     Register reg = as_Register($src$$reg);
3260     __ cmpw(reg, zr);
3261   %}
3262 
3263   enc_class aarch64_enc_b(label lbl) %{
3264     MacroAssembler _masm(&cbuf);
3265     Label *L = $lbl$$label;
3266     __ b(*L);
3267   %}
3268 
3269   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3270     MacroAssembler _masm(&cbuf);
3271     Label *L = $lbl$$label;
3272     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3273   %}
3274 
3275   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3276     MacroAssembler _masm(&cbuf);
3277     Label *L = $lbl$$label;
3278     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3279   %}
3280 
3281   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3282   %{
3283      Register sub_reg = as_Register($sub$$reg);
3284      Register super_reg = as_Register($super$$reg);
3285      Register temp_reg = as_Register($temp$$reg);
3286      Register result_reg = as_Register($result$$reg);
3287 
3288      Label miss;
3289      MacroAssembler _masm(&cbuf);
3290      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3291                                      NULL, &miss,
3292                                      /*set_cond_codes:*/ true);
3293      if ($primary) {
3294        __ mov(result_reg, zr);
3295      }
3296      __ bind(miss);
3297   %}
3298 
3299   enc_class aarch64_enc_java_static_call(method meth) %{
3300     MacroAssembler _masm(&cbuf);
3301 
3302     address addr = (address)$meth$$method;
3303     address call;
3304     if (!_method) {
3305       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3306       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
3307     } else {
3308       int method_index = resolved_method_index(cbuf);
3309       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3310                                                   : static_call_Relocation::spec(method_index);
3311       call = __ trampoline_call(Address(addr, rspec), &cbuf);
3312 
3313       // Emit stub for static call
3314       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
3315       if (stub == NULL) {
3316         ciEnv::current()->record_failure("CodeCache is full");
3317         return;
3318       }
3319     }
3320     if (call == NULL) {
3321       ciEnv::current()->record_failure("CodeCache is full");
3322       return;
3323     }
3324   %}
3325 
3326   enc_class aarch64_enc_java_dynamic_call(method meth) %{
3327     MacroAssembler _masm(&cbuf);
3328     int method_index = resolved_method_index(cbuf);
3329     address call = __ ic_call((address)$meth$$method, method_index);
3330     if (call == NULL) {
3331       ciEnv::current()->record_failure("CodeCache is full");
3332       return;
3333     }
3334   %}
3335 
3336   enc_class aarch64_enc_call_epilog() %{
3337     MacroAssembler _masm(&cbuf);
3338     if (VerifyStackAtCalls) {
3339       // Check that stack depth is unchanged: find majik cookie on stack
3340       __ call_Unimplemented();
3341     }
3342   %}
3343 
3344   enc_class aarch64_enc_java_to_runtime(method meth) %{
3345     MacroAssembler _masm(&cbuf);
3346 
3347     // some calls to generated routines (arraycopy code) are scheduled
3348     // by C2 as runtime calls. if so we can call them using a br (they
3349     // will be in a reachable segment) otherwise we have to use a blrt
3350     // which loads the absolute address into a register.
3351     address entry = (address)$meth$$method;
3352     CodeBlob *cb = CodeCache::find_blob(entry);
3353     if (cb) {
3354       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3355       if (call == NULL) {
3356         ciEnv::current()->record_failure("CodeCache is full");
3357         return;
3358       }
3359     } else {
3360       int gpcnt;
3361       int fpcnt;
3362       int rtype;
3363       getCallInfo(tf(), gpcnt, fpcnt, rtype);
3364       Label retaddr;
3365       __ adr(rscratch2, retaddr);
3366       __ lea(rscratch1, RuntimeAddress(entry));
3367       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3368       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3369       __ blrt(rscratch1, gpcnt, fpcnt, rtype);
3370       __ bind(retaddr);
3371       __ add(sp, sp, 2 * wordSize);
3372     }
3373   %}
3374 
3375   enc_class aarch64_enc_rethrow() %{
3376     MacroAssembler _masm(&cbuf);
3377     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3378   %}
3379 
3380   enc_class aarch64_enc_ret() %{
3381     MacroAssembler _masm(&cbuf);
3382     __ ret(lr);
3383   %}
3384 
3385   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3386     MacroAssembler _masm(&cbuf);
3387     Register target_reg = as_Register($jump_target$$reg);
3388     __ br(target_reg);
3389   %}
3390 
3391   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3392     MacroAssembler _masm(&cbuf);
3393     Register target_reg = as_Register($jump_target$$reg);
3394     // exception oop should be in r0
3395     // ret addr has been popped into lr
3396     // callee expects it in r3
3397     __ mov(r3, lr);
3398     __ br(target_reg);
3399   %}
3400 
3401   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3402     MacroAssembler _masm(&cbuf);
3403     Register oop = as_Register($object$$reg);
3404     Register box = as_Register($box$$reg);
3405     Register disp_hdr = as_Register($tmp$$reg);
3406     Register tmp = as_Register($tmp2$$reg);
3407     Label cont;
3408     Label object_has_monitor;
3409     Label cas_failed;
3410 
3411     assert_different_registers(oop, box, tmp, disp_hdr);
3412 
3413     // Load markOop from object into displaced_header.
3414     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3415 
3416     if (UseBiasedLocking && !UseOptoBiasInlining) {
3417       __ biased_locking_enter(box, oop, disp_hdr, tmp, true, cont);
3418     }
3419 
3420     // Check for existing monitor
3421     __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor);
3422 
3423     // Set tmp to be (markOop of object | UNLOCK_VALUE).
3424     __ orr(tmp, disp_hdr, markOopDesc::unlocked_value);
3425 
3426     // Initialize the box. (Must happen before we update the object mark!)
3427     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3428 
3429     // Compare object markOop with an unlocked value (tmp) and if
3430     // equal exchange the stack address of our box with object markOop.
3431     // On failure disp_hdr contains the possibly locked markOop.
3432     __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
3433                /*release*/ true, /*weak*/ false, disp_hdr);
3434     __ br(Assembler::EQ, cont);
3435 
3436     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3437 
3438     // If the compare-and-exchange succeeded, then we found an unlocked
3439     // object, will have now locked it will continue at label cont
3440 
3441     __ bind(cas_failed);
3442     // We did not see an unlocked object so try the fast recursive case.
3443 
3444     // Check if the owner is self by comparing the value in the
3445     // markOop of object (disp_hdr) with the stack pointer.
3446     __ mov(rscratch1, sp);
3447     __ sub(disp_hdr, disp_hdr, rscratch1);
3448     __ mov(tmp, (address) (~(os::vm_page_size()-1) | (uintptr_t)markOopDesc::lock_mask_in_place));
3449     // If condition is true we are cont and hence we can store 0 as the
3450     // displaced header in the box, which indicates that it is a recursive lock.
3451     __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
3452     __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3453 
3454     __ b(cont);
3455 
3456     // Handle existing monitor.
3457     __ bind(object_has_monitor);
3458 
3459     // The object's monitor m is unlocked iff m->owner == NULL,
3460     // otherwise m->owner may contain a thread or a stack address.
3461     //
3462     // Try to CAS m->owner from NULL to current thread.
3463     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markOopDesc::monitor_value));
3464     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
3465                /*release*/ true, /*weak*/ false, noreg); // Sets flags for result
3466 
3467     // Store a non-null value into the box to avoid looking like a re-entrant
3468     // lock. The fast-path monitor unlock code checks for
3469     // markOopDesc::monitor_value so use markOopDesc::unused_mark which has the
3470     // relevant bit set, and also matches ObjectSynchronizer::slow_enter.
3471     __ mov(tmp, (address)markOopDesc::unused_mark());
3472     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3473 
3474     __ bind(cont);
3475     // flag == EQ indicates success
3476     // flag == NE indicates failure
3477   %}
3478 
3479   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3480     MacroAssembler _masm(&cbuf);
3481     Register oop = as_Register($object$$reg);
3482     Register box = as_Register($box$$reg);
3483     Register disp_hdr = as_Register($tmp$$reg);
3484     Register tmp = as_Register($tmp2$$reg);
3485     Label cont;
3486     Label object_has_monitor;
3487 
3488     assert_different_registers(oop, box, tmp, disp_hdr);
3489 
3490     if (UseBiasedLocking && !UseOptoBiasInlining) {
3491       __ biased_locking_exit(oop, tmp, cont);
3492     }
3493 
3494     // Find the lock address and load the displaced header from the stack.
3495     __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3496 
3497     // If the displaced header is 0, we have a recursive unlock.
3498     __ cmp(disp_hdr, zr);
3499     __ br(Assembler::EQ, cont);
3500 
3501     // Handle existing monitor.
3502     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3503     __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor);
3504 
3505     // Check if it is still a light weight lock, this is is true if we
3506     // see the stack address of the basicLock in the markOop of the
3507     // object.
3508 
3509     __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
3510                /*release*/ true, /*weak*/ false, tmp);
3511     __ b(cont);
3512 
3513     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3514 
3515     // Handle existing monitor.
3516     __ bind(object_has_monitor);
3517     __ add(tmp, tmp, -markOopDesc::monitor_value); // monitor
3518     __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3519     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3520     __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner.
3521     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions
3522     __ cmp(rscratch1, zr); // Sets flags for result
3523     __ br(Assembler::NE, cont);
3524 
3525     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
3526     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
3527     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3528     __ cmp(rscratch1, zr); // Sets flags for result
3529     __ cbnz(rscratch1, cont);
3530     // need a release store here
3531     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3532     __ stlr(zr, tmp); // set unowned
3533 
3534     __ bind(cont);
3535     // flag == EQ indicates success
3536     // flag == NE indicates failure
3537   %}
3538 
3539 %}
3540 
3541 //----------FRAME--------------------------------------------------------------
3542 // Definition of frame structure and management information.
3543 //
3544 //  S T A C K   L A Y O U T    Allocators stack-slot number
3545 //                             |   (to get allocators register number
3546 //  G  Owned by    |        |  v    add OptoReg::stack0())
3547 //  r   CALLER     |        |
3548 //  o     |        +--------+      pad to even-align allocators stack-slot
3549 //  w     V        |  pad0  |        numbers; owned by CALLER
3550 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
3551 //  h     ^        |   in   |  5
3552 //        |        |  args  |  4   Holes in incoming args owned by SELF
3553 //  |     |        |        |  3
3554 //  |     |        +--------+
3555 //  V     |        | old out|      Empty on Intel, window on Sparc
3556 //        |    old |preserve|      Must be even aligned.
3557 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
3558 //        |        |   in   |  3   area for Intel ret address
3559 //     Owned by    |preserve|      Empty on Sparc.
3560 //       SELF      +--------+
3561 //        |        |  pad2  |  2   pad to align old SP
3562 //        |        +--------+  1
3563 //        |        | locks  |  0
3564 //        |        +--------+----> OptoReg::stack0(), even aligned
3565 //        |        |  pad1  | 11   pad to align new SP
3566 //        |        +--------+
3567 //        |        |        | 10
3568 //        |        | spills |  9   spills
3569 //        V        |        |  8   (pad0 slot for callee)
3570 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
3571 //        ^        |  out   |  7
3572 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
3573 //     Owned by    +--------+
3574 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
3575 //        |    new |preserve|      Must be even-aligned.
3576 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
3577 //        |        |        |
3578 //
3579 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
3580 //         known from SELF's arguments and the Java calling convention.
3581 //         Region 6-7 is determined per call site.
3582 // Note 2: If the calling convention leaves holes in the incoming argument
3583 //         area, those holes are owned by SELF.  Holes in the outgoing area
3584 //         are owned by the CALLEE.  Holes should not be nessecary in the
3585 //         incoming area, as the Java calling convention is completely under
3586 //         the control of the AD file.  Doubles can be sorted and packed to
3587 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
3588 //         varargs C calling conventions.
3589 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
3590 //         even aligned with pad0 as needed.
3591 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
3592 //           (the latter is true on Intel but is it false on AArch64?)
3593 //         region 6-11 is even aligned; it may be padded out more so that
3594 //         the region from SP to FP meets the minimum stack alignment.
3595 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3596 //         alignment.  Region 11, pad1, may be dynamically extended so that
3597 //         SP meets the minimum alignment.
3598 
3599 frame %{
3600   // What direction does stack grow in (assumed to be same for C & Java)
3601   stack_direction(TOWARDS_LOW);
3602 
3603   // These three registers define part of the calling convention
3604   // between compiled code and the interpreter.
3605 
3606   // Inline Cache Register or methodOop for I2C.
3607   inline_cache_reg(R12);
3608 
3609   // Method Oop Register when calling interpreter.
3610   interpreter_method_oop_reg(R12);
3611 
3612   // Number of stack slots consumed by locking an object
3613   sync_stack_slots(2);
3614 
3615   // Compiled code's Frame Pointer
3616   frame_pointer(R31);
3617 
3618   // Interpreter stores its frame pointer in a register which is
3619   // stored to the stack by I2CAdaptors.
3620   // I2CAdaptors convert from interpreted java to compiled java.
3621   interpreter_frame_pointer(R29);
3622 
3623   // Stack alignment requirement
3624   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3625 
3626   // Number of stack slots between incoming argument block and the start of
3627   // a new frame.  The PROLOG must add this many slots to the stack.  The
3628   // EPILOG must remove this many slots. aarch64 needs two slots for
3629   // return address and fp.
3630   // TODO think this is correct but check
3631   in_preserve_stack_slots(4);
3632 
3633   // Number of outgoing stack slots killed above the out_preserve_stack_slots
3634   // for calls to C.  Supports the var-args backing area for register parms.
3635   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3636 
3637   // The after-PROLOG location of the return address.  Location of
3638   // return address specifies a type (REG or STACK) and a number
3639   // representing the register number (i.e. - use a register name) or
3640   // stack slot.
3641   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3642   // Otherwise, it is above the locks and verification slot and alignment word
3643   // TODO this may well be correct but need to check why that - 2 is there
3644   // ppc port uses 0 but we definitely need to allow for fixed_slots
3645   // which folds in the space used for monitors
3646   return_addr(STACK - 2 +
3647               align_up((Compile::current()->in_preserve_stack_slots() +
3648                         Compile::current()->fixed_slots()),
3649                        stack_alignment_in_slots()));
3650 
3651   // Body of function which returns an integer array locating
3652   // arguments either in registers or in stack slots.  Passed an array
3653   // of ideal registers called "sig" and a "length" count.  Stack-slot
3654   // offsets are based on outgoing arguments, i.e. a CALLER setting up
3655   // arguments for a CALLEE.  Incoming stack arguments are
3656   // automatically biased by the preserve_stack_slots field above.
3657 
3658   calling_convention
3659   %{
3660     // No difference between ingoing/outgoing just pass false
3661     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
3662   %}
3663 
3664   c_calling_convention
3665   %{
3666     // This is obviously always outgoing
3667     (void) SharedRuntime::c_calling_convention(sig_bt, regs, NULL, length);
3668   %}
3669 
3670   // Location of compiled Java return values.  Same as C for now.
3671   return_value
3672   %{
3673     // TODO do we allow ideal_reg == Op_RegN???
3674     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3675            "only return normal values");
3676 
3677     static const int lo[Op_RegL + 1] = { // enum name
3678       0,                                 // Op_Node
3679       0,                                 // Op_Set
3680       R0_num,                            // Op_RegN
3681       R0_num,                            // Op_RegI
3682       R0_num,                            // Op_RegP
3683       V0_num,                            // Op_RegF
3684       V0_num,                            // Op_RegD
3685       R0_num                             // Op_RegL
3686     };
3687 
3688     static const int hi[Op_RegL + 1] = { // enum name
3689       0,                                 // Op_Node
3690       0,                                 // Op_Set
3691       OptoReg::Bad,                       // Op_RegN
3692       OptoReg::Bad,                      // Op_RegI
3693       R0_H_num,                          // Op_RegP
3694       OptoReg::Bad,                      // Op_RegF
3695       V0_H_num,                          // Op_RegD
3696       R0_H_num                           // Op_RegL
3697     };
3698 
3699     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3700   %}
3701 %}
3702 
3703 //----------ATTRIBUTES---------------------------------------------------------
3704 //----------Operand Attributes-------------------------------------------------
3705 op_attrib op_cost(1);        // Required cost attribute
3706 
3707 //----------Instruction Attributes---------------------------------------------
3708 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3709 ins_attrib ins_size(32);        // Required size attribute (in bits)
3710 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3711                                 // a non-matching short branch variant
3712                                 // of some long branch?
3713 ins_attrib ins_alignment(4);    // Required alignment attribute (must
3714                                 // be a power of 2) specifies the
3715                                 // alignment that some part of the
3716                                 // instruction (not necessarily the
3717                                 // start) requires.  If > 1, a
3718                                 // compute_padding() function must be
3719                                 // provided for the instruction
3720 
3721 //----------OPERANDS-----------------------------------------------------------
3722 // Operand definitions must precede instruction definitions for correct parsing
3723 // in the ADLC because operands constitute user defined types which are used in
3724 // instruction definitions.
3725 
3726 //----------Simple Operands----------------------------------------------------
3727 
3728 // Integer operands 32 bit
3729 // 32 bit immediate
3730 operand immI()
3731 %{
3732   match(ConI);
3733 
3734   op_cost(0);
3735   format %{ %}
3736   interface(CONST_INTER);
3737 %}
3738 
3739 // 32 bit zero
3740 operand immI0()
3741 %{
3742   predicate(n->get_int() == 0);
3743   match(ConI);
3744 
3745   op_cost(0);
3746   format %{ %}
3747   interface(CONST_INTER);
3748 %}
3749 
3750 // 32 bit unit increment
3751 operand immI_1()
3752 %{
3753   predicate(n->get_int() == 1);
3754   match(ConI);
3755 
3756   op_cost(0);
3757   format %{ %}
3758   interface(CONST_INTER);
3759 %}
3760 
3761 // 32 bit unit decrement
3762 operand immI_M1()
3763 %{
3764   predicate(n->get_int() == -1);
3765   match(ConI);
3766 
3767   op_cost(0);
3768   format %{ %}
3769   interface(CONST_INTER);
3770 %}
3771 
3772 // Shift values for add/sub extension shift
3773 operand immIExt()
3774 %{
3775   predicate(0 <= n->get_int() && (n->get_int() <= 4));
3776   match(ConI);
3777 
3778   op_cost(0);
3779   format %{ %}
3780   interface(CONST_INTER);
3781 %}
3782 
3783 operand immI_le_4()
3784 %{
3785   predicate(n->get_int() <= 4);
3786   match(ConI);
3787 
3788   op_cost(0);
3789   format %{ %}
3790   interface(CONST_INTER);
3791 %}
3792 
3793 operand immI_31()
3794 %{
3795   predicate(n->get_int() == 31);
3796   match(ConI);
3797 
3798   op_cost(0);
3799   format %{ %}
3800   interface(CONST_INTER);
3801 %}
3802 
3803 operand immI_8()
3804 %{
3805   predicate(n->get_int() == 8);
3806   match(ConI);
3807 
3808   op_cost(0);
3809   format %{ %}
3810   interface(CONST_INTER);
3811 %}
3812 
3813 operand immI_16()
3814 %{
3815   predicate(n->get_int() == 16);
3816   match(ConI);
3817 
3818   op_cost(0);
3819   format %{ %}
3820   interface(CONST_INTER);
3821 %}
3822 
3823 operand immI_24()
3824 %{
3825   predicate(n->get_int() == 24);
3826   match(ConI);
3827 
3828   op_cost(0);
3829   format %{ %}
3830   interface(CONST_INTER);
3831 %}
3832 
3833 operand immI_32()
3834 %{
3835   predicate(n->get_int() == 32);
3836   match(ConI);
3837 
3838   op_cost(0);
3839   format %{ %}
3840   interface(CONST_INTER);
3841 %}
3842 
3843 operand immI_48()
3844 %{
3845   predicate(n->get_int() == 48);
3846   match(ConI);
3847 
3848   op_cost(0);
3849   format %{ %}
3850   interface(CONST_INTER);
3851 %}
3852 
3853 operand immI_56()
3854 %{
3855   predicate(n->get_int() == 56);
3856   match(ConI);
3857 
3858   op_cost(0);
3859   format %{ %}
3860   interface(CONST_INTER);
3861 %}
3862 
3863 operand immI_63()
3864 %{
3865   predicate(n->get_int() == 63);
3866   match(ConI);
3867 
3868   op_cost(0);
3869   format %{ %}
3870   interface(CONST_INTER);
3871 %}
3872 
3873 operand immI_64()
3874 %{
3875   predicate(n->get_int() == 64);
3876   match(ConI);
3877 
3878   op_cost(0);
3879   format %{ %}
3880   interface(CONST_INTER);
3881 %}
3882 
3883 operand immI_255()
3884 %{
3885   predicate(n->get_int() == 255);
3886   match(ConI);
3887 
3888   op_cost(0);
3889   format %{ %}
3890   interface(CONST_INTER);
3891 %}
3892 
3893 operand immI_65535()
3894 %{
3895   predicate(n->get_int() == 65535);
3896   match(ConI);
3897 
3898   op_cost(0);
3899   format %{ %}
3900   interface(CONST_INTER);
3901 %}
3902 
3903 operand immL_255()
3904 %{
3905   predicate(n->get_long() == 255L);
3906   match(ConL);
3907 
3908   op_cost(0);
3909   format %{ %}
3910   interface(CONST_INTER);
3911 %}
3912 
3913 operand immL_65535()
3914 %{
3915   predicate(n->get_long() == 65535L);
3916   match(ConL);
3917 
3918   op_cost(0);
3919   format %{ %}
3920   interface(CONST_INTER);
3921 %}
3922 
3923 operand immL_4294967295()
3924 %{
3925   predicate(n->get_long() == 4294967295L);
3926   match(ConL);
3927 
3928   op_cost(0);
3929   format %{ %}
3930   interface(CONST_INTER);
3931 %}
3932 
3933 operand immL_bitmask()
3934 %{
3935   predicate(((n->get_long() & 0xc000000000000000l) == 0)
3936             && is_power_of_2(n->get_long() + 1));
3937   match(ConL);
3938 
3939   op_cost(0);
3940   format %{ %}
3941   interface(CONST_INTER);
3942 %}
3943 
3944 operand immI_bitmask()
3945 %{
3946   predicate(((n->get_int() & 0xc0000000) == 0)
3947             && is_power_of_2(n->get_int() + 1));
3948   match(ConI);
3949 
3950   op_cost(0);
3951   format %{ %}
3952   interface(CONST_INTER);
3953 %}
3954 
3955 // Scale values for scaled offset addressing modes (up to long but not quad)
3956 operand immIScale()
3957 %{
3958   predicate(0 <= n->get_int() && (n->get_int() <= 3));
3959   match(ConI);
3960 
3961   op_cost(0);
3962   format %{ %}
3963   interface(CONST_INTER);
3964 %}
3965 
3966 // 26 bit signed offset -- for pc-relative branches
3967 operand immI26()
3968 %{
3969   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
3970   match(ConI);
3971 
3972   op_cost(0);
3973   format %{ %}
3974   interface(CONST_INTER);
3975 %}
3976 
3977 // 19 bit signed offset -- for pc-relative loads
3978 operand immI19()
3979 %{
3980   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
3981   match(ConI);
3982 
3983   op_cost(0);
3984   format %{ %}
3985   interface(CONST_INTER);
3986 %}
3987 
3988 // 12 bit unsigned offset -- for base plus immediate loads
3989 operand immIU12()
3990 %{
3991   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
3992   match(ConI);
3993 
3994   op_cost(0);
3995   format %{ %}
3996   interface(CONST_INTER);
3997 %}
3998 
3999 operand immLU12()
4000 %{
4001   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4002   match(ConL);
4003 
4004   op_cost(0);
4005   format %{ %}
4006   interface(CONST_INTER);
4007 %}
4008 
4009 // Offset for scaled or unscaled immediate loads and stores
4010 operand immIOffset()
4011 %{
4012   predicate(Address::offset_ok_for_immed(n->get_int()));
4013   match(ConI);
4014 
4015   op_cost(0);
4016   format %{ %}
4017   interface(CONST_INTER);
4018 %}
4019 
4020 operand immIOffset4()
4021 %{
4022   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4023   match(ConI);
4024 
4025   op_cost(0);
4026   format %{ %}
4027   interface(CONST_INTER);
4028 %}
4029 
4030 operand immIOffset8()
4031 %{
4032   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4033   match(ConI);
4034 
4035   op_cost(0);
4036   format %{ %}
4037   interface(CONST_INTER);
4038 %}
4039 
4040 operand immIOffset16()
4041 %{
4042   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4043   match(ConI);
4044 
4045   op_cost(0);
4046   format %{ %}
4047   interface(CONST_INTER);
4048 %}
4049 
4050 operand immLoffset()
4051 %{
4052   predicate(Address::offset_ok_for_immed(n->get_long()));
4053   match(ConL);
4054 
4055   op_cost(0);
4056   format %{ %}
4057   interface(CONST_INTER);
4058 %}
4059 
4060 operand immLoffset4()
4061 %{
4062   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4063   match(ConL);
4064 
4065   op_cost(0);
4066   format %{ %}
4067   interface(CONST_INTER);
4068 %}
4069 
4070 operand immLoffset8()
4071 %{
4072   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4073   match(ConL);
4074 
4075   op_cost(0);
4076   format %{ %}
4077   interface(CONST_INTER);
4078 %}
4079 
4080 operand immLoffset16()
4081 %{
4082   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4083   match(ConL);
4084 
4085   op_cost(0);
4086   format %{ %}
4087   interface(CONST_INTER);
4088 %}
4089 
4090 // 32 bit integer valid for add sub immediate
4091 operand immIAddSub()
4092 %{
4093   predicate(Assembler::operand_valid_for_add_sub_immediate((long)n->get_int()));
4094   match(ConI);
4095   op_cost(0);
4096   format %{ %}
4097   interface(CONST_INTER);
4098 %}
4099 
4100 // 32 bit unsigned integer valid for logical immediate
4101 // TODO -- check this is right when e.g the mask is 0x80000000
4102 operand immILog()
4103 %{
4104   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (unsigned long)n->get_int()));
4105   match(ConI);
4106 
4107   op_cost(0);
4108   format %{ %}
4109   interface(CONST_INTER);
4110 %}
4111 
4112 // Integer operands 64 bit
4113 // 64 bit immediate
4114 operand immL()
4115 %{
4116   match(ConL);
4117 
4118   op_cost(0);
4119   format %{ %}
4120   interface(CONST_INTER);
4121 %}
4122 
4123 // 64 bit zero
4124 operand immL0()
4125 %{
4126   predicate(n->get_long() == 0);
4127   match(ConL);
4128 
4129   op_cost(0);
4130   format %{ %}
4131   interface(CONST_INTER);
4132 %}
4133 
4134 // 64 bit unit increment
4135 operand immL_1()
4136 %{
4137   predicate(n->get_long() == 1);
4138   match(ConL);
4139 
4140   op_cost(0);
4141   format %{ %}
4142   interface(CONST_INTER);
4143 %}
4144 
4145 // 64 bit unit decrement
4146 operand immL_M1()
4147 %{
4148   predicate(n->get_long() == -1);
4149   match(ConL);
4150 
4151   op_cost(0);
4152   format %{ %}
4153   interface(CONST_INTER);
4154 %}
4155 
4156 // 32 bit offset of pc in thread anchor
4157 
4158 operand immL_pc_off()
4159 %{
4160   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4161                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4162   match(ConL);
4163 
4164   op_cost(0);
4165   format %{ %}
4166   interface(CONST_INTER);
4167 %}
4168 
4169 // 64 bit integer valid for add sub immediate
4170 operand immLAddSub()
4171 %{
4172   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4173   match(ConL);
4174   op_cost(0);
4175   format %{ %}
4176   interface(CONST_INTER);
4177 %}
4178 
4179 // 64 bit integer valid for logical immediate
4180 operand immLLog()
4181 %{
4182   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (unsigned long)n->get_long()));
4183   match(ConL);
4184   op_cost(0);
4185   format %{ %}
4186   interface(CONST_INTER);
4187 %}
4188 
4189 // Long Immediate: low 32-bit mask
4190 operand immL_32bits()
4191 %{
4192   predicate(n->get_long() == 0xFFFFFFFFL);
4193   match(ConL);
4194   op_cost(0);
4195   format %{ %}
4196   interface(CONST_INTER);
4197 %}
4198 
4199 // Pointer operands
4200 // Pointer Immediate
4201 operand immP()
4202 %{
4203   match(ConP);
4204 
4205   op_cost(0);
4206   format %{ %}
4207   interface(CONST_INTER);
4208 %}
4209 
4210 // NULL Pointer Immediate
4211 operand immP0()
4212 %{
4213   predicate(n->get_ptr() == 0);
4214   match(ConP);
4215 
4216   op_cost(0);
4217   format %{ %}
4218   interface(CONST_INTER);
4219 %}
4220 
4221 // Pointer Immediate One
4222 // this is used in object initialization (initial object header)
4223 operand immP_1()
4224 %{
4225   predicate(n->get_ptr() == 1);
4226   match(ConP);
4227 
4228   op_cost(0);
4229   format %{ %}
4230   interface(CONST_INTER);
4231 %}
4232 
4233 // Polling Page Pointer Immediate
4234 operand immPollPage()
4235 %{
4236   predicate((address)n->get_ptr() == os::get_polling_page());
4237   match(ConP);
4238 
4239   op_cost(0);
4240   format %{ %}
4241   interface(CONST_INTER);
4242 %}
4243 
4244 // Card Table Byte Map Base
4245 operand immByteMapBase()
4246 %{
4247   // Get base of card map
4248   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4249             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4250   match(ConP);
4251 
4252   op_cost(0);
4253   format %{ %}
4254   interface(CONST_INTER);
4255 %}
4256 
4257 // Pointer Immediate Minus One
4258 // this is used when we want to write the current PC to the thread anchor
4259 operand immP_M1()
4260 %{
4261   predicate(n->get_ptr() == -1);
4262   match(ConP);
4263 
4264   op_cost(0);
4265   format %{ %}
4266   interface(CONST_INTER);
4267 %}
4268 
4269 // Pointer Immediate Minus Two
4270 // this is used when we want to write the current PC to the thread anchor
4271 operand immP_M2()
4272 %{
4273   predicate(n->get_ptr() == -2);
4274   match(ConP);
4275 
4276   op_cost(0);
4277   format %{ %}
4278   interface(CONST_INTER);
4279 %}
4280 
4281 // Float and Double operands
4282 // Double Immediate
4283 operand immD()
4284 %{
4285   match(ConD);
4286   op_cost(0);
4287   format %{ %}
4288   interface(CONST_INTER);
4289 %}
4290 
4291 // Double Immediate: +0.0d
4292 operand immD0()
4293 %{
4294   predicate(jlong_cast(n->getd()) == 0);
4295   match(ConD);
4296 
4297   op_cost(0);
4298   format %{ %}
4299   interface(CONST_INTER);
4300 %}
4301 
4302 // constant 'double +0.0'.
4303 operand immDPacked()
4304 %{
4305   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4306   match(ConD);
4307   op_cost(0);
4308   format %{ %}
4309   interface(CONST_INTER);
4310 %}
4311 
4312 // Float Immediate
4313 operand immF()
4314 %{
4315   match(ConF);
4316   op_cost(0);
4317   format %{ %}
4318   interface(CONST_INTER);
4319 %}
4320 
4321 // Float Immediate: +0.0f.
4322 operand immF0()
4323 %{
4324   predicate(jint_cast(n->getf()) == 0);
4325   match(ConF);
4326 
4327   op_cost(0);
4328   format %{ %}
4329   interface(CONST_INTER);
4330 %}
4331 
4332 //
4333 operand immFPacked()
4334 %{
4335   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4336   match(ConF);
4337   op_cost(0);
4338   format %{ %}
4339   interface(CONST_INTER);
4340 %}
4341 
4342 // Narrow pointer operands
4343 // Narrow Pointer Immediate
4344 operand immN()
4345 %{
4346   match(ConN);
4347 
4348   op_cost(0);
4349   format %{ %}
4350   interface(CONST_INTER);
4351 %}
4352 
4353 // Narrow NULL Pointer Immediate
4354 operand immN0()
4355 %{
4356   predicate(n->get_narrowcon() == 0);
4357   match(ConN);
4358 
4359   op_cost(0);
4360   format %{ %}
4361   interface(CONST_INTER);
4362 %}
4363 
4364 operand immNKlass()
4365 %{
4366   match(ConNKlass);
4367 
4368   op_cost(0);
4369   format %{ %}
4370   interface(CONST_INTER);
4371 %}
4372 
4373 // Integer 32 bit Register Operands
4374 // Integer 32 bitRegister (excludes SP)
4375 operand iRegI()
4376 %{
4377   constraint(ALLOC_IN_RC(any_reg32));
4378   match(RegI);
4379   match(iRegINoSp);
4380   op_cost(0);
4381   format %{ %}
4382   interface(REG_INTER);
4383 %}
4384 
4385 // Integer 32 bit Register not Special
4386 operand iRegINoSp()
4387 %{
4388   constraint(ALLOC_IN_RC(no_special_reg32));
4389   match(RegI);
4390   op_cost(0);
4391   format %{ %}
4392   interface(REG_INTER);
4393 %}
4394 
4395 // Integer 64 bit Register Operands
4396 // Integer 64 bit Register (includes SP)
4397 operand iRegL()
4398 %{
4399   constraint(ALLOC_IN_RC(any_reg));
4400   match(RegL);
4401   match(iRegLNoSp);
4402   op_cost(0);
4403   format %{ %}
4404   interface(REG_INTER);
4405 %}
4406 
4407 // Integer 64 bit Register not Special
4408 operand iRegLNoSp()
4409 %{
4410   constraint(ALLOC_IN_RC(no_special_reg));
4411   match(RegL);
4412   match(iRegL_R0);
4413   format %{ %}
4414   interface(REG_INTER);
4415 %}
4416 
4417 // Pointer Register Operands
4418 // Pointer Register
4419 operand iRegP()
4420 %{
4421   constraint(ALLOC_IN_RC(ptr_reg));
4422   match(RegP);
4423   match(iRegPNoSp);
4424   match(iRegP_R0);
4425   //match(iRegP_R2);
4426   //match(iRegP_R4);
4427   //match(iRegP_R5);
4428   match(thread_RegP);
4429   op_cost(0);
4430   format %{ %}
4431   interface(REG_INTER);
4432 %}
4433 
4434 // Pointer 64 bit Register not Special
4435 operand iRegPNoSp()
4436 %{
4437   constraint(ALLOC_IN_RC(no_special_ptr_reg));
4438   match(RegP);
4439   // match(iRegP);
4440   // match(iRegP_R0);
4441   // match(iRegP_R2);
4442   // match(iRegP_R4);
4443   // match(iRegP_R5);
4444   // match(thread_RegP);
4445   op_cost(0);
4446   format %{ %}
4447   interface(REG_INTER);
4448 %}
4449 
4450 // Pointer 64 bit Register R0 only
4451 operand iRegP_R0()
4452 %{
4453   constraint(ALLOC_IN_RC(r0_reg));
4454   match(RegP);
4455   // match(iRegP);
4456   match(iRegPNoSp);
4457   op_cost(0);
4458   format %{ %}
4459   interface(REG_INTER);
4460 %}
4461 
4462 // Pointer 64 bit Register R1 only
4463 operand iRegP_R1()
4464 %{
4465   constraint(ALLOC_IN_RC(r1_reg));
4466   match(RegP);
4467   // match(iRegP);
4468   match(iRegPNoSp);
4469   op_cost(0);
4470   format %{ %}
4471   interface(REG_INTER);
4472 %}
4473 
4474 // Pointer 64 bit Register R2 only
4475 operand iRegP_R2()
4476 %{
4477   constraint(ALLOC_IN_RC(r2_reg));
4478   match(RegP);
4479   // match(iRegP);
4480   match(iRegPNoSp);
4481   op_cost(0);
4482   format %{ %}
4483   interface(REG_INTER);
4484 %}
4485 
4486 // Pointer 64 bit Register R3 only
4487 operand iRegP_R3()
4488 %{
4489   constraint(ALLOC_IN_RC(r3_reg));
4490   match(RegP);
4491   // match(iRegP);
4492   match(iRegPNoSp);
4493   op_cost(0);
4494   format %{ %}
4495   interface(REG_INTER);
4496 %}
4497 
4498 // Pointer 64 bit Register R4 only
4499 operand iRegP_R4()
4500 %{
4501   constraint(ALLOC_IN_RC(r4_reg));
4502   match(RegP);
4503   // match(iRegP);
4504   match(iRegPNoSp);
4505   op_cost(0);
4506   format %{ %}
4507   interface(REG_INTER);
4508 %}
4509 
4510 // Pointer 64 bit Register R5 only
4511 operand iRegP_R5()
4512 %{
4513   constraint(ALLOC_IN_RC(r5_reg));
4514   match(RegP);
4515   // match(iRegP);
4516   match(iRegPNoSp);
4517   op_cost(0);
4518   format %{ %}
4519   interface(REG_INTER);
4520 %}
4521 
4522 // Pointer 64 bit Register R10 only
4523 operand iRegP_R10()
4524 %{
4525   constraint(ALLOC_IN_RC(r10_reg));
4526   match(RegP);
4527   // match(iRegP);
4528   match(iRegPNoSp);
4529   op_cost(0);
4530   format %{ %}
4531   interface(REG_INTER);
4532 %}
4533 
4534 // Long 64 bit Register R0 only
4535 operand iRegL_R0()
4536 %{
4537   constraint(ALLOC_IN_RC(r0_reg));
4538   match(RegL);
4539   match(iRegLNoSp);
4540   op_cost(0);
4541   format %{ %}
4542   interface(REG_INTER);
4543 %}
4544 
4545 // Long 64 bit Register R2 only
4546 operand iRegL_R2()
4547 %{
4548   constraint(ALLOC_IN_RC(r2_reg));
4549   match(RegL);
4550   match(iRegLNoSp);
4551   op_cost(0);
4552   format %{ %}
4553   interface(REG_INTER);
4554 %}
4555 
4556 // Long 64 bit Register R3 only
4557 operand iRegL_R3()
4558 %{
4559   constraint(ALLOC_IN_RC(r3_reg));
4560   match(RegL);
4561   match(iRegLNoSp);
4562   op_cost(0);
4563   format %{ %}
4564   interface(REG_INTER);
4565 %}
4566 
4567 // Long 64 bit Register R11 only
4568 operand iRegL_R11()
4569 %{
4570   constraint(ALLOC_IN_RC(r11_reg));
4571   match(RegL);
4572   match(iRegLNoSp);
4573   op_cost(0);
4574   format %{ %}
4575   interface(REG_INTER);
4576 %}
4577 
4578 // Pointer 64 bit Register FP only
4579 operand iRegP_FP()
4580 %{
4581   constraint(ALLOC_IN_RC(fp_reg));
4582   match(RegP);
4583   // match(iRegP);
4584   op_cost(0);
4585   format %{ %}
4586   interface(REG_INTER);
4587 %}
4588 
4589 // Register R0 only
4590 operand iRegI_R0()
4591 %{
4592   constraint(ALLOC_IN_RC(int_r0_reg));
4593   match(RegI);
4594   match(iRegINoSp);
4595   op_cost(0);
4596   format %{ %}
4597   interface(REG_INTER);
4598 %}
4599 
4600 // Register R2 only
4601 operand iRegI_R2()
4602 %{
4603   constraint(ALLOC_IN_RC(int_r2_reg));
4604   match(RegI);
4605   match(iRegINoSp);
4606   op_cost(0);
4607   format %{ %}
4608   interface(REG_INTER);
4609 %}
4610 
4611 // Register R3 only
4612 operand iRegI_R3()
4613 %{
4614   constraint(ALLOC_IN_RC(int_r3_reg));
4615   match(RegI);
4616   match(iRegINoSp);
4617   op_cost(0);
4618   format %{ %}
4619   interface(REG_INTER);
4620 %}
4621 
4622 
4623 // Register R4 only
4624 operand iRegI_R4()
4625 %{
4626   constraint(ALLOC_IN_RC(int_r4_reg));
4627   match(RegI);
4628   match(iRegINoSp);
4629   op_cost(0);
4630   format %{ %}
4631   interface(REG_INTER);
4632 %}
4633 
4634 
4635 // Pointer Register Operands
4636 // Narrow Pointer Register
4637 operand iRegN()
4638 %{
4639   constraint(ALLOC_IN_RC(any_reg32));
4640   match(RegN);
4641   match(iRegNNoSp);
4642   op_cost(0);
4643   format %{ %}
4644   interface(REG_INTER);
4645 %}
4646 
4647 operand iRegN_R0()
4648 %{
4649   constraint(ALLOC_IN_RC(r0_reg));
4650   match(iRegN);
4651   op_cost(0);
4652   format %{ %}
4653   interface(REG_INTER);
4654 %}
4655 
4656 operand iRegN_R2()
4657 %{
4658   constraint(ALLOC_IN_RC(r2_reg));
4659   match(iRegN);
4660   op_cost(0);
4661   format %{ %}
4662   interface(REG_INTER);
4663 %}
4664 
4665 operand iRegN_R3()
4666 %{
4667   constraint(ALLOC_IN_RC(r3_reg));
4668   match(iRegN);
4669   op_cost(0);
4670   format %{ %}
4671   interface(REG_INTER);
4672 %}
4673 
4674 // Integer 64 bit Register not Special
4675 operand iRegNNoSp()
4676 %{
4677   constraint(ALLOC_IN_RC(no_special_reg32));
4678   match(RegN);
4679   op_cost(0);
4680   format %{ %}
4681   interface(REG_INTER);
4682 %}
4683 
4684 // heap base register -- used for encoding immN0
4685 
4686 operand iRegIHeapbase()
4687 %{
4688   constraint(ALLOC_IN_RC(heapbase_reg));
4689   match(RegI);
4690   op_cost(0);
4691   format %{ %}
4692   interface(REG_INTER);
4693 %}
4694 
4695 // Float Register
4696 // Float register operands
4697 operand vRegF()
4698 %{
4699   constraint(ALLOC_IN_RC(float_reg));
4700   match(RegF);
4701 
4702   op_cost(0);
4703   format %{ %}
4704   interface(REG_INTER);
4705 %}
4706 
4707 // Double Register
4708 // Double register operands
4709 operand vRegD()
4710 %{
4711   constraint(ALLOC_IN_RC(double_reg));
4712   match(RegD);
4713 
4714   op_cost(0);
4715   format %{ %}
4716   interface(REG_INTER);
4717 %}
4718 
4719 operand vecD()
4720 %{
4721   constraint(ALLOC_IN_RC(vectord_reg));
4722   match(VecD);
4723 
4724   op_cost(0);
4725   format %{ %}
4726   interface(REG_INTER);
4727 %}
4728 
4729 operand vecX()
4730 %{
4731   constraint(ALLOC_IN_RC(vectorx_reg));
4732   match(VecX);
4733 
4734   op_cost(0);
4735   format %{ %}
4736   interface(REG_INTER);
4737 %}
4738 
4739 operand vRegD_V0()
4740 %{
4741   constraint(ALLOC_IN_RC(v0_reg));
4742   match(RegD);
4743   op_cost(0);
4744   format %{ %}
4745   interface(REG_INTER);
4746 %}
4747 
4748 operand vRegD_V1()
4749 %{
4750   constraint(ALLOC_IN_RC(v1_reg));
4751   match(RegD);
4752   op_cost(0);
4753   format %{ %}
4754   interface(REG_INTER);
4755 %}
4756 
4757 operand vRegD_V2()
4758 %{
4759   constraint(ALLOC_IN_RC(v2_reg));
4760   match(RegD);
4761   op_cost(0);
4762   format %{ %}
4763   interface(REG_INTER);
4764 %}
4765 
4766 operand vRegD_V3()
4767 %{
4768   constraint(ALLOC_IN_RC(v3_reg));
4769   match(RegD);
4770   op_cost(0);
4771   format %{ %}
4772   interface(REG_INTER);
4773 %}
4774 
4775 // Flags register, used as output of signed compare instructions
4776 
4777 // note that on AArch64 we also use this register as the output for
4778 // for floating point compare instructions (CmpF CmpD). this ensures
4779 // that ordered inequality tests use GT, GE, LT or LE none of which
4780 // pass through cases where the result is unordered i.e. one or both
4781 // inputs to the compare is a NaN. this means that the ideal code can
4782 // replace e.g. a GT with an LE and not end up capturing the NaN case
4783 // (where the comparison should always fail). EQ and NE tests are
4784 // always generated in ideal code so that unordered folds into the NE
4785 // case, matching the behaviour of AArch64 NE.
4786 //
4787 // This differs from x86 where the outputs of FP compares use a
4788 // special FP flags registers and where compares based on this
4789 // register are distinguished into ordered inequalities (cmpOpUCF) and
4790 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
4791 // to explicitly handle the unordered case in branches. x86 also has
4792 // to include extra CMoveX rules to accept a cmpOpUCF input.
4793 
4794 operand rFlagsReg()
4795 %{
4796   constraint(ALLOC_IN_RC(int_flags));
4797   match(RegFlags);
4798 
4799   op_cost(0);
4800   format %{ "RFLAGS" %}
4801   interface(REG_INTER);
4802 %}
4803 
4804 // Flags register, used as output of unsigned compare instructions
4805 operand rFlagsRegU()
4806 %{
4807   constraint(ALLOC_IN_RC(int_flags));
4808   match(RegFlags);
4809 
4810   op_cost(0);
4811   format %{ "RFLAGSU" %}
4812   interface(REG_INTER);
4813 %}
4814 
4815 // Special Registers
4816 
4817 // Method Register
4818 operand inline_cache_RegP(iRegP reg)
4819 %{
4820   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
4821   match(reg);
4822   match(iRegPNoSp);
4823   op_cost(0);
4824   format %{ %}
4825   interface(REG_INTER);
4826 %}
4827 
4828 operand interpreter_method_oop_RegP(iRegP reg)
4829 %{
4830   constraint(ALLOC_IN_RC(method_reg)); // interpreter_method_oop_reg
4831   match(reg);
4832   match(iRegPNoSp);
4833   op_cost(0);
4834   format %{ %}
4835   interface(REG_INTER);
4836 %}
4837 
4838 // Thread Register
4839 operand thread_RegP(iRegP reg)
4840 %{
4841   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
4842   match(reg);
4843   op_cost(0);
4844   format %{ %}
4845   interface(REG_INTER);
4846 %}
4847 
4848 operand lr_RegP(iRegP reg)
4849 %{
4850   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
4851   match(reg);
4852   op_cost(0);
4853   format %{ %}
4854   interface(REG_INTER);
4855 %}
4856 
4857 //----------Memory Operands----------------------------------------------------
4858 
4859 operand indirect(iRegP reg)
4860 %{
4861   constraint(ALLOC_IN_RC(ptr_reg));
4862   match(reg);
4863   op_cost(0);
4864   format %{ "[$reg]" %}
4865   interface(MEMORY_INTER) %{
4866     base($reg);
4867     index(0xffffffff);
4868     scale(0x0);
4869     disp(0x0);
4870   %}
4871 %}
4872 
4873 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
4874 %{
4875   constraint(ALLOC_IN_RC(ptr_reg));
4876   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
4877   match(AddP reg (LShiftL (ConvI2L ireg) scale));
4878   op_cost(0);
4879   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
4880   interface(MEMORY_INTER) %{
4881     base($reg);
4882     index($ireg);
4883     scale($scale);
4884     disp(0x0);
4885   %}
4886 %}
4887 
4888 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
4889 %{
4890   constraint(ALLOC_IN_RC(ptr_reg));
4891   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
4892   match(AddP reg (LShiftL lreg scale));
4893   op_cost(0);
4894   format %{ "$reg, $lreg lsl($scale)" %}
4895   interface(MEMORY_INTER) %{
4896     base($reg);
4897     index($lreg);
4898     scale($scale);
4899     disp(0x0);
4900   %}
4901 %}
4902 
4903 operand indIndexI2L(iRegP reg, iRegI ireg)
4904 %{
4905   constraint(ALLOC_IN_RC(ptr_reg));
4906   match(AddP reg (ConvI2L ireg));
4907   op_cost(0);
4908   format %{ "$reg, $ireg, 0, I2L" %}
4909   interface(MEMORY_INTER) %{
4910     base($reg);
4911     index($ireg);
4912     scale(0x0);
4913     disp(0x0);
4914   %}
4915 %}
4916 
4917 operand indIndex(iRegP reg, iRegL lreg)
4918 %{
4919   constraint(ALLOC_IN_RC(ptr_reg));
4920   match(AddP reg lreg);
4921   op_cost(0);
4922   format %{ "$reg, $lreg" %}
4923   interface(MEMORY_INTER) %{
4924     base($reg);
4925     index($lreg);
4926     scale(0x0);
4927     disp(0x0);
4928   %}
4929 %}
4930 
4931 operand indOffI(iRegP reg, immIOffset off)
4932 %{
4933   constraint(ALLOC_IN_RC(ptr_reg));
4934   match(AddP reg off);
4935   op_cost(0);
4936   format %{ "[$reg, $off]" %}
4937   interface(MEMORY_INTER) %{
4938     base($reg);
4939     index(0xffffffff);
4940     scale(0x0);
4941     disp($off);
4942   %}
4943 %}
4944 
4945 operand indOffI4(iRegP reg, immIOffset4 off)
4946 %{
4947   constraint(ALLOC_IN_RC(ptr_reg));
4948   match(AddP reg off);
4949   op_cost(0);
4950   format %{ "[$reg, $off]" %}
4951   interface(MEMORY_INTER) %{
4952     base($reg);
4953     index(0xffffffff);
4954     scale(0x0);
4955     disp($off);
4956   %}
4957 %}
4958 
4959 operand indOffI8(iRegP reg, immIOffset8 off)
4960 %{
4961   constraint(ALLOC_IN_RC(ptr_reg));
4962   match(AddP reg off);
4963   op_cost(0);
4964   format %{ "[$reg, $off]" %}
4965   interface(MEMORY_INTER) %{
4966     base($reg);
4967     index(0xffffffff);
4968     scale(0x0);
4969     disp($off);
4970   %}
4971 %}
4972 
4973 operand indOffI16(iRegP reg, immIOffset16 off)
4974 %{
4975   constraint(ALLOC_IN_RC(ptr_reg));
4976   match(AddP reg off);
4977   op_cost(0);
4978   format %{ "[$reg, $off]" %}
4979   interface(MEMORY_INTER) %{
4980     base($reg);
4981     index(0xffffffff);
4982     scale(0x0);
4983     disp($off);
4984   %}
4985 %}
4986 
4987 operand indOffL(iRegP reg, immLoffset off)
4988 %{
4989   constraint(ALLOC_IN_RC(ptr_reg));
4990   match(AddP reg off);
4991   op_cost(0);
4992   format %{ "[$reg, $off]" %}
4993   interface(MEMORY_INTER) %{
4994     base($reg);
4995     index(0xffffffff);
4996     scale(0x0);
4997     disp($off);
4998   %}
4999 %}
5000 
5001 operand indOffL4(iRegP reg, immLoffset4 off)
5002 %{
5003   constraint(ALLOC_IN_RC(ptr_reg));
5004   match(AddP reg off);
5005   op_cost(0);
5006   format %{ "[$reg, $off]" %}
5007   interface(MEMORY_INTER) %{
5008     base($reg);
5009     index(0xffffffff);
5010     scale(0x0);
5011     disp($off);
5012   %}
5013 %}
5014 
5015 operand indOffL8(iRegP reg, immLoffset8 off)
5016 %{
5017   constraint(ALLOC_IN_RC(ptr_reg));
5018   match(AddP reg off);
5019   op_cost(0);
5020   format %{ "[$reg, $off]" %}
5021   interface(MEMORY_INTER) %{
5022     base($reg);
5023     index(0xffffffff);
5024     scale(0x0);
5025     disp($off);
5026   %}
5027 %}
5028 
5029 operand indOffL16(iRegP reg, immLoffset16 off)
5030 %{
5031   constraint(ALLOC_IN_RC(ptr_reg));
5032   match(AddP reg off);
5033   op_cost(0);
5034   format %{ "[$reg, $off]" %}
5035   interface(MEMORY_INTER) %{
5036     base($reg);
5037     index(0xffffffff);
5038     scale(0x0);
5039     disp($off);
5040   %}
5041 %}
5042 
5043 operand indirectN(iRegN reg)
5044 %{
5045   predicate(Universe::narrow_oop_shift() == 0);
5046   constraint(ALLOC_IN_RC(ptr_reg));
5047   match(DecodeN reg);
5048   op_cost(0);
5049   format %{ "[$reg]\t# narrow" %}
5050   interface(MEMORY_INTER) %{
5051     base($reg);
5052     index(0xffffffff);
5053     scale(0x0);
5054     disp(0x0);
5055   %}
5056 %}
5057 
5058 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5059 %{
5060   predicate(Universe::narrow_oop_shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5061   constraint(ALLOC_IN_RC(ptr_reg));
5062   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5063   op_cost(0);
5064   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5065   interface(MEMORY_INTER) %{
5066     base($reg);
5067     index($ireg);
5068     scale($scale);
5069     disp(0x0);
5070   %}
5071 %}
5072 
5073 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5074 %{
5075   predicate(Universe::narrow_oop_shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5076   constraint(ALLOC_IN_RC(ptr_reg));
5077   match(AddP (DecodeN reg) (LShiftL lreg scale));
5078   op_cost(0);
5079   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5080   interface(MEMORY_INTER) %{
5081     base($reg);
5082     index($lreg);
5083     scale($scale);
5084     disp(0x0);
5085   %}
5086 %}
5087 
5088 operand indIndexI2LN(iRegN reg, iRegI ireg)
5089 %{
5090   predicate(Universe::narrow_oop_shift() == 0);
5091   constraint(ALLOC_IN_RC(ptr_reg));
5092   match(AddP (DecodeN reg) (ConvI2L ireg));
5093   op_cost(0);
5094   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5095   interface(MEMORY_INTER) %{
5096     base($reg);
5097     index($ireg);
5098     scale(0x0);
5099     disp(0x0);
5100   %}
5101 %}
5102 
5103 operand indIndexN(iRegN reg, iRegL lreg)
5104 %{
5105   predicate(Universe::narrow_oop_shift() == 0);
5106   constraint(ALLOC_IN_RC(ptr_reg));
5107   match(AddP (DecodeN reg) lreg);
5108   op_cost(0);
5109   format %{ "$reg, $lreg\t# narrow" %}
5110   interface(MEMORY_INTER) %{
5111     base($reg);
5112     index($lreg);
5113     scale(0x0);
5114     disp(0x0);
5115   %}
5116 %}
5117 
5118 operand indOffIN(iRegN reg, immIOffset off)
5119 %{
5120   predicate(Universe::narrow_oop_shift() == 0);
5121   constraint(ALLOC_IN_RC(ptr_reg));
5122   match(AddP (DecodeN reg) off);
5123   op_cost(0);
5124   format %{ "[$reg, $off]\t# narrow" %}
5125   interface(MEMORY_INTER) %{
5126     base($reg);
5127     index(0xffffffff);
5128     scale(0x0);
5129     disp($off);
5130   %}
5131 %}
5132 
5133 operand indOffLN(iRegN reg, immLoffset off)
5134 %{
5135   predicate(Universe::narrow_oop_shift() == 0);
5136   constraint(ALLOC_IN_RC(ptr_reg));
5137   match(AddP (DecodeN reg) off);
5138   op_cost(0);
5139   format %{ "[$reg, $off]\t# narrow" %}
5140   interface(MEMORY_INTER) %{
5141     base($reg);
5142     index(0xffffffff);
5143     scale(0x0);
5144     disp($off);
5145   %}
5146 %}
5147 
5148 
5149 
5150 // AArch64 opto stubs need to write to the pc slot in the thread anchor
5151 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
5152 %{
5153   constraint(ALLOC_IN_RC(ptr_reg));
5154   match(AddP reg off);
5155   op_cost(0);
5156   format %{ "[$reg, $off]" %}
5157   interface(MEMORY_INTER) %{
5158     base($reg);
5159     index(0xffffffff);
5160     scale(0x0);
5161     disp($off);
5162   %}
5163 %}
5164 
5165 //----------Special Memory Operands--------------------------------------------
5166 // Stack Slot Operand - This operand is used for loading and storing temporary
5167 //                      values on the stack where a match requires a value to
5168 //                      flow through memory.
5169 operand stackSlotP(sRegP reg)
5170 %{
5171   constraint(ALLOC_IN_RC(stack_slots));
5172   op_cost(100);
5173   // No match rule because this operand is only generated in matching
5174   // match(RegP);
5175   format %{ "[$reg]" %}
5176   interface(MEMORY_INTER) %{
5177     base(0x1e);  // RSP
5178     index(0x0);  // No Index
5179     scale(0x0);  // No Scale
5180     disp($reg);  // Stack Offset
5181   %}
5182 %}
5183 
5184 operand stackSlotI(sRegI reg)
5185 %{
5186   constraint(ALLOC_IN_RC(stack_slots));
5187   // No match rule because this operand is only generated in matching
5188   // match(RegI);
5189   format %{ "[$reg]" %}
5190   interface(MEMORY_INTER) %{
5191     base(0x1e);  // RSP
5192     index(0x0);  // No Index
5193     scale(0x0);  // No Scale
5194     disp($reg);  // Stack Offset
5195   %}
5196 %}
5197 
5198 operand stackSlotF(sRegF reg)
5199 %{
5200   constraint(ALLOC_IN_RC(stack_slots));
5201   // No match rule because this operand is only generated in matching
5202   // match(RegF);
5203   format %{ "[$reg]" %}
5204   interface(MEMORY_INTER) %{
5205     base(0x1e);  // RSP
5206     index(0x0);  // No Index
5207     scale(0x0);  // No Scale
5208     disp($reg);  // Stack Offset
5209   %}
5210 %}
5211 
5212 operand stackSlotD(sRegD reg)
5213 %{
5214   constraint(ALLOC_IN_RC(stack_slots));
5215   // No match rule because this operand is only generated in matching
5216   // match(RegD);
5217   format %{ "[$reg]" %}
5218   interface(MEMORY_INTER) %{
5219     base(0x1e);  // RSP
5220     index(0x0);  // No Index
5221     scale(0x0);  // No Scale
5222     disp($reg);  // Stack Offset
5223   %}
5224 %}
5225 
5226 operand stackSlotL(sRegL reg)
5227 %{
5228   constraint(ALLOC_IN_RC(stack_slots));
5229   // No match rule because this operand is only generated in matching
5230   // match(RegL);
5231   format %{ "[$reg]" %}
5232   interface(MEMORY_INTER) %{
5233     base(0x1e);  // RSP
5234     index(0x0);  // No Index
5235     scale(0x0);  // No Scale
5236     disp($reg);  // Stack Offset
5237   %}
5238 %}
5239 
5240 // Operands for expressing Control Flow
5241 // NOTE: Label is a predefined operand which should not be redefined in
5242 //       the AD file. It is generically handled within the ADLC.
5243 
5244 //----------Conditional Branch Operands----------------------------------------
5245 // Comparison Op  - This is the operation of the comparison, and is limited to
5246 //                  the following set of codes:
5247 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5248 //
5249 // Other attributes of the comparison, such as unsignedness, are specified
5250 // by the comparison instruction that sets a condition code flags register.
5251 // That result is represented by a flags operand whose subtype is appropriate
5252 // to the unsignedness (etc.) of the comparison.
5253 //
5254 // Later, the instruction which matches both the Comparison Op (a Bool) and
5255 // the flags (produced by the Cmp) specifies the coding of the comparison op
5256 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5257 
5258 // used for signed integral comparisons and fp comparisons
5259 
5260 operand cmpOp()
5261 %{
5262   match(Bool);
5263 
5264   format %{ "" %}
5265   interface(COND_INTER) %{
5266     equal(0x0, "eq");
5267     not_equal(0x1, "ne");
5268     less(0xb, "lt");
5269     greater_equal(0xa, "ge");
5270     less_equal(0xd, "le");
5271     greater(0xc, "gt");
5272     overflow(0x6, "vs");
5273     no_overflow(0x7, "vc");
5274   %}
5275 %}
5276 
5277 // used for unsigned integral comparisons
5278 
5279 operand cmpOpU()
5280 %{
5281   match(Bool);
5282 
5283   format %{ "" %}
5284   interface(COND_INTER) %{
5285     equal(0x0, "eq");
5286     not_equal(0x1, "ne");
5287     less(0x3, "lo");
5288     greater_equal(0x2, "hs");
5289     less_equal(0x9, "ls");
5290     greater(0x8, "hi");
5291     overflow(0x6, "vs");
5292     no_overflow(0x7, "vc");
5293   %}
5294 %}
5295 
5296 // used for certain integral comparisons which can be
5297 // converted to cbxx or tbxx instructions
5298 
5299 operand cmpOpEqNe()
5300 %{
5301   match(Bool);
5302   match(CmpOp);
5303   op_cost(0);
5304   predicate(n->as_Bool()->_test._test == BoolTest::ne
5305             || n->as_Bool()->_test._test == BoolTest::eq);
5306 
5307   format %{ "" %}
5308   interface(COND_INTER) %{
5309     equal(0x0, "eq");
5310     not_equal(0x1, "ne");
5311     less(0xb, "lt");
5312     greater_equal(0xa, "ge");
5313     less_equal(0xd, "le");
5314     greater(0xc, "gt");
5315     overflow(0x6, "vs");
5316     no_overflow(0x7, "vc");
5317   %}
5318 %}
5319 
5320 // used for certain integral comparisons which can be
5321 // converted to cbxx or tbxx instructions
5322 
5323 operand cmpOpLtGe()
5324 %{
5325   match(Bool);
5326   match(CmpOp);
5327   op_cost(0);
5328 
5329   predicate(n->as_Bool()->_test._test == BoolTest::lt
5330             || n->as_Bool()->_test._test == BoolTest::ge);
5331 
5332   format %{ "" %}
5333   interface(COND_INTER) %{
5334     equal(0x0, "eq");
5335     not_equal(0x1, "ne");
5336     less(0xb, "lt");
5337     greater_equal(0xa, "ge");
5338     less_equal(0xd, "le");
5339     greater(0xc, "gt");
5340     overflow(0x6, "vs");
5341     no_overflow(0x7, "vc");
5342   %}
5343 %}
5344 
5345 // used for certain unsigned integral comparisons which can be
5346 // converted to cbxx or tbxx instructions
5347 
5348 operand cmpOpUEqNeLtGe()
5349 %{
5350   match(Bool);
5351   match(CmpOp);
5352   op_cost(0);
5353 
5354   predicate(n->as_Bool()->_test._test == BoolTest::eq
5355             || n->as_Bool()->_test._test == BoolTest::ne
5356             || n->as_Bool()->_test._test == BoolTest::lt
5357             || n->as_Bool()->_test._test == BoolTest::ge);
5358 
5359   format %{ "" %}
5360   interface(COND_INTER) %{
5361     equal(0x0, "eq");
5362     not_equal(0x1, "ne");
5363     less(0xb, "lt");
5364     greater_equal(0xa, "ge");
5365     less_equal(0xd, "le");
5366     greater(0xc, "gt");
5367     overflow(0x6, "vs");
5368     no_overflow(0x7, "vc");
5369   %}
5370 %}
5371 
5372 // Special operand allowing long args to int ops to be truncated for free
5373 
5374 operand iRegL2I(iRegL reg) %{
5375 
5376   op_cost(0);
5377 
5378   match(ConvL2I reg);
5379 
5380   format %{ "l2i($reg)" %}
5381 
5382   interface(REG_INTER)
5383 %}
5384 
5385 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5386 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5387 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5388 
5389 //----------OPERAND CLASSES----------------------------------------------------
5390 // Operand Classes are groups of operands that are used as to simplify
5391 // instruction definitions by not requiring the AD writer to specify
5392 // separate instructions for every form of operand when the
5393 // instruction accepts multiple operand types with the same basic
5394 // encoding and format. The classic case of this is memory operands.
5395 
5396 // memory is used to define read/write location for load/store
5397 // instruction defs. we can turn a memory op into an Address
5398 
5399 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI, indOffL,
5400                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
5401 
5402 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5403 // operations. it allows the src to be either an iRegI or a (ConvL2I
5404 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5405 // can be elided because the 32-bit instruction will just employ the
5406 // lower 32 bits anyway.
5407 //
5408 // n.b. this does not elide all L2I conversions. if the truncated
5409 // value is consumed by more than one operation then the ConvL2I
5410 // cannot be bundled into the consuming nodes so an l2i gets planted
5411 // (actually a movw $dst $src) and the downstream instructions consume
5412 // the result of the l2i as an iRegI input. That's a shame since the
5413 // movw is actually redundant but its not too costly.
5414 
5415 opclass iRegIorL2I(iRegI, iRegL2I);
5416 
5417 //----------PIPELINE-----------------------------------------------------------
5418 // Rules which define the behavior of the target architectures pipeline.
5419 
5420 // For specific pipelines, eg A53, define the stages of that pipeline
5421 //pipe_desc(ISS, EX1, EX2, WR);
5422 #define ISS S0
5423 #define EX1 S1
5424 #define EX2 S2
5425 #define WR  S3
5426 
5427 // Integer ALU reg operation
5428 pipeline %{
5429 
5430 attributes %{
5431   // ARM instructions are of fixed length
5432   fixed_size_instructions;        // Fixed size instructions TODO does
5433   max_instructions_per_bundle = 2;   // A53 = 2, A57 = 4
5434   // ARM instructions come in 32-bit word units
5435   instruction_unit_size = 4;         // An instruction is 4 bytes long
5436   instruction_fetch_unit_size = 64;  // The processor fetches one line
5437   instruction_fetch_units = 1;       // of 64 bytes
5438 
5439   // List of nop instructions
5440   nops( MachNop );
5441 %}
5442 
5443 // We don't use an actual pipeline model so don't care about resources
5444 // or description. we do use pipeline classes to introduce fixed
5445 // latencies
5446 
5447 //----------RESOURCES----------------------------------------------------------
5448 // Resources are the functional units available to the machine
5449 
5450 resources( INS0, INS1, INS01 = INS0 | INS1,
5451            ALU0, ALU1, ALU = ALU0 | ALU1,
5452            MAC,
5453            DIV,
5454            BRANCH,
5455            LDST,
5456            NEON_FP);
5457 
5458 //----------PIPELINE DESCRIPTION-----------------------------------------------
5459 // Pipeline Description specifies the stages in the machine's pipeline
5460 
5461 // Define the pipeline as a generic 6 stage pipeline
5462 pipe_desc(S0, S1, S2, S3, S4, S5);
5463 
5464 //----------PIPELINE CLASSES---------------------------------------------------
5465 // Pipeline Classes describe the stages in which input and output are
5466 // referenced by the hardware pipeline.
5467 
5468 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5469 %{
5470   single_instruction;
5471   src1   : S1(read);
5472   src2   : S2(read);
5473   dst    : S5(write);
5474   INS01  : ISS;
5475   NEON_FP : S5;
5476 %}
5477 
5478 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5479 %{
5480   single_instruction;
5481   src1   : S1(read);
5482   src2   : S2(read);
5483   dst    : S5(write);
5484   INS01  : ISS;
5485   NEON_FP : S5;
5486 %}
5487 
5488 pipe_class fp_uop_s(vRegF dst, vRegF src)
5489 %{
5490   single_instruction;
5491   src    : S1(read);
5492   dst    : S5(write);
5493   INS01  : ISS;
5494   NEON_FP : S5;
5495 %}
5496 
5497 pipe_class fp_uop_d(vRegD dst, vRegD src)
5498 %{
5499   single_instruction;
5500   src    : S1(read);
5501   dst    : S5(write);
5502   INS01  : ISS;
5503   NEON_FP : S5;
5504 %}
5505 
5506 pipe_class fp_d2f(vRegF dst, vRegD src)
5507 %{
5508   single_instruction;
5509   src    : S1(read);
5510   dst    : S5(write);
5511   INS01  : ISS;
5512   NEON_FP : S5;
5513 %}
5514 
5515 pipe_class fp_f2d(vRegD dst, vRegF src)
5516 %{
5517   single_instruction;
5518   src    : S1(read);
5519   dst    : S5(write);
5520   INS01  : ISS;
5521   NEON_FP : S5;
5522 %}
5523 
5524 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5525 %{
5526   single_instruction;
5527   src    : S1(read);
5528   dst    : S5(write);
5529   INS01  : ISS;
5530   NEON_FP : S5;
5531 %}
5532 
5533 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5534 %{
5535   single_instruction;
5536   src    : S1(read);
5537   dst    : S5(write);
5538   INS01  : ISS;
5539   NEON_FP : S5;
5540 %}
5541 
5542 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5543 %{
5544   single_instruction;
5545   src    : S1(read);
5546   dst    : S5(write);
5547   INS01  : ISS;
5548   NEON_FP : S5;
5549 %}
5550 
5551 pipe_class fp_l2f(vRegF dst, iRegL src)
5552 %{
5553   single_instruction;
5554   src    : S1(read);
5555   dst    : S5(write);
5556   INS01  : ISS;
5557   NEON_FP : S5;
5558 %}
5559 
5560 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5561 %{
5562   single_instruction;
5563   src    : S1(read);
5564   dst    : S5(write);
5565   INS01  : ISS;
5566   NEON_FP : S5;
5567 %}
5568 
5569 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5570 %{
5571   single_instruction;
5572   src    : S1(read);
5573   dst    : S5(write);
5574   INS01  : ISS;
5575   NEON_FP : S5;
5576 %}
5577 
5578 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5579 %{
5580   single_instruction;
5581   src    : S1(read);
5582   dst    : S5(write);
5583   INS01  : ISS;
5584   NEON_FP : S5;
5585 %}
5586 
5587 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5588 %{
5589   single_instruction;
5590   src    : S1(read);
5591   dst    : S5(write);
5592   INS01  : ISS;
5593   NEON_FP : S5;
5594 %}
5595 
5596 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
5597 %{
5598   single_instruction;
5599   src1   : S1(read);
5600   src2   : S2(read);
5601   dst    : S5(write);
5602   INS0   : ISS;
5603   NEON_FP : S5;
5604 %}
5605 
5606 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
5607 %{
5608   single_instruction;
5609   src1   : S1(read);
5610   src2   : S2(read);
5611   dst    : S5(write);
5612   INS0   : ISS;
5613   NEON_FP : S5;
5614 %}
5615 
5616 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
5617 %{
5618   single_instruction;
5619   cr     : S1(read);
5620   src1   : S1(read);
5621   src2   : S1(read);
5622   dst    : S3(write);
5623   INS01  : ISS;
5624   NEON_FP : S3;
5625 %}
5626 
5627 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
5628 %{
5629   single_instruction;
5630   cr     : S1(read);
5631   src1   : S1(read);
5632   src2   : S1(read);
5633   dst    : S3(write);
5634   INS01  : ISS;
5635   NEON_FP : S3;
5636 %}
5637 
5638 pipe_class fp_imm_s(vRegF dst)
5639 %{
5640   single_instruction;
5641   dst    : S3(write);
5642   INS01  : ISS;
5643   NEON_FP : S3;
5644 %}
5645 
5646 pipe_class fp_imm_d(vRegD dst)
5647 %{
5648   single_instruction;
5649   dst    : S3(write);
5650   INS01  : ISS;
5651   NEON_FP : S3;
5652 %}
5653 
5654 pipe_class fp_load_constant_s(vRegF dst)
5655 %{
5656   single_instruction;
5657   dst    : S4(write);
5658   INS01  : ISS;
5659   NEON_FP : S4;
5660 %}
5661 
5662 pipe_class fp_load_constant_d(vRegD dst)
5663 %{
5664   single_instruction;
5665   dst    : S4(write);
5666   INS01  : ISS;
5667   NEON_FP : S4;
5668 %}
5669 
5670 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
5671 %{
5672   single_instruction;
5673   dst    : S5(write);
5674   src1   : S1(read);
5675   src2   : S1(read);
5676   INS01  : ISS;
5677   NEON_FP : S5;
5678 %}
5679 
5680 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
5681 %{
5682   single_instruction;
5683   dst    : S5(write);
5684   src1   : S1(read);
5685   src2   : S1(read);
5686   INS0   : ISS;
5687   NEON_FP : S5;
5688 %}
5689 
5690 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
5691 %{
5692   single_instruction;
5693   dst    : S5(write);
5694   src1   : S1(read);
5695   src2   : S1(read);
5696   dst    : S1(read);
5697   INS01  : ISS;
5698   NEON_FP : S5;
5699 %}
5700 
5701 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
5702 %{
5703   single_instruction;
5704   dst    : S5(write);
5705   src1   : S1(read);
5706   src2   : S1(read);
5707   dst    : S1(read);
5708   INS0   : ISS;
5709   NEON_FP : S5;
5710 %}
5711 
5712 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
5713 %{
5714   single_instruction;
5715   dst    : S4(write);
5716   src1   : S2(read);
5717   src2   : S2(read);
5718   INS01  : ISS;
5719   NEON_FP : S4;
5720 %}
5721 
5722 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
5723 %{
5724   single_instruction;
5725   dst    : S4(write);
5726   src1   : S2(read);
5727   src2   : S2(read);
5728   INS0   : ISS;
5729   NEON_FP : S4;
5730 %}
5731 
5732 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
5733 %{
5734   single_instruction;
5735   dst    : S3(write);
5736   src1   : S2(read);
5737   src2   : S2(read);
5738   INS01  : ISS;
5739   NEON_FP : S3;
5740 %}
5741 
5742 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
5743 %{
5744   single_instruction;
5745   dst    : S3(write);
5746   src1   : S2(read);
5747   src2   : S2(read);
5748   INS0   : ISS;
5749   NEON_FP : S3;
5750 %}
5751 
5752 pipe_class vshift64(vecD dst, vecD src, vecX shift)
5753 %{
5754   single_instruction;
5755   dst    : S3(write);
5756   src    : S1(read);
5757   shift  : S1(read);
5758   INS01  : ISS;
5759   NEON_FP : S3;
5760 %}
5761 
5762 pipe_class vshift128(vecX dst, vecX src, vecX shift)
5763 %{
5764   single_instruction;
5765   dst    : S3(write);
5766   src    : S1(read);
5767   shift  : S1(read);
5768   INS0   : ISS;
5769   NEON_FP : S3;
5770 %}
5771 
5772 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
5773 %{
5774   single_instruction;
5775   dst    : S3(write);
5776   src    : S1(read);
5777   INS01  : ISS;
5778   NEON_FP : S3;
5779 %}
5780 
5781 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
5782 %{
5783   single_instruction;
5784   dst    : S3(write);
5785   src    : S1(read);
5786   INS0   : ISS;
5787   NEON_FP : S3;
5788 %}
5789 
5790 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
5791 %{
5792   single_instruction;
5793   dst    : S5(write);
5794   src1   : S1(read);
5795   src2   : S1(read);
5796   INS01  : ISS;
5797   NEON_FP : S5;
5798 %}
5799 
5800 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
5801 %{
5802   single_instruction;
5803   dst    : S5(write);
5804   src1   : S1(read);
5805   src2   : S1(read);
5806   INS0   : ISS;
5807   NEON_FP : S5;
5808 %}
5809 
5810 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
5811 %{
5812   single_instruction;
5813   dst    : S5(write);
5814   src1   : S1(read);
5815   src2   : S1(read);
5816   INS0   : ISS;
5817   NEON_FP : S5;
5818 %}
5819 
5820 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
5821 %{
5822   single_instruction;
5823   dst    : S5(write);
5824   src1   : S1(read);
5825   src2   : S1(read);
5826   INS0   : ISS;
5827   NEON_FP : S5;
5828 %}
5829 
5830 pipe_class vsqrt_fp128(vecX dst, vecX src)
5831 %{
5832   single_instruction;
5833   dst    : S5(write);
5834   src    : S1(read);
5835   INS0   : ISS;
5836   NEON_FP : S5;
5837 %}
5838 
5839 pipe_class vunop_fp64(vecD dst, vecD src)
5840 %{
5841   single_instruction;
5842   dst    : S5(write);
5843   src    : S1(read);
5844   INS01  : ISS;
5845   NEON_FP : S5;
5846 %}
5847 
5848 pipe_class vunop_fp128(vecX dst, vecX src)
5849 %{
5850   single_instruction;
5851   dst    : S5(write);
5852   src    : S1(read);
5853   INS0   : ISS;
5854   NEON_FP : S5;
5855 %}
5856 
5857 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
5858 %{
5859   single_instruction;
5860   dst    : S3(write);
5861   src    : S1(read);
5862   INS01  : ISS;
5863   NEON_FP : S3;
5864 %}
5865 
5866 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
5867 %{
5868   single_instruction;
5869   dst    : S3(write);
5870   src    : S1(read);
5871   INS01  : ISS;
5872   NEON_FP : S3;
5873 %}
5874 
5875 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
5876 %{
5877   single_instruction;
5878   dst    : S3(write);
5879   src    : S1(read);
5880   INS01  : ISS;
5881   NEON_FP : S3;
5882 %}
5883 
5884 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
5885 %{
5886   single_instruction;
5887   dst    : S3(write);
5888   src    : S1(read);
5889   INS01  : ISS;
5890   NEON_FP : S3;
5891 %}
5892 
5893 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
5894 %{
5895   single_instruction;
5896   dst    : S3(write);
5897   src    : S1(read);
5898   INS01  : ISS;
5899   NEON_FP : S3;
5900 %}
5901 
5902 pipe_class vmovi_reg_imm64(vecD dst)
5903 %{
5904   single_instruction;
5905   dst    : S3(write);
5906   INS01  : ISS;
5907   NEON_FP : S3;
5908 %}
5909 
5910 pipe_class vmovi_reg_imm128(vecX dst)
5911 %{
5912   single_instruction;
5913   dst    : S3(write);
5914   INS0   : ISS;
5915   NEON_FP : S3;
5916 %}
5917 
5918 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
5919 %{
5920   single_instruction;
5921   dst    : S5(write);
5922   mem    : ISS(read);
5923   INS01  : ISS;
5924   NEON_FP : S3;
5925 %}
5926 
5927 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
5928 %{
5929   single_instruction;
5930   dst    : S5(write);
5931   mem    : ISS(read);
5932   INS01  : ISS;
5933   NEON_FP : S3;
5934 %}
5935 
5936 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
5937 %{
5938   single_instruction;
5939   mem    : ISS(read);
5940   src    : S2(read);
5941   INS01  : ISS;
5942   NEON_FP : S3;
5943 %}
5944 
5945 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
5946 %{
5947   single_instruction;
5948   mem    : ISS(read);
5949   src    : S2(read);
5950   INS01  : ISS;
5951   NEON_FP : S3;
5952 %}
5953 
5954 //------- Integer ALU operations --------------------------
5955 
5956 // Integer ALU reg-reg operation
5957 // Operands needed in EX1, result generated in EX2
5958 // Eg.  ADD     x0, x1, x2
5959 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
5960 %{
5961   single_instruction;
5962   dst    : EX2(write);
5963   src1   : EX1(read);
5964   src2   : EX1(read);
5965   INS01  : ISS; // Dual issue as instruction 0 or 1
5966   ALU    : EX2;
5967 %}
5968 
5969 // Integer ALU reg-reg operation with constant shift
5970 // Shifted register must be available in LATE_ISS instead of EX1
5971 // Eg.  ADD     x0, x1, x2, LSL #2
5972 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
5973 %{
5974   single_instruction;
5975   dst    : EX2(write);
5976   src1   : EX1(read);
5977   src2   : ISS(read);
5978   INS01  : ISS;
5979   ALU    : EX2;
5980 %}
5981 
5982 // Integer ALU reg operation with constant shift
5983 // Eg.  LSL     x0, x1, #shift
5984 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
5985 %{
5986   single_instruction;
5987   dst    : EX2(write);
5988   src1   : ISS(read);
5989   INS01  : ISS;
5990   ALU    : EX2;
5991 %}
5992 
5993 // Integer ALU reg-reg operation with variable shift
5994 // Both operands must be available in LATE_ISS instead of EX1
5995 // Result is available in EX1 instead of EX2
5996 // Eg.  LSLV    x0, x1, x2
5997 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
5998 %{
5999   single_instruction;
6000   dst    : EX1(write);
6001   src1   : ISS(read);
6002   src2   : ISS(read);
6003   INS01  : ISS;
6004   ALU    : EX1;
6005 %}
6006 
6007 // Integer ALU reg-reg operation with extract
6008 // As for _vshift above, but result generated in EX2
6009 // Eg.  EXTR    x0, x1, x2, #N
6010 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6011 %{
6012   single_instruction;
6013   dst    : EX2(write);
6014   src1   : ISS(read);
6015   src2   : ISS(read);
6016   INS1   : ISS; // Can only dual issue as Instruction 1
6017   ALU    : EX1;
6018 %}
6019 
6020 // Integer ALU reg operation
6021 // Eg.  NEG     x0, x1
6022 pipe_class ialu_reg(iRegI dst, iRegI src)
6023 %{
6024   single_instruction;
6025   dst    : EX2(write);
6026   src    : EX1(read);
6027   INS01  : ISS;
6028   ALU    : EX2;
6029 %}
6030 
6031 // Integer ALU reg mmediate operation
6032 // Eg.  ADD     x0, x1, #N
6033 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6034 %{
6035   single_instruction;
6036   dst    : EX2(write);
6037   src1   : EX1(read);
6038   INS01  : ISS;
6039   ALU    : EX2;
6040 %}
6041 
6042 // Integer ALU immediate operation (no source operands)
6043 // Eg.  MOV     x0, #N
6044 pipe_class ialu_imm(iRegI dst)
6045 %{
6046   single_instruction;
6047   dst    : EX1(write);
6048   INS01  : ISS;
6049   ALU    : EX1;
6050 %}
6051 
6052 //------- Compare operation -------------------------------
6053 
6054 // Compare reg-reg
6055 // Eg.  CMP     x0, x1
6056 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6057 %{
6058   single_instruction;
6059 //  fixed_latency(16);
6060   cr     : EX2(write);
6061   op1    : EX1(read);
6062   op2    : EX1(read);
6063   INS01  : ISS;
6064   ALU    : EX2;
6065 %}
6066 
6067 // Compare reg-reg
6068 // Eg.  CMP     x0, #N
6069 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6070 %{
6071   single_instruction;
6072 //  fixed_latency(16);
6073   cr     : EX2(write);
6074   op1    : EX1(read);
6075   INS01  : ISS;
6076   ALU    : EX2;
6077 %}
6078 
6079 //------- Conditional instructions ------------------------
6080 
6081 // Conditional no operands
6082 // Eg.  CSINC   x0, zr, zr, <cond>
6083 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6084 %{
6085   single_instruction;
6086   cr     : EX1(read);
6087   dst    : EX2(write);
6088   INS01  : ISS;
6089   ALU    : EX2;
6090 %}
6091 
6092 // Conditional 2 operand
6093 // EG.  CSEL    X0, X1, X2, <cond>
6094 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6095 %{
6096   single_instruction;
6097   cr     : EX1(read);
6098   src1   : EX1(read);
6099   src2   : EX1(read);
6100   dst    : EX2(write);
6101   INS01  : ISS;
6102   ALU    : EX2;
6103 %}
6104 
6105 // Conditional 2 operand
6106 // EG.  CSEL    X0, X1, X2, <cond>
6107 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6108 %{
6109   single_instruction;
6110   cr     : EX1(read);
6111   src    : EX1(read);
6112   dst    : EX2(write);
6113   INS01  : ISS;
6114   ALU    : EX2;
6115 %}
6116 
6117 //------- Multiply pipeline operations --------------------
6118 
6119 // Multiply reg-reg
6120 // Eg.  MUL     w0, w1, w2
6121 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6122 %{
6123   single_instruction;
6124   dst    : WR(write);
6125   src1   : ISS(read);
6126   src2   : ISS(read);
6127   INS01  : ISS;
6128   MAC    : WR;
6129 %}
6130 
6131 // Multiply accumulate
6132 // Eg.  MADD    w0, w1, w2, w3
6133 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6134 %{
6135   single_instruction;
6136   dst    : WR(write);
6137   src1   : ISS(read);
6138   src2   : ISS(read);
6139   src3   : ISS(read);
6140   INS01  : ISS;
6141   MAC    : WR;
6142 %}
6143 
6144 // Eg.  MUL     w0, w1, w2
6145 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6146 %{
6147   single_instruction;
6148   fixed_latency(3); // Maximum latency for 64 bit mul
6149   dst    : WR(write);
6150   src1   : ISS(read);
6151   src2   : ISS(read);
6152   INS01  : ISS;
6153   MAC    : WR;
6154 %}
6155 
6156 // Multiply accumulate
6157 // Eg.  MADD    w0, w1, w2, w3
6158 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6159 %{
6160   single_instruction;
6161   fixed_latency(3); // Maximum latency for 64 bit mul
6162   dst    : WR(write);
6163   src1   : ISS(read);
6164   src2   : ISS(read);
6165   src3   : ISS(read);
6166   INS01  : ISS;
6167   MAC    : WR;
6168 %}
6169 
6170 //------- Divide pipeline operations --------------------
6171 
6172 // Eg.  SDIV    w0, w1, w2
6173 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6174 %{
6175   single_instruction;
6176   fixed_latency(8); // Maximum latency for 32 bit divide
6177   dst    : WR(write);
6178   src1   : ISS(read);
6179   src2   : ISS(read);
6180   INS0   : ISS; // Can only dual issue as instruction 0
6181   DIV    : WR;
6182 %}
6183 
6184 // Eg.  SDIV    x0, x1, x2
6185 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6186 %{
6187   single_instruction;
6188   fixed_latency(16); // Maximum latency for 64 bit divide
6189   dst    : WR(write);
6190   src1   : ISS(read);
6191   src2   : ISS(read);
6192   INS0   : ISS; // Can only dual issue as instruction 0
6193   DIV    : WR;
6194 %}
6195 
6196 //------- Load pipeline operations ------------------------
6197 
6198 // Load - prefetch
6199 // Eg.  PFRM    <mem>
6200 pipe_class iload_prefetch(memory mem)
6201 %{
6202   single_instruction;
6203   mem    : ISS(read);
6204   INS01  : ISS;
6205   LDST   : WR;
6206 %}
6207 
6208 // Load - reg, mem
6209 // Eg.  LDR     x0, <mem>
6210 pipe_class iload_reg_mem(iRegI dst, memory mem)
6211 %{
6212   single_instruction;
6213   dst    : WR(write);
6214   mem    : ISS(read);
6215   INS01  : ISS;
6216   LDST   : WR;
6217 %}
6218 
6219 // Load - reg, reg
6220 // Eg.  LDR     x0, [sp, x1]
6221 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6222 %{
6223   single_instruction;
6224   dst    : WR(write);
6225   src    : ISS(read);
6226   INS01  : ISS;
6227   LDST   : WR;
6228 %}
6229 
6230 //------- Store pipeline operations -----------------------
6231 
6232 // Store - zr, mem
6233 // Eg.  STR     zr, <mem>
6234 pipe_class istore_mem(memory mem)
6235 %{
6236   single_instruction;
6237   mem    : ISS(read);
6238   INS01  : ISS;
6239   LDST   : WR;
6240 %}
6241 
6242 // Store - reg, mem
6243 // Eg.  STR     x0, <mem>
6244 pipe_class istore_reg_mem(iRegI src, memory mem)
6245 %{
6246   single_instruction;
6247   mem    : ISS(read);
6248   src    : EX2(read);
6249   INS01  : ISS;
6250   LDST   : WR;
6251 %}
6252 
6253 // Store - reg, reg
6254 // Eg. STR      x0, [sp, x1]
6255 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6256 %{
6257   single_instruction;
6258   dst    : ISS(read);
6259   src    : EX2(read);
6260   INS01  : ISS;
6261   LDST   : WR;
6262 %}
6263 
6264 //------- Store pipeline operations -----------------------
6265 
6266 // Branch
6267 pipe_class pipe_branch()
6268 %{
6269   single_instruction;
6270   INS01  : ISS;
6271   BRANCH : EX1;
6272 %}
6273 
6274 // Conditional branch
6275 pipe_class pipe_branch_cond(rFlagsReg cr)
6276 %{
6277   single_instruction;
6278   cr     : EX1(read);
6279   INS01  : ISS;
6280   BRANCH : EX1;
6281 %}
6282 
6283 // Compare & Branch
6284 // EG.  CBZ/CBNZ
6285 pipe_class pipe_cmp_branch(iRegI op1)
6286 %{
6287   single_instruction;
6288   op1    : EX1(read);
6289   INS01  : ISS;
6290   BRANCH : EX1;
6291 %}
6292 
6293 //------- Synchronisation operations ----------------------
6294 
6295 // Any operation requiring serialization.
6296 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
6297 pipe_class pipe_serial()
6298 %{
6299   single_instruction;
6300   force_serialization;
6301   fixed_latency(16);
6302   INS01  : ISS(2); // Cannot dual issue with any other instruction
6303   LDST   : WR;
6304 %}
6305 
6306 // Generic big/slow expanded idiom - also serialized
6307 pipe_class pipe_slow()
6308 %{
6309   instruction_count(10);
6310   multiple_bundles;
6311   force_serialization;
6312   fixed_latency(16);
6313   INS01  : ISS(2); // Cannot dual issue with any other instruction
6314   LDST   : WR;
6315 %}
6316 
6317 // Empty pipeline class
6318 pipe_class pipe_class_empty()
6319 %{
6320   single_instruction;
6321   fixed_latency(0);
6322 %}
6323 
6324 // Default pipeline class.
6325 pipe_class pipe_class_default()
6326 %{
6327   single_instruction;
6328   fixed_latency(2);
6329 %}
6330 
6331 // Pipeline class for compares.
6332 pipe_class pipe_class_compare()
6333 %{
6334   single_instruction;
6335   fixed_latency(16);
6336 %}
6337 
6338 // Pipeline class for memory operations.
6339 pipe_class pipe_class_memory()
6340 %{
6341   single_instruction;
6342   fixed_latency(16);
6343 %}
6344 
6345 // Pipeline class for call.
6346 pipe_class pipe_class_call()
6347 %{
6348   single_instruction;
6349   fixed_latency(100);
6350 %}
6351 
6352 // Define the class for the Nop node.
6353 define %{
6354    MachNop = pipe_class_empty;
6355 %}
6356 
6357 %}
6358 //----------INSTRUCTIONS-------------------------------------------------------
6359 //
6360 // match      -- States which machine-independent subtree may be replaced
6361 //               by this instruction.
6362 // ins_cost   -- The estimated cost of this instruction is used by instruction
6363 //               selection to identify a minimum cost tree of machine
6364 //               instructions that matches a tree of machine-independent
6365 //               instructions.
6366 // format     -- A string providing the disassembly for this instruction.
6367 //               The value of an instruction's operand may be inserted
6368 //               by referring to it with a '$' prefix.
6369 // opcode     -- Three instruction opcodes may be provided.  These are referred
6370 //               to within an encode class as $primary, $secondary, and $tertiary
6371 //               rrspectively.  The primary opcode is commonly used to
6372 //               indicate the type of machine instruction, while secondary
6373 //               and tertiary are often used for prefix options or addressing
6374 //               modes.
6375 // ins_encode -- A list of encode classes with parameters. The encode class
6376 //               name must have been defined in an 'enc_class' specification
6377 //               in the encode section of the architecture description.
6378 
6379 // ============================================================================
6380 // Memory (Load/Store) Instructions
6381 
6382 // Load Instructions
6383 
6384 // Load Byte (8 bit signed)
6385 instruct loadB(iRegINoSp dst, memory mem)
6386 %{
6387   match(Set dst (LoadB mem));
6388   predicate(!needs_acquiring_load(n));
6389 
6390   ins_cost(4 * INSN_COST);
6391   format %{ "ldrsbw  $dst, $mem\t# byte" %}
6392 
6393   ins_encode(aarch64_enc_ldrsbw(dst, mem));
6394 
6395   ins_pipe(iload_reg_mem);
6396 %}
6397 
6398 // Load Byte (8 bit signed) into long
6399 instruct loadB2L(iRegLNoSp dst, memory mem)
6400 %{
6401   match(Set dst (ConvI2L (LoadB mem)));
6402   predicate(!needs_acquiring_load(n->in(1)));
6403 
6404   ins_cost(4 * INSN_COST);
6405   format %{ "ldrsb  $dst, $mem\t# byte" %}
6406 
6407   ins_encode(aarch64_enc_ldrsb(dst, mem));
6408 
6409   ins_pipe(iload_reg_mem);
6410 %}
6411 
6412 // Load Byte (8 bit unsigned)
6413 instruct loadUB(iRegINoSp dst, memory mem)
6414 %{
6415   match(Set dst (LoadUB mem));
6416   predicate(!needs_acquiring_load(n));
6417 
6418   ins_cost(4 * INSN_COST);
6419   format %{ "ldrbw  $dst, $mem\t# byte" %}
6420 
6421   ins_encode(aarch64_enc_ldrb(dst, mem));
6422 
6423   ins_pipe(iload_reg_mem);
6424 %}
6425 
6426 // Load Byte (8 bit unsigned) into long
6427 instruct loadUB2L(iRegLNoSp dst, memory mem)
6428 %{
6429   match(Set dst (ConvI2L (LoadUB mem)));
6430   predicate(!needs_acquiring_load(n->in(1)));
6431 
6432   ins_cost(4 * INSN_COST);
6433   format %{ "ldrb  $dst, $mem\t# byte" %}
6434 
6435   ins_encode(aarch64_enc_ldrb(dst, mem));
6436 
6437   ins_pipe(iload_reg_mem);
6438 %}
6439 
6440 // Load Short (16 bit signed)
6441 instruct loadS(iRegINoSp dst, memory mem)
6442 %{
6443   match(Set dst (LoadS mem));
6444   predicate(!needs_acquiring_load(n));
6445 
6446   ins_cost(4 * INSN_COST);
6447   format %{ "ldrshw  $dst, $mem\t# short" %}
6448 
6449   ins_encode(aarch64_enc_ldrshw(dst, mem));
6450 
6451   ins_pipe(iload_reg_mem);
6452 %}
6453 
6454 // Load Short (16 bit signed) into long
6455 instruct loadS2L(iRegLNoSp dst, memory mem)
6456 %{
6457   match(Set dst (ConvI2L (LoadS mem)));
6458   predicate(!needs_acquiring_load(n->in(1)));
6459 
6460   ins_cost(4 * INSN_COST);
6461   format %{ "ldrsh  $dst, $mem\t# short" %}
6462 
6463   ins_encode(aarch64_enc_ldrsh(dst, mem));
6464 
6465   ins_pipe(iload_reg_mem);
6466 %}
6467 
6468 // Load Char (16 bit unsigned)
6469 instruct loadUS(iRegINoSp dst, memory mem)
6470 %{
6471   match(Set dst (LoadUS mem));
6472   predicate(!needs_acquiring_load(n));
6473 
6474   ins_cost(4 * INSN_COST);
6475   format %{ "ldrh  $dst, $mem\t# short" %}
6476 
6477   ins_encode(aarch64_enc_ldrh(dst, mem));
6478 
6479   ins_pipe(iload_reg_mem);
6480 %}
6481 
6482 // Load Short/Char (16 bit unsigned) into long
6483 instruct loadUS2L(iRegLNoSp dst, memory mem)
6484 %{
6485   match(Set dst (ConvI2L (LoadUS mem)));
6486   predicate(!needs_acquiring_load(n->in(1)));
6487 
6488   ins_cost(4 * INSN_COST);
6489   format %{ "ldrh  $dst, $mem\t# short" %}
6490 
6491   ins_encode(aarch64_enc_ldrh(dst, mem));
6492 
6493   ins_pipe(iload_reg_mem);
6494 %}
6495 
6496 // Load Integer (32 bit signed)
6497 instruct loadI(iRegINoSp dst, memory mem)
6498 %{
6499   match(Set dst (LoadI mem));
6500   predicate(!needs_acquiring_load(n));
6501 
6502   ins_cost(4 * INSN_COST);
6503   format %{ "ldrw  $dst, $mem\t# int" %}
6504 
6505   ins_encode(aarch64_enc_ldrw(dst, mem));
6506 
6507   ins_pipe(iload_reg_mem);
6508 %}
6509 
6510 // Load Integer (32 bit signed) into long
6511 instruct loadI2L(iRegLNoSp dst, memory mem)
6512 %{
6513   match(Set dst (ConvI2L (LoadI mem)));
6514   predicate(!needs_acquiring_load(n->in(1)));
6515 
6516   ins_cost(4 * INSN_COST);
6517   format %{ "ldrsw  $dst, $mem\t# int" %}
6518 
6519   ins_encode(aarch64_enc_ldrsw(dst, mem));
6520 
6521   ins_pipe(iload_reg_mem);
6522 %}
6523 
6524 // Load Integer (32 bit unsigned) into long
6525 instruct loadUI2L(iRegLNoSp dst, memory mem, immL_32bits mask)
6526 %{
6527   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6528   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6529 
6530   ins_cost(4 * INSN_COST);
6531   format %{ "ldrw  $dst, $mem\t# int" %}
6532 
6533   ins_encode(aarch64_enc_ldrw(dst, mem));
6534 
6535   ins_pipe(iload_reg_mem);
6536 %}
6537 
6538 // Load Long (64 bit signed)
6539 instruct loadL(iRegLNoSp dst, memory mem)
6540 %{
6541   match(Set dst (LoadL mem));
6542   predicate(!needs_acquiring_load(n));
6543 
6544   ins_cost(4 * INSN_COST);
6545   format %{ "ldr  $dst, $mem\t# int" %}
6546 
6547   ins_encode(aarch64_enc_ldr(dst, mem));
6548 
6549   ins_pipe(iload_reg_mem);
6550 %}
6551 
6552 // Load Range
6553 instruct loadRange(iRegINoSp dst, memory mem)
6554 %{
6555   match(Set dst (LoadRange mem));
6556 
6557   ins_cost(4 * INSN_COST);
6558   format %{ "ldrw  $dst, $mem\t# range" %}
6559 
6560   ins_encode(aarch64_enc_ldrw(dst, mem));
6561 
6562   ins_pipe(iload_reg_mem);
6563 %}
6564 
6565 // Load Pointer
6566 instruct loadP(iRegPNoSp dst, memory mem)
6567 %{
6568   match(Set dst (LoadP mem));
6569   predicate(!needs_acquiring_load(n));
6570 
6571   ins_cost(4 * INSN_COST);
6572   format %{ "ldr  $dst, $mem\t# ptr" %}
6573 
6574   ins_encode(aarch64_enc_ldr(dst, mem));
6575 
6576   ins_pipe(iload_reg_mem);
6577 %}
6578 
6579 // Load Compressed Pointer
6580 instruct loadN(iRegNNoSp dst, memory mem)
6581 %{
6582   match(Set dst (LoadN mem));
6583   predicate(!needs_acquiring_load(n));
6584 
6585   ins_cost(4 * INSN_COST);
6586   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
6587 
6588   ins_encode(aarch64_enc_ldrw(dst, mem));
6589 
6590   ins_pipe(iload_reg_mem);
6591 %}
6592 
6593 // Load Klass Pointer
6594 instruct loadKlass(iRegPNoSp dst, memory mem)
6595 %{
6596   match(Set dst (LoadKlass mem));
6597   predicate(!needs_acquiring_load(n));
6598 
6599   ins_cost(4 * INSN_COST);
6600   format %{ "ldr  $dst, $mem\t# class" %}
6601 
6602   ins_encode(aarch64_enc_ldr(dst, mem));
6603 
6604   ins_pipe(iload_reg_mem);
6605 %}
6606 
6607 // Load Narrow Klass Pointer
6608 instruct loadNKlass(iRegNNoSp dst, memory mem)
6609 %{
6610   match(Set dst (LoadNKlass mem));
6611   predicate(!needs_acquiring_load(n));
6612 
6613   ins_cost(4 * INSN_COST);
6614   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
6615 
6616   ins_encode(aarch64_enc_ldrw(dst, mem));
6617 
6618   ins_pipe(iload_reg_mem);
6619 %}
6620 
6621 // Load Float
6622 instruct loadF(vRegF dst, memory mem)
6623 %{
6624   match(Set dst (LoadF mem));
6625   predicate(!needs_acquiring_load(n));
6626 
6627   ins_cost(4 * INSN_COST);
6628   format %{ "ldrs  $dst, $mem\t# float" %}
6629 
6630   ins_encode( aarch64_enc_ldrs(dst, mem) );
6631 
6632   ins_pipe(pipe_class_memory);
6633 %}
6634 
6635 // Load Double
6636 instruct loadD(vRegD dst, memory mem)
6637 %{
6638   match(Set dst (LoadD mem));
6639   predicate(!needs_acquiring_load(n));
6640 
6641   ins_cost(4 * INSN_COST);
6642   format %{ "ldrd  $dst, $mem\t# double" %}
6643 
6644   ins_encode( aarch64_enc_ldrd(dst, mem) );
6645 
6646   ins_pipe(pipe_class_memory);
6647 %}
6648 
6649 
6650 // Load Int Constant
6651 instruct loadConI(iRegINoSp dst, immI src)
6652 %{
6653   match(Set dst src);
6654 
6655   ins_cost(INSN_COST);
6656   format %{ "mov $dst, $src\t# int" %}
6657 
6658   ins_encode( aarch64_enc_movw_imm(dst, src) );
6659 
6660   ins_pipe(ialu_imm);
6661 %}
6662 
6663 // Load Long Constant
6664 instruct loadConL(iRegLNoSp dst, immL src)
6665 %{
6666   match(Set dst src);
6667 
6668   ins_cost(INSN_COST);
6669   format %{ "mov $dst, $src\t# long" %}
6670 
6671   ins_encode( aarch64_enc_mov_imm(dst, src) );
6672 
6673   ins_pipe(ialu_imm);
6674 %}
6675 
6676 // Load Pointer Constant
6677 
6678 instruct loadConP(iRegPNoSp dst, immP con)
6679 %{
6680   match(Set dst con);
6681 
6682   ins_cost(INSN_COST * 4);
6683   format %{
6684     "mov  $dst, $con\t# ptr\n\t"
6685   %}
6686 
6687   ins_encode(aarch64_enc_mov_p(dst, con));
6688 
6689   ins_pipe(ialu_imm);
6690 %}
6691 
6692 // Load Null Pointer Constant
6693 
6694 instruct loadConP0(iRegPNoSp dst, immP0 con)
6695 %{
6696   match(Set dst con);
6697 
6698   ins_cost(INSN_COST);
6699   format %{ "mov  $dst, $con\t# NULL ptr" %}
6700 
6701   ins_encode(aarch64_enc_mov_p0(dst, con));
6702 
6703   ins_pipe(ialu_imm);
6704 %}
6705 
6706 // Load Pointer Constant One
6707 
6708 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6709 %{
6710   match(Set dst con);
6711 
6712   ins_cost(INSN_COST);
6713   format %{ "mov  $dst, $con\t# NULL ptr" %}
6714 
6715   ins_encode(aarch64_enc_mov_p1(dst, con));
6716 
6717   ins_pipe(ialu_imm);
6718 %}
6719 
6720 // Load Poll Page Constant
6721 
6722 instruct loadConPollPage(iRegPNoSp dst, immPollPage con)
6723 %{
6724   match(Set dst con);
6725 
6726   ins_cost(INSN_COST);
6727   format %{ "adr  $dst, $con\t# Poll Page Ptr" %}
6728 
6729   ins_encode(aarch64_enc_mov_poll_page(dst, con));
6730 
6731   ins_pipe(ialu_imm);
6732 %}
6733 
6734 // Load Byte Map Base Constant
6735 
6736 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6737 %{
6738   match(Set dst con);
6739 
6740   ins_cost(INSN_COST);
6741   format %{ "adr  $dst, $con\t# Byte Map Base" %}
6742 
6743   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
6744 
6745   ins_pipe(ialu_imm);
6746 %}
6747 
6748 // Load Narrow Pointer Constant
6749 
6750 instruct loadConN(iRegNNoSp dst, immN con)
6751 %{
6752   match(Set dst con);
6753 
6754   ins_cost(INSN_COST * 4);
6755   format %{ "mov  $dst, $con\t# compressed ptr" %}
6756 
6757   ins_encode(aarch64_enc_mov_n(dst, con));
6758 
6759   ins_pipe(ialu_imm);
6760 %}
6761 
6762 // Load Narrow Null Pointer Constant
6763 
6764 instruct loadConN0(iRegNNoSp dst, immN0 con)
6765 %{
6766   match(Set dst con);
6767 
6768   ins_cost(INSN_COST);
6769   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
6770 
6771   ins_encode(aarch64_enc_mov_n0(dst, con));
6772 
6773   ins_pipe(ialu_imm);
6774 %}
6775 
6776 // Load Narrow Klass Constant
6777 
6778 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6779 %{
6780   match(Set dst con);
6781 
6782   ins_cost(INSN_COST);
6783   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
6784 
6785   ins_encode(aarch64_enc_mov_nk(dst, con));
6786 
6787   ins_pipe(ialu_imm);
6788 %}
6789 
6790 // Load Packed Float Constant
6791 
6792 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6793   match(Set dst con);
6794   ins_cost(INSN_COST * 4);
6795   format %{ "fmovs  $dst, $con"%}
6796   ins_encode %{
6797     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6798   %}
6799 
6800   ins_pipe(fp_imm_s);
6801 %}
6802 
6803 // Load Float Constant
6804 
6805 instruct loadConF(vRegF dst, immF con) %{
6806   match(Set dst con);
6807 
6808   ins_cost(INSN_COST * 4);
6809 
6810   format %{
6811     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6812   %}
6813 
6814   ins_encode %{
6815     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6816   %}
6817 
6818   ins_pipe(fp_load_constant_s);
6819 %}
6820 
6821 // Load Packed Double Constant
6822 
6823 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6824   match(Set dst con);
6825   ins_cost(INSN_COST);
6826   format %{ "fmovd  $dst, $con"%}
6827   ins_encode %{
6828     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6829   %}
6830 
6831   ins_pipe(fp_imm_d);
6832 %}
6833 
6834 // Load Double Constant
6835 
6836 instruct loadConD(vRegD dst, immD con) %{
6837   match(Set dst con);
6838 
6839   ins_cost(INSN_COST * 5);
6840   format %{
6841     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6842   %}
6843 
6844   ins_encode %{
6845     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6846   %}
6847 
6848   ins_pipe(fp_load_constant_d);
6849 %}
6850 
6851 // Store Instructions
6852 
6853 // Store CMS card-mark Immediate
6854 instruct storeimmCM0(immI0 zero, memory mem)
6855 %{
6856   match(Set mem (StoreCM mem zero));
6857   predicate(unnecessary_storestore(n));
6858 
6859   ins_cost(INSN_COST);
6860   format %{ "storestore (elided)\n\t"
6861             "strb zr, $mem\t# byte" %}
6862 
6863   ins_encode(aarch64_enc_strb0(mem));
6864 
6865   ins_pipe(istore_mem);
6866 %}
6867 
6868 // Store CMS card-mark Immediate with intervening StoreStore
6869 // needed when using CMS with no conditional card marking
6870 instruct storeimmCM0_ordered(immI0 zero, memory mem)
6871 %{
6872   match(Set mem (StoreCM mem zero));
6873 
6874   ins_cost(INSN_COST * 2);
6875   format %{ "storestore\n\t"
6876             "dmb ishst"
6877             "\n\tstrb zr, $mem\t# byte" %}
6878 
6879   ins_encode(aarch64_enc_strb0_ordered(mem));
6880 
6881   ins_pipe(istore_mem);
6882 %}
6883 
6884 // Store Byte
6885 instruct storeB(iRegIorL2I src, memory mem)
6886 %{
6887   match(Set mem (StoreB mem src));
6888   predicate(!needs_releasing_store(n));
6889 
6890   ins_cost(INSN_COST);
6891   format %{ "strb  $src, $mem\t# byte" %}
6892 
6893   ins_encode(aarch64_enc_strb(src, mem));
6894 
6895   ins_pipe(istore_reg_mem);
6896 %}
6897 
6898 
6899 instruct storeimmB0(immI0 zero, memory mem)
6900 %{
6901   match(Set mem (StoreB mem zero));
6902   predicate(!needs_releasing_store(n));
6903 
6904   ins_cost(INSN_COST);
6905   format %{ "strb rscractch2, $mem\t# byte" %}
6906 
6907   ins_encode(aarch64_enc_strb0(mem));
6908 
6909   ins_pipe(istore_mem);
6910 %}
6911 
6912 // Store Char/Short
6913 instruct storeC(iRegIorL2I src, memory mem)
6914 %{
6915   match(Set mem (StoreC mem src));
6916   predicate(!needs_releasing_store(n));
6917 
6918   ins_cost(INSN_COST);
6919   format %{ "strh  $src, $mem\t# short" %}
6920 
6921   ins_encode(aarch64_enc_strh(src, mem));
6922 
6923   ins_pipe(istore_reg_mem);
6924 %}
6925 
6926 instruct storeimmC0(immI0 zero, memory mem)
6927 %{
6928   match(Set mem (StoreC mem zero));
6929   predicate(!needs_releasing_store(n));
6930 
6931   ins_cost(INSN_COST);
6932   format %{ "strh  zr, $mem\t# short" %}
6933 
6934   ins_encode(aarch64_enc_strh0(mem));
6935 
6936   ins_pipe(istore_mem);
6937 %}
6938 
6939 // Store Integer
6940 
6941 instruct storeI(iRegIorL2I src, memory mem)
6942 %{
6943   match(Set mem(StoreI mem src));
6944   predicate(!needs_releasing_store(n));
6945 
6946   ins_cost(INSN_COST);
6947   format %{ "strw  $src, $mem\t# int" %}
6948 
6949   ins_encode(aarch64_enc_strw(src, mem));
6950 
6951   ins_pipe(istore_reg_mem);
6952 %}
6953 
6954 instruct storeimmI0(immI0 zero, memory mem)
6955 %{
6956   match(Set mem(StoreI mem zero));
6957   predicate(!needs_releasing_store(n));
6958 
6959   ins_cost(INSN_COST);
6960   format %{ "strw  zr, $mem\t# int" %}
6961 
6962   ins_encode(aarch64_enc_strw0(mem));
6963 
6964   ins_pipe(istore_mem);
6965 %}
6966 
6967 // Store Long (64 bit signed)
6968 instruct storeL(iRegL src, memory mem)
6969 %{
6970   match(Set mem (StoreL mem src));
6971   predicate(!needs_releasing_store(n));
6972 
6973   ins_cost(INSN_COST);
6974   format %{ "str  $src, $mem\t# int" %}
6975 
6976   ins_encode(aarch64_enc_str(src, mem));
6977 
6978   ins_pipe(istore_reg_mem);
6979 %}
6980 
6981 // Store Long (64 bit signed)
6982 instruct storeimmL0(immL0 zero, memory mem)
6983 %{
6984   match(Set mem (StoreL mem zero));
6985   predicate(!needs_releasing_store(n));
6986 
6987   ins_cost(INSN_COST);
6988   format %{ "str  zr, $mem\t# int" %}
6989 
6990   ins_encode(aarch64_enc_str0(mem));
6991 
6992   ins_pipe(istore_mem);
6993 %}
6994 
6995 // Store Pointer
6996 instruct storeP(iRegP src, memory mem)
6997 %{
6998   match(Set mem (StoreP mem src));
6999   predicate(!needs_releasing_store(n));
7000 
7001   ins_cost(INSN_COST);
7002   format %{ "str  $src, $mem\t# ptr" %}
7003 
7004   ins_encode(aarch64_enc_str(src, mem));
7005 
7006   ins_pipe(istore_reg_mem);
7007 %}
7008 
7009 // Store Pointer
7010 instruct storeimmP0(immP0 zero, memory mem)
7011 %{
7012   match(Set mem (StoreP mem zero));
7013   predicate(!needs_releasing_store(n));
7014 
7015   ins_cost(INSN_COST);
7016   format %{ "str zr, $mem\t# ptr" %}
7017 
7018   ins_encode(aarch64_enc_str0(mem));
7019 
7020   ins_pipe(istore_mem);
7021 %}
7022 
7023 // Store Compressed Pointer
7024 instruct storeN(iRegN src, memory mem)
7025 %{
7026   match(Set mem (StoreN mem src));
7027   predicate(!needs_releasing_store(n));
7028 
7029   ins_cost(INSN_COST);
7030   format %{ "strw  $src, $mem\t# compressed ptr" %}
7031 
7032   ins_encode(aarch64_enc_strw(src, mem));
7033 
7034   ins_pipe(istore_reg_mem);
7035 %}
7036 
7037 instruct storeImmN0(iRegIHeapbase heapbase, immN0 zero, memory mem)
7038 %{
7039   match(Set mem (StoreN mem zero));
7040   predicate(Universe::narrow_oop_base() == NULL &&
7041             Universe::narrow_klass_base() == NULL &&
7042             (!needs_releasing_store(n)));
7043 
7044   ins_cost(INSN_COST);
7045   format %{ "strw  rheapbase, $mem\t# compressed ptr (rheapbase==0)" %}
7046 
7047   ins_encode(aarch64_enc_strw(heapbase, mem));
7048 
7049   ins_pipe(istore_reg_mem);
7050 %}
7051 
7052 // Store Float
7053 instruct storeF(vRegF src, memory mem)
7054 %{
7055   match(Set mem (StoreF mem src));
7056   predicate(!needs_releasing_store(n));
7057 
7058   ins_cost(INSN_COST);
7059   format %{ "strs  $src, $mem\t# float" %}
7060 
7061   ins_encode( aarch64_enc_strs(src, mem) );
7062 
7063   ins_pipe(pipe_class_memory);
7064 %}
7065 
7066 // TODO
7067 // implement storeImmF0 and storeFImmPacked
7068 
7069 // Store Double
7070 instruct storeD(vRegD src, memory mem)
7071 %{
7072   match(Set mem (StoreD mem src));
7073   predicate(!needs_releasing_store(n));
7074 
7075   ins_cost(INSN_COST);
7076   format %{ "strd  $src, $mem\t# double" %}
7077 
7078   ins_encode( aarch64_enc_strd(src, mem) );
7079 
7080   ins_pipe(pipe_class_memory);
7081 %}
7082 
7083 // Store Compressed Klass Pointer
7084 instruct storeNKlass(iRegN src, memory mem)
7085 %{
7086   predicate(!needs_releasing_store(n));
7087   match(Set mem (StoreNKlass mem src));
7088 
7089   ins_cost(INSN_COST);
7090   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
7091 
7092   ins_encode(aarch64_enc_strw(src, mem));
7093 
7094   ins_pipe(istore_reg_mem);
7095 %}
7096 
7097 // TODO
7098 // implement storeImmD0 and storeDImmPacked
7099 
7100 // prefetch instructions
7101 // Must be safe to execute with invalid address (cannot fault).
7102 
7103 instruct prefetchalloc( memory mem ) %{
7104   match(PrefetchAllocation mem);
7105 
7106   ins_cost(INSN_COST);
7107   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7108 
7109   ins_encode( aarch64_enc_prefetchw(mem) );
7110 
7111   ins_pipe(iload_prefetch);
7112 %}
7113 
7114 //  ---------------- volatile loads and stores ----------------
7115 
7116 // Load Byte (8 bit signed)
7117 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7118 %{
7119   match(Set dst (LoadB mem));
7120 
7121   ins_cost(VOLATILE_REF_COST);
7122   format %{ "ldarsb  $dst, $mem\t# byte" %}
7123 
7124   ins_encode(aarch64_enc_ldarsb(dst, mem));
7125 
7126   ins_pipe(pipe_serial);
7127 %}
7128 
7129 // Load Byte (8 bit signed) into long
7130 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7131 %{
7132   match(Set dst (ConvI2L (LoadB mem)));
7133 
7134   ins_cost(VOLATILE_REF_COST);
7135   format %{ "ldarsb  $dst, $mem\t# byte" %}
7136 
7137   ins_encode(aarch64_enc_ldarsb(dst, mem));
7138 
7139   ins_pipe(pipe_serial);
7140 %}
7141 
7142 // Load Byte (8 bit unsigned)
7143 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7144 %{
7145   match(Set dst (LoadUB mem));
7146 
7147   ins_cost(VOLATILE_REF_COST);
7148   format %{ "ldarb  $dst, $mem\t# byte" %}
7149 
7150   ins_encode(aarch64_enc_ldarb(dst, mem));
7151 
7152   ins_pipe(pipe_serial);
7153 %}
7154 
7155 // Load Byte (8 bit unsigned) into long
7156 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7157 %{
7158   match(Set dst (ConvI2L (LoadUB mem)));
7159 
7160   ins_cost(VOLATILE_REF_COST);
7161   format %{ "ldarb  $dst, $mem\t# byte" %}
7162 
7163   ins_encode(aarch64_enc_ldarb(dst, mem));
7164 
7165   ins_pipe(pipe_serial);
7166 %}
7167 
7168 // Load Short (16 bit signed)
7169 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7170 %{
7171   match(Set dst (LoadS mem));
7172 
7173   ins_cost(VOLATILE_REF_COST);
7174   format %{ "ldarshw  $dst, $mem\t# short" %}
7175 
7176   ins_encode(aarch64_enc_ldarshw(dst, mem));
7177 
7178   ins_pipe(pipe_serial);
7179 %}
7180 
7181 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7182 %{
7183   match(Set dst (LoadUS mem));
7184 
7185   ins_cost(VOLATILE_REF_COST);
7186   format %{ "ldarhw  $dst, $mem\t# short" %}
7187 
7188   ins_encode(aarch64_enc_ldarhw(dst, mem));
7189 
7190   ins_pipe(pipe_serial);
7191 %}
7192 
7193 // Load Short/Char (16 bit unsigned) into long
7194 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7195 %{
7196   match(Set dst (ConvI2L (LoadUS mem)));
7197 
7198   ins_cost(VOLATILE_REF_COST);
7199   format %{ "ldarh  $dst, $mem\t# short" %}
7200 
7201   ins_encode(aarch64_enc_ldarh(dst, mem));
7202 
7203   ins_pipe(pipe_serial);
7204 %}
7205 
7206 // Load Short/Char (16 bit signed) into long
7207 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7208 %{
7209   match(Set dst (ConvI2L (LoadS mem)));
7210 
7211   ins_cost(VOLATILE_REF_COST);
7212   format %{ "ldarh  $dst, $mem\t# short" %}
7213 
7214   ins_encode(aarch64_enc_ldarsh(dst, mem));
7215 
7216   ins_pipe(pipe_serial);
7217 %}
7218 
7219 // Load Integer (32 bit signed)
7220 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7221 %{
7222   match(Set dst (LoadI mem));
7223 
7224   ins_cost(VOLATILE_REF_COST);
7225   format %{ "ldarw  $dst, $mem\t# int" %}
7226 
7227   ins_encode(aarch64_enc_ldarw(dst, mem));
7228 
7229   ins_pipe(pipe_serial);
7230 %}
7231 
7232 // Load Integer (32 bit unsigned) into long
7233 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7234 %{
7235   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7236 
7237   ins_cost(VOLATILE_REF_COST);
7238   format %{ "ldarw  $dst, $mem\t# int" %}
7239 
7240   ins_encode(aarch64_enc_ldarw(dst, mem));
7241 
7242   ins_pipe(pipe_serial);
7243 %}
7244 
7245 // Load Long (64 bit signed)
7246 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7247 %{
7248   match(Set dst (LoadL mem));
7249 
7250   ins_cost(VOLATILE_REF_COST);
7251   format %{ "ldar  $dst, $mem\t# int" %}
7252 
7253   ins_encode(aarch64_enc_ldar(dst, mem));
7254 
7255   ins_pipe(pipe_serial);
7256 %}
7257 
7258 // Load Pointer
7259 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7260 %{
7261   match(Set dst (LoadP mem));
7262 
7263   ins_cost(VOLATILE_REF_COST);
7264   format %{ "ldar  $dst, $mem\t# ptr" %}
7265 
7266   ins_encode(aarch64_enc_ldar(dst, mem));
7267 
7268   ins_pipe(pipe_serial);
7269 %}
7270 
7271 // Load Compressed Pointer
7272 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7273 %{
7274   match(Set dst (LoadN mem));
7275 
7276   ins_cost(VOLATILE_REF_COST);
7277   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
7278 
7279   ins_encode(aarch64_enc_ldarw(dst, mem));
7280 
7281   ins_pipe(pipe_serial);
7282 %}
7283 
7284 // Load Float
7285 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7286 %{
7287   match(Set dst (LoadF mem));
7288 
7289   ins_cost(VOLATILE_REF_COST);
7290   format %{ "ldars  $dst, $mem\t# float" %}
7291 
7292   ins_encode( aarch64_enc_fldars(dst, mem) );
7293 
7294   ins_pipe(pipe_serial);
7295 %}
7296 
7297 // Load Double
7298 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7299 %{
7300   match(Set dst (LoadD mem));
7301 
7302   ins_cost(VOLATILE_REF_COST);
7303   format %{ "ldard  $dst, $mem\t# double" %}
7304 
7305   ins_encode( aarch64_enc_fldard(dst, mem) );
7306 
7307   ins_pipe(pipe_serial);
7308 %}
7309 
7310 // Store Byte
7311 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7312 %{
7313   match(Set mem (StoreB mem src));
7314 
7315   ins_cost(VOLATILE_REF_COST);
7316   format %{ "stlrb  $src, $mem\t# byte" %}
7317 
7318   ins_encode(aarch64_enc_stlrb(src, mem));
7319 
7320   ins_pipe(pipe_class_memory);
7321 %}
7322 
7323 // Store Char/Short
7324 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7325 %{
7326   match(Set mem (StoreC mem src));
7327 
7328   ins_cost(VOLATILE_REF_COST);
7329   format %{ "stlrh  $src, $mem\t# short" %}
7330 
7331   ins_encode(aarch64_enc_stlrh(src, mem));
7332 
7333   ins_pipe(pipe_class_memory);
7334 %}
7335 
7336 // Store Integer
7337 
7338 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7339 %{
7340   match(Set mem(StoreI mem src));
7341 
7342   ins_cost(VOLATILE_REF_COST);
7343   format %{ "stlrw  $src, $mem\t# int" %}
7344 
7345   ins_encode(aarch64_enc_stlrw(src, mem));
7346 
7347   ins_pipe(pipe_class_memory);
7348 %}
7349 
7350 // Store Long (64 bit signed)
7351 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7352 %{
7353   match(Set mem (StoreL mem src));
7354 
7355   ins_cost(VOLATILE_REF_COST);
7356   format %{ "stlr  $src, $mem\t# int" %}
7357 
7358   ins_encode(aarch64_enc_stlr(src, mem));
7359 
7360   ins_pipe(pipe_class_memory);
7361 %}
7362 
7363 // Store Pointer
7364 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7365 %{
7366   match(Set mem (StoreP mem src));
7367 
7368   ins_cost(VOLATILE_REF_COST);
7369   format %{ "stlr  $src, $mem\t# ptr" %}
7370 
7371   ins_encode(aarch64_enc_stlr(src, mem));
7372 
7373   ins_pipe(pipe_class_memory);
7374 %}
7375 
7376 // Store Compressed Pointer
7377 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7378 %{
7379   match(Set mem (StoreN mem src));
7380 
7381   ins_cost(VOLATILE_REF_COST);
7382   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
7383 
7384   ins_encode(aarch64_enc_stlrw(src, mem));
7385 
7386   ins_pipe(pipe_class_memory);
7387 %}
7388 
7389 // Store Float
7390 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7391 %{
7392   match(Set mem (StoreF mem src));
7393 
7394   ins_cost(VOLATILE_REF_COST);
7395   format %{ "stlrs  $src, $mem\t# float" %}
7396 
7397   ins_encode( aarch64_enc_fstlrs(src, mem) );
7398 
7399   ins_pipe(pipe_class_memory);
7400 %}
7401 
7402 // TODO
7403 // implement storeImmF0 and storeFImmPacked
7404 
7405 // Store Double
7406 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7407 %{
7408   match(Set mem (StoreD mem src));
7409 
7410   ins_cost(VOLATILE_REF_COST);
7411   format %{ "stlrd  $src, $mem\t# double" %}
7412 
7413   ins_encode( aarch64_enc_fstlrd(src, mem) );
7414 
7415   ins_pipe(pipe_class_memory);
7416 %}
7417 
7418 //  ---------------- end of volatile loads and stores ----------------
7419 
7420 // ============================================================================
7421 // BSWAP Instructions
7422 
7423 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7424   match(Set dst (ReverseBytesI src));
7425 
7426   ins_cost(INSN_COST);
7427   format %{ "revw  $dst, $src" %}
7428 
7429   ins_encode %{
7430     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7431   %}
7432 
7433   ins_pipe(ialu_reg);
7434 %}
7435 
7436 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7437   match(Set dst (ReverseBytesL src));
7438 
7439   ins_cost(INSN_COST);
7440   format %{ "rev  $dst, $src" %}
7441 
7442   ins_encode %{
7443     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7444   %}
7445 
7446   ins_pipe(ialu_reg);
7447 %}
7448 
7449 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7450   match(Set dst (ReverseBytesUS src));
7451 
7452   ins_cost(INSN_COST);
7453   format %{ "rev16w  $dst, $src" %}
7454 
7455   ins_encode %{
7456     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7457   %}
7458 
7459   ins_pipe(ialu_reg);
7460 %}
7461 
7462 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7463   match(Set dst (ReverseBytesS src));
7464 
7465   ins_cost(INSN_COST);
7466   format %{ "rev16w  $dst, $src\n\t"
7467             "sbfmw $dst, $dst, #0, #15" %}
7468 
7469   ins_encode %{
7470     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7471     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7472   %}
7473 
7474   ins_pipe(ialu_reg);
7475 %}
7476 
7477 // ============================================================================
7478 // Zero Count Instructions
7479 
7480 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7481   match(Set dst (CountLeadingZerosI src));
7482 
7483   ins_cost(INSN_COST);
7484   format %{ "clzw  $dst, $src" %}
7485   ins_encode %{
7486     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7487   %}
7488 
7489   ins_pipe(ialu_reg);
7490 %}
7491 
7492 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7493   match(Set dst (CountLeadingZerosL src));
7494 
7495   ins_cost(INSN_COST);
7496   format %{ "clz   $dst, $src" %}
7497   ins_encode %{
7498     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7499   %}
7500 
7501   ins_pipe(ialu_reg);
7502 %}
7503 
7504 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7505   match(Set dst (CountTrailingZerosI src));
7506 
7507   ins_cost(INSN_COST * 2);
7508   format %{ "rbitw  $dst, $src\n\t"
7509             "clzw   $dst, $dst" %}
7510   ins_encode %{
7511     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7512     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7513   %}
7514 
7515   ins_pipe(ialu_reg);
7516 %}
7517 
7518 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7519   match(Set dst (CountTrailingZerosL src));
7520 
7521   ins_cost(INSN_COST * 2);
7522   format %{ "rbit   $dst, $src\n\t"
7523             "clz    $dst, $dst" %}
7524   ins_encode %{
7525     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7526     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7527   %}
7528 
7529   ins_pipe(ialu_reg);
7530 %}
7531 
7532 //---------- Population Count Instructions -------------------------------------
7533 //
7534 
7535 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7536   predicate(UsePopCountInstruction);
7537   match(Set dst (PopCountI src));
7538   effect(TEMP tmp);
7539   ins_cost(INSN_COST * 13);
7540 
7541   format %{ "movw   $src, $src\n\t"
7542             "mov    $tmp, $src\t# vector (1D)\n\t"
7543             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
7544             "addv   $tmp, $tmp\t# vector (8B)\n\t"
7545             "mov    $dst, $tmp\t# vector (1D)" %}
7546   ins_encode %{
7547     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7548     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
7549     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7550     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7551     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7552   %}
7553 
7554   ins_pipe(pipe_class_default);
7555 %}
7556 
7557 instruct popCountI_mem(iRegINoSp dst, memory mem, vRegF tmp) %{
7558   predicate(UsePopCountInstruction);
7559   match(Set dst (PopCountI (LoadI mem)));
7560   effect(TEMP tmp);
7561   ins_cost(INSN_COST * 13);
7562 
7563   format %{ "ldrs   $tmp, $mem\n\t"
7564             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
7565             "addv   $tmp, $tmp\t# vector (8B)\n\t"
7566             "mov    $dst, $tmp\t# vector (1D)" %}
7567   ins_encode %{
7568     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7569     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7570                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
7571     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7572     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7573     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7574   %}
7575 
7576   ins_pipe(pipe_class_default);
7577 %}
7578 
7579 // Note: Long.bitCount(long) returns an int.
7580 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7581   predicate(UsePopCountInstruction);
7582   match(Set dst (PopCountL src));
7583   effect(TEMP tmp);
7584   ins_cost(INSN_COST * 13);
7585 
7586   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
7587             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
7588             "addv   $tmp, $tmp\t# vector (8B)\n\t"
7589             "mov    $dst, $tmp\t# vector (1D)" %}
7590   ins_encode %{
7591     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
7592     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7593     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7594     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7595   %}
7596 
7597   ins_pipe(pipe_class_default);
7598 %}
7599 
7600 instruct popCountL_mem(iRegINoSp dst, memory mem, vRegD tmp) %{
7601   predicate(UsePopCountInstruction);
7602   match(Set dst (PopCountL (LoadL mem)));
7603   effect(TEMP tmp);
7604   ins_cost(INSN_COST * 13);
7605 
7606   format %{ "ldrd   $tmp, $mem\n\t"
7607             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
7608             "addv   $tmp, $tmp\t# vector (8B)\n\t"
7609             "mov    $dst, $tmp\t# vector (1D)" %}
7610   ins_encode %{
7611     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7612     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7613                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
7614     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7615     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7616     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7617   %}
7618 
7619   ins_pipe(pipe_class_default);
7620 %}
7621 
7622 // ============================================================================
7623 // MemBar Instruction
7624 
7625 instruct load_fence() %{
7626   match(LoadFence);
7627   ins_cost(VOLATILE_REF_COST);
7628 
7629   format %{ "load_fence" %}
7630 
7631   ins_encode %{
7632     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7633   %}
7634   ins_pipe(pipe_serial);
7635 %}
7636 
7637 instruct unnecessary_membar_acquire() %{
7638   predicate(unnecessary_acquire(n));
7639   match(MemBarAcquire);
7640   ins_cost(0);
7641 
7642   format %{ "membar_acquire (elided)" %}
7643 
7644   ins_encode %{
7645     __ block_comment("membar_acquire (elided)");
7646   %}
7647 
7648   ins_pipe(pipe_class_empty);
7649 %}
7650 
7651 instruct membar_acquire() %{
7652   match(MemBarAcquire);
7653   ins_cost(VOLATILE_REF_COST);
7654 
7655   format %{ "membar_acquire\n\t"
7656             "dmb ish" %}
7657 
7658   ins_encode %{
7659     __ block_comment("membar_acquire");
7660     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7661   %}
7662 
7663   ins_pipe(pipe_serial);
7664 %}
7665 
7666 
7667 instruct membar_acquire_lock() %{
7668   match(MemBarAcquireLock);
7669   ins_cost(VOLATILE_REF_COST);
7670 
7671   format %{ "membar_acquire_lock (elided)" %}
7672 
7673   ins_encode %{
7674     __ block_comment("membar_acquire_lock (elided)");
7675   %}
7676 
7677   ins_pipe(pipe_serial);
7678 %}
7679 
7680 instruct store_fence() %{
7681   match(StoreFence);
7682   ins_cost(VOLATILE_REF_COST);
7683 
7684   format %{ "store_fence" %}
7685 
7686   ins_encode %{
7687     __ membar(Assembler::LoadStore|Assembler::StoreStore);
7688   %}
7689   ins_pipe(pipe_serial);
7690 %}
7691 
7692 instruct unnecessary_membar_release() %{
7693   predicate(unnecessary_release(n));
7694   match(MemBarRelease);
7695   ins_cost(0);
7696 
7697   format %{ "membar_release (elided)" %}
7698 
7699   ins_encode %{
7700     __ block_comment("membar_release (elided)");
7701   %}
7702   ins_pipe(pipe_serial);
7703 %}
7704 
7705 instruct membar_release() %{
7706   match(MemBarRelease);
7707   ins_cost(VOLATILE_REF_COST);
7708 
7709   format %{ "membar_release\n\t"
7710             "dmb ish" %}
7711 
7712   ins_encode %{
7713     __ block_comment("membar_release");
7714     __ membar(Assembler::LoadStore|Assembler::StoreStore);
7715   %}
7716   ins_pipe(pipe_serial);
7717 %}
7718 
7719 instruct membar_storestore() %{
7720   match(MemBarStoreStore);
7721   ins_cost(VOLATILE_REF_COST);
7722 
7723   format %{ "MEMBAR-store-store" %}
7724 
7725   ins_encode %{
7726     __ membar(Assembler::StoreStore);
7727   %}
7728   ins_pipe(pipe_serial);
7729 %}
7730 
7731 instruct membar_release_lock() %{
7732   match(MemBarReleaseLock);
7733   ins_cost(VOLATILE_REF_COST);
7734 
7735   format %{ "membar_release_lock (elided)" %}
7736 
7737   ins_encode %{
7738     __ block_comment("membar_release_lock (elided)");
7739   %}
7740 
7741   ins_pipe(pipe_serial);
7742 %}
7743 
7744 instruct unnecessary_membar_volatile() %{
7745   predicate(unnecessary_volatile(n));
7746   match(MemBarVolatile);
7747   ins_cost(0);
7748 
7749   format %{ "membar_volatile (elided)" %}
7750 
7751   ins_encode %{
7752     __ block_comment("membar_volatile (elided)");
7753   %}
7754 
7755   ins_pipe(pipe_serial);
7756 %}
7757 
7758 instruct membar_volatile() %{
7759   match(MemBarVolatile);
7760   ins_cost(VOLATILE_REF_COST*100);
7761 
7762   format %{ "membar_volatile\n\t"
7763              "dmb ish"%}
7764 
7765   ins_encode %{
7766     __ block_comment("membar_volatile");
7767     __ membar(Assembler::StoreLoad);
7768   %}
7769 
7770   ins_pipe(pipe_serial);
7771 %}
7772 
7773 // ============================================================================
7774 // Cast/Convert Instructions
7775 
7776 instruct castX2P(iRegPNoSp dst, iRegL src) %{
7777   match(Set dst (CastX2P src));
7778 
7779   ins_cost(INSN_COST);
7780   format %{ "mov $dst, $src\t# long -> ptr" %}
7781 
7782   ins_encode %{
7783     if ($dst$$reg != $src$$reg) {
7784       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7785     }
7786   %}
7787 
7788   ins_pipe(ialu_reg);
7789 %}
7790 
7791 instruct castP2X(iRegLNoSp dst, iRegP src) %{
7792   match(Set dst (CastP2X src));
7793 
7794   ins_cost(INSN_COST);
7795   format %{ "mov $dst, $src\t# ptr -> long" %}
7796 
7797   ins_encode %{
7798     if ($dst$$reg != $src$$reg) {
7799       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7800     }
7801   %}
7802 
7803   ins_pipe(ialu_reg);
7804 %}
7805 
7806 // Convert oop into int for vectors alignment masking
7807 instruct convP2I(iRegINoSp dst, iRegP src) %{
7808   match(Set dst (ConvL2I (CastP2X src)));
7809 
7810   ins_cost(INSN_COST);
7811   format %{ "movw $dst, $src\t# ptr -> int" %}
7812   ins_encode %{
7813     __ movw($dst$$Register, $src$$Register);
7814   %}
7815 
7816   ins_pipe(ialu_reg);
7817 %}
7818 
7819 // Convert compressed oop into int for vectors alignment masking
7820 // in case of 32bit oops (heap < 4Gb).
7821 instruct convN2I(iRegINoSp dst, iRegN src)
7822 %{
7823   predicate(Universe::narrow_oop_shift() == 0);
7824   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
7825 
7826   ins_cost(INSN_COST);
7827   format %{ "mov dst, $src\t# compressed ptr -> int" %}
7828   ins_encode %{
7829     __ movw($dst$$Register, $src$$Register);
7830   %}
7831 
7832   ins_pipe(ialu_reg);
7833 %}
7834 
7835 
7836 // Convert oop pointer into compressed form
7837 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
7838   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
7839   match(Set dst (EncodeP src));
7840   effect(KILL cr);
7841   ins_cost(INSN_COST * 3);
7842   format %{ "encode_heap_oop $dst, $src" %}
7843   ins_encode %{
7844     Register s = $src$$Register;
7845     Register d = $dst$$Register;
7846     __ encode_heap_oop(d, s);
7847   %}
7848   ins_pipe(ialu_reg);
7849 %}
7850 
7851 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
7852   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
7853   match(Set dst (EncodeP src));
7854   ins_cost(INSN_COST * 3);
7855   format %{ "encode_heap_oop_not_null $dst, $src" %}
7856   ins_encode %{
7857     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
7858   %}
7859   ins_pipe(ialu_reg);
7860 %}
7861 
7862 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
7863   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
7864             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
7865   match(Set dst (DecodeN src));
7866   ins_cost(INSN_COST * 3);
7867   format %{ "decode_heap_oop $dst, $src" %}
7868   ins_encode %{
7869     Register s = $src$$Register;
7870     Register d = $dst$$Register;
7871     __ decode_heap_oop(d, s);
7872   %}
7873   ins_pipe(ialu_reg);
7874 %}
7875 
7876 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
7877   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
7878             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
7879   match(Set dst (DecodeN src));
7880   ins_cost(INSN_COST * 3);
7881   format %{ "decode_heap_oop_not_null $dst, $src" %}
7882   ins_encode %{
7883     Register s = $src$$Register;
7884     Register d = $dst$$Register;
7885     __ decode_heap_oop_not_null(d, s);
7886   %}
7887   ins_pipe(ialu_reg);
7888 %}
7889 
7890 // n.b. AArch64 implementations of encode_klass_not_null and
7891 // decode_klass_not_null do not modify the flags register so, unlike
7892 // Intel, we don't kill CR as a side effect here
7893 
7894 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
7895   match(Set dst (EncodePKlass src));
7896 
7897   ins_cost(INSN_COST * 3);
7898   format %{ "encode_klass_not_null $dst,$src" %}
7899 
7900   ins_encode %{
7901     Register src_reg = as_Register($src$$reg);
7902     Register dst_reg = as_Register($dst$$reg);
7903     __ encode_klass_not_null(dst_reg, src_reg);
7904   %}
7905 
7906    ins_pipe(ialu_reg);
7907 %}
7908 
7909 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
7910   match(Set dst (DecodeNKlass src));
7911 
7912   ins_cost(INSN_COST * 3);
7913   format %{ "decode_klass_not_null $dst,$src" %}
7914 
7915   ins_encode %{
7916     Register src_reg = as_Register($src$$reg);
7917     Register dst_reg = as_Register($dst$$reg);
7918     if (dst_reg != src_reg) {
7919       __ decode_klass_not_null(dst_reg, src_reg);
7920     } else {
7921       __ decode_klass_not_null(dst_reg);
7922     }
7923   %}
7924 
7925    ins_pipe(ialu_reg);
7926 %}
7927 
7928 instruct checkCastPP(iRegPNoSp dst)
7929 %{
7930   match(Set dst (CheckCastPP dst));
7931 
7932   size(0);
7933   format %{ "# checkcastPP of $dst" %}
7934   ins_encode(/* empty encoding */);
7935   ins_pipe(pipe_class_empty);
7936 %}
7937 
7938 instruct castPP(iRegPNoSp dst)
7939 %{
7940   match(Set dst (CastPP dst));
7941 
7942   size(0);
7943   format %{ "# castPP of $dst" %}
7944   ins_encode(/* empty encoding */);
7945   ins_pipe(pipe_class_empty);
7946 %}
7947 
7948 instruct castII(iRegI dst)
7949 %{
7950   match(Set dst (CastII dst));
7951 
7952   size(0);
7953   format %{ "# castII of $dst" %}
7954   ins_encode(/* empty encoding */);
7955   ins_cost(0);
7956   ins_pipe(pipe_class_empty);
7957 %}
7958 
7959 // ============================================================================
7960 // Atomic operation instructions
7961 //
7962 // Intel and SPARC both implement Ideal Node LoadPLocked and
7963 // Store{PIL}Conditional instructions using a normal load for the
7964 // LoadPLocked and a CAS for the Store{PIL}Conditional.
7965 //
7966 // The ideal code appears only to use LoadPLocked/StorePLocked as a
7967 // pair to lock object allocations from Eden space when not using
7968 // TLABs.
7969 //
7970 // There does not appear to be a Load{IL}Locked Ideal Node and the
7971 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
7972 // and to use StoreIConditional only for 32-bit and StoreLConditional
7973 // only for 64-bit.
7974 //
7975 // We implement LoadPLocked and StorePLocked instructions using,
7976 // respectively the AArch64 hw load-exclusive and store-conditional
7977 // instructions. Whereas we must implement each of
7978 // Store{IL}Conditional using a CAS which employs a pair of
7979 // instructions comprising a load-exclusive followed by a
7980 // store-conditional.
7981 
7982 
7983 // Locked-load (linked load) of the current heap-top
7984 // used when updating the eden heap top
7985 // implemented using ldaxr on AArch64
7986 
7987 instruct loadPLocked(iRegPNoSp dst, indirect mem)
7988 %{
7989   match(Set dst (LoadPLocked mem));
7990 
7991   ins_cost(VOLATILE_REF_COST);
7992 
7993   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
7994 
7995   ins_encode(aarch64_enc_ldaxr(dst, mem));
7996 
7997   ins_pipe(pipe_serial);
7998 %}
7999 
8000 // Conditional-store of the updated heap-top.
8001 // Used during allocation of the shared heap.
8002 // Sets flag (EQ) on success.
8003 // implemented using stlxr on AArch64.
8004 
8005 instruct storePConditional(memory heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
8006 %{
8007   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
8008 
8009   ins_cost(VOLATILE_REF_COST);
8010 
8011  // TODO
8012  // do we need to do a store-conditional release or can we just use a
8013  // plain store-conditional?
8014 
8015   format %{
8016     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
8017     "cmpw rscratch1, zr\t# EQ on successful write"
8018   %}
8019 
8020   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
8021 
8022   ins_pipe(pipe_serial);
8023 %}
8024 
8025 
8026 // storeLConditional is used by PhaseMacroExpand::expand_lock_node
8027 // when attempting to rebias a lock towards the current thread.  We
8028 // must use the acquire form of cmpxchg in order to guarantee acquire
8029 // semantics in this case.
8030 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
8031 %{
8032   match(Set cr (StoreLConditional mem (Binary oldval newval)));
8033 
8034   ins_cost(VOLATILE_REF_COST);
8035 
8036   format %{
8037     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
8038     "cmpw rscratch1, zr\t# EQ on successful write"
8039   %}
8040 
8041   ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
8042 
8043   ins_pipe(pipe_slow);
8044 %}
8045 
8046 // storeIConditional also has acquire semantics, for no better reason
8047 // than matching storeLConditional.  At the time of writing this
8048 // comment storeIConditional was not used anywhere by AArch64.
8049 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
8050 %{
8051   match(Set cr (StoreIConditional mem (Binary oldval newval)));
8052 
8053   ins_cost(VOLATILE_REF_COST);
8054 
8055   format %{
8056     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
8057     "cmpw rscratch1, zr\t# EQ on successful write"
8058   %}
8059 
8060   ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
8061 
8062   ins_pipe(pipe_slow);
8063 %}
8064 
8065 // standard CompareAndSwapX when we are using barriers
8066 // these have higher priority than the rules selected by a predicate
8067 
8068 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8069 // can't match them
8070 
8071 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8072 
8073   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8074   ins_cost(2 * VOLATILE_REF_COST);
8075 
8076   effect(KILL cr);
8077 
8078   format %{
8079     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8080     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8081   %}
8082 
8083   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8084             aarch64_enc_cset_eq(res));
8085 
8086   ins_pipe(pipe_slow);
8087 %}
8088 
8089 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8090 
8091   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8092   ins_cost(2 * VOLATILE_REF_COST);
8093 
8094   effect(KILL cr);
8095 
8096   format %{
8097     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8098     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8099   %}
8100 
8101   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8102             aarch64_enc_cset_eq(res));
8103 
8104   ins_pipe(pipe_slow);
8105 %}
8106 
8107 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8108 
8109   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8110   ins_cost(2 * VOLATILE_REF_COST);
8111 
8112   effect(KILL cr);
8113 
8114  format %{
8115     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8116     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8117  %}
8118 
8119  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8120             aarch64_enc_cset_eq(res));
8121 
8122   ins_pipe(pipe_slow);
8123 %}
8124 
8125 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8126 
8127   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8128   ins_cost(2 * VOLATILE_REF_COST);
8129 
8130   effect(KILL cr);
8131 
8132  format %{
8133     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8134     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8135  %}
8136 
8137  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8138             aarch64_enc_cset_eq(res));
8139 
8140   ins_pipe(pipe_slow);
8141 %}
8142 
8143 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8144 
8145   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8146   ins_cost(2 * VOLATILE_REF_COST);
8147 
8148   effect(KILL cr);
8149 
8150  format %{
8151     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8152     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8153  %}
8154 
8155  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8156             aarch64_enc_cset_eq(res));
8157 
8158   ins_pipe(pipe_slow);
8159 %}
8160 
8161 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8162 
8163   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8164   ins_cost(2 * VOLATILE_REF_COST);
8165 
8166   effect(KILL cr);
8167 
8168  format %{
8169     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8170     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8171  %}
8172 
8173  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8174             aarch64_enc_cset_eq(res));
8175 
8176   ins_pipe(pipe_slow);
8177 %}
8178 
8179 // alternative CompareAndSwapX when we are eliding barriers
8180 
8181 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8182 
8183   predicate(needs_acquiring_load_exclusive(n));
8184   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8185   ins_cost(VOLATILE_REF_COST);
8186 
8187   effect(KILL cr);
8188 
8189   format %{
8190     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8191     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8192   %}
8193 
8194   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8195             aarch64_enc_cset_eq(res));
8196 
8197   ins_pipe(pipe_slow);
8198 %}
8199 
8200 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8201 
8202   predicate(needs_acquiring_load_exclusive(n));
8203   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8204   ins_cost(VOLATILE_REF_COST);
8205 
8206   effect(KILL cr);
8207 
8208   format %{
8209     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8210     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8211   %}
8212 
8213   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8214             aarch64_enc_cset_eq(res));
8215 
8216   ins_pipe(pipe_slow);
8217 %}
8218 
8219 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8220 
8221   predicate(needs_acquiring_load_exclusive(n));
8222   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8223   ins_cost(VOLATILE_REF_COST);
8224 
8225   effect(KILL cr);
8226 
8227  format %{
8228     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8229     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8230  %}
8231 
8232  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8233             aarch64_enc_cset_eq(res));
8234 
8235   ins_pipe(pipe_slow);
8236 %}
8237 
8238 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8239 
8240   predicate(needs_acquiring_load_exclusive(n));
8241   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8242   ins_cost(VOLATILE_REF_COST);
8243 
8244   effect(KILL cr);
8245 
8246  format %{
8247     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8248     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8249  %}
8250 
8251  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8252             aarch64_enc_cset_eq(res));
8253 
8254   ins_pipe(pipe_slow);
8255 %}
8256 
8257 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8258 
8259   predicate(needs_acquiring_load_exclusive(n));
8260   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8261   ins_cost(VOLATILE_REF_COST);
8262 
8263   effect(KILL cr);
8264 
8265  format %{
8266     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8267     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8268  %}
8269 
8270  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8271             aarch64_enc_cset_eq(res));
8272 
8273   ins_pipe(pipe_slow);
8274 %}
8275 
8276 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8277 
8278   predicate(needs_acquiring_load_exclusive(n));
8279   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8280   ins_cost(VOLATILE_REF_COST);
8281 
8282   effect(KILL cr);
8283 
8284  format %{
8285     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8286     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8287  %}
8288 
8289  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8290             aarch64_enc_cset_eq(res));
8291 
8292   ins_pipe(pipe_slow);
8293 %}
8294 
8295 
8296 // ---------------------------------------------------------------------
8297 
8298 
8299 // BEGIN This section of the file is automatically generated. Do not edit --------------
8300 
8301 // Sundry CAS operations.  Note that release is always true,
8302 // regardless of the memory ordering of the CAS.  This is because we
8303 // need the volatile case to be sequentially consistent but there is
8304 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
8305 // can't check the type of memory ordering here, so we always emit a
8306 // STLXR.
8307 
8308 // This section is generated from aarch64_ad_cas.m4
8309 
8310 
8311 
8312 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8313   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8314   ins_cost(2 * VOLATILE_REF_COST);
8315   effect(TEMP_DEF res, KILL cr);
8316   format %{
8317     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8318   %}
8319   ins_encode %{
8320     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8321                Assembler::byte, /*acquire*/ false, /*release*/ true,
8322                /*weak*/ false, $res$$Register);
8323     __ sxtbw($res$$Register, $res$$Register);
8324   %}
8325   ins_pipe(pipe_slow);
8326 %}
8327 
8328 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8329   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8330   ins_cost(2 * VOLATILE_REF_COST);
8331   effect(TEMP_DEF res, KILL cr);
8332   format %{
8333     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8334   %}
8335   ins_encode %{
8336     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8337                Assembler::halfword, /*acquire*/ false, /*release*/ true,
8338                /*weak*/ false, $res$$Register);
8339     __ sxthw($res$$Register, $res$$Register);
8340   %}
8341   ins_pipe(pipe_slow);
8342 %}
8343 
8344 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8345   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8346   ins_cost(2 * VOLATILE_REF_COST);
8347   effect(TEMP_DEF res, KILL cr);
8348   format %{
8349     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8350   %}
8351   ins_encode %{
8352     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8353                Assembler::word, /*acquire*/ false, /*release*/ true,
8354                /*weak*/ false, $res$$Register);
8355   %}
8356   ins_pipe(pipe_slow);
8357 %}
8358 
8359 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8360   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8361   ins_cost(2 * VOLATILE_REF_COST);
8362   effect(TEMP_DEF res, KILL cr);
8363   format %{
8364     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8365   %}
8366   ins_encode %{
8367     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8368                Assembler::xword, /*acquire*/ false, /*release*/ true,
8369                /*weak*/ false, $res$$Register);
8370   %}
8371   ins_pipe(pipe_slow);
8372 %}
8373 
8374 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8375   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8376   ins_cost(2 * VOLATILE_REF_COST);
8377   effect(TEMP_DEF res, KILL cr);
8378   format %{
8379     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8380   %}
8381   ins_encode %{
8382     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8383                Assembler::word, /*acquire*/ false, /*release*/ true,
8384                /*weak*/ false, $res$$Register);
8385   %}
8386   ins_pipe(pipe_slow);
8387 %}
8388 
8389 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8390   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8391   ins_cost(2 * VOLATILE_REF_COST);
8392   effect(TEMP_DEF res, KILL cr);
8393   format %{
8394     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8395   %}
8396   ins_encode %{
8397     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8398                Assembler::xword, /*acquire*/ false, /*release*/ true,
8399                /*weak*/ false, $res$$Register);
8400   %}
8401   ins_pipe(pipe_slow);
8402 %}
8403 
8404 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8405   predicate(needs_acquiring_load_exclusive(n));
8406   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8407   ins_cost(VOLATILE_REF_COST);
8408   effect(TEMP_DEF res, KILL cr);
8409   format %{
8410     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8411   %}
8412   ins_encode %{
8413     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8414                Assembler::byte, /*acquire*/ true, /*release*/ true,
8415                /*weak*/ false, $res$$Register);
8416     __ sxtbw($res$$Register, $res$$Register);
8417   %}
8418   ins_pipe(pipe_slow);
8419 %}
8420 
8421 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8422   predicate(needs_acquiring_load_exclusive(n));
8423   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8424   ins_cost(VOLATILE_REF_COST);
8425   effect(TEMP_DEF res, KILL cr);
8426   format %{
8427     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8428   %}
8429   ins_encode %{
8430     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8431                Assembler::halfword, /*acquire*/ true, /*release*/ true,
8432                /*weak*/ false, $res$$Register);
8433     __ sxthw($res$$Register, $res$$Register);
8434   %}
8435   ins_pipe(pipe_slow);
8436 %}
8437 
8438 
8439 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8440   predicate(needs_acquiring_load_exclusive(n));
8441   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8442   ins_cost(VOLATILE_REF_COST);
8443   effect(TEMP_DEF res, KILL cr);
8444   format %{
8445     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8446   %}
8447   ins_encode %{
8448     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8449                Assembler::word, /*acquire*/ true, /*release*/ true,
8450                /*weak*/ false, $res$$Register);
8451   %}
8452   ins_pipe(pipe_slow);
8453 %}
8454 
8455 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8456   predicate(needs_acquiring_load_exclusive(n));
8457   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8458   ins_cost(VOLATILE_REF_COST);
8459   effect(TEMP_DEF res, KILL cr);
8460   format %{
8461     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8462   %}
8463   ins_encode %{
8464     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8465                Assembler::xword, /*acquire*/ true, /*release*/ true,
8466                /*weak*/ false, $res$$Register);
8467   %}
8468   ins_pipe(pipe_slow);
8469 %}
8470 
8471 
8472 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8473   predicate(needs_acquiring_load_exclusive(n));
8474   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8475   ins_cost(VOLATILE_REF_COST);
8476   effect(TEMP_DEF res, KILL cr);
8477   format %{
8478     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8479   %}
8480   ins_encode %{
8481     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8482                Assembler::word, /*acquire*/ true, /*release*/ true,
8483                /*weak*/ false, $res$$Register);
8484   %}
8485   ins_pipe(pipe_slow);
8486 %}
8487 
8488 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8489   predicate(needs_acquiring_load_exclusive(n));
8490   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8491   ins_cost(VOLATILE_REF_COST);
8492   effect(TEMP_DEF res, KILL cr);
8493   format %{
8494     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8495   %}
8496   ins_encode %{
8497     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8498                Assembler::xword, /*acquire*/ true, /*release*/ true,
8499                /*weak*/ false, $res$$Register);
8500   %}
8501   ins_pipe(pipe_slow);
8502 %}
8503 
8504 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8505   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8506   ins_cost(2 * VOLATILE_REF_COST);
8507   effect(KILL cr);
8508   format %{
8509     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8510     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8511   %}
8512   ins_encode %{
8513     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8514                Assembler::byte, /*acquire*/ false, /*release*/ true,
8515                /*weak*/ true, noreg);
8516     __ csetw($res$$Register, Assembler::EQ);
8517   %}
8518   ins_pipe(pipe_slow);
8519 %}
8520 
8521 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8522   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8523   ins_cost(2 * VOLATILE_REF_COST);
8524   effect(KILL cr);
8525   format %{
8526     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8527     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8528   %}
8529   ins_encode %{
8530     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8531                Assembler::halfword, /*acquire*/ false, /*release*/ true,
8532                /*weak*/ true, noreg);
8533     __ csetw($res$$Register, Assembler::EQ);
8534   %}
8535   ins_pipe(pipe_slow);
8536 %}
8537 
8538 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8539   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8540   ins_cost(2 * VOLATILE_REF_COST);
8541   effect(KILL cr);
8542   format %{
8543     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8544     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8545   %}
8546   ins_encode %{
8547     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8548                Assembler::word, /*acquire*/ false, /*release*/ true,
8549                /*weak*/ true, noreg);
8550     __ csetw($res$$Register, Assembler::EQ);
8551   %}
8552   ins_pipe(pipe_slow);
8553 %}
8554 
8555 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8556   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8557   ins_cost(2 * VOLATILE_REF_COST);
8558   effect(KILL cr);
8559   format %{
8560     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8561     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8562   %}
8563   ins_encode %{
8564     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8565                Assembler::xword, /*acquire*/ false, /*release*/ true,
8566                /*weak*/ true, noreg);
8567     __ csetw($res$$Register, Assembler::EQ);
8568   %}
8569   ins_pipe(pipe_slow);
8570 %}
8571 
8572 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8573   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8574   ins_cost(2 * VOLATILE_REF_COST);
8575   effect(KILL cr);
8576   format %{
8577     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8578     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8579   %}
8580   ins_encode %{
8581     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8582                Assembler::word, /*acquire*/ false, /*release*/ true,
8583                /*weak*/ true, noreg);
8584     __ csetw($res$$Register, Assembler::EQ);
8585   %}
8586   ins_pipe(pipe_slow);
8587 %}
8588 
8589 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8590   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8591   ins_cost(2 * VOLATILE_REF_COST);
8592   effect(KILL cr);
8593   format %{
8594     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8595     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8596   %}
8597   ins_encode %{
8598     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8599                Assembler::xword, /*acquire*/ false, /*release*/ true,
8600                /*weak*/ true, noreg);
8601     __ csetw($res$$Register, Assembler::EQ);
8602   %}
8603   ins_pipe(pipe_slow);
8604 %}
8605 
8606 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8607   predicate(needs_acquiring_load_exclusive(n));
8608   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8609   ins_cost(VOLATILE_REF_COST);
8610   effect(KILL cr);
8611   format %{
8612     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8613     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8614   %}
8615   ins_encode %{
8616     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8617                Assembler::byte, /*acquire*/ true, /*release*/ true,
8618                /*weak*/ true, noreg);
8619     __ csetw($res$$Register, Assembler::EQ);
8620   %}
8621   ins_pipe(pipe_slow);
8622 %}
8623 
8624 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8625   predicate(needs_acquiring_load_exclusive(n));
8626   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8627   ins_cost(VOLATILE_REF_COST);
8628   effect(KILL cr);
8629   format %{
8630     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8631     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8632   %}
8633   ins_encode %{
8634     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8635                Assembler::halfword, /*acquire*/ true, /*release*/ true,
8636                /*weak*/ true, noreg);
8637     __ csetw($res$$Register, Assembler::EQ);
8638   %}
8639   ins_pipe(pipe_slow);
8640 %}
8641 
8642 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8643   predicate(needs_acquiring_load_exclusive(n));
8644   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8645   ins_cost(VOLATILE_REF_COST);
8646   effect(KILL cr);
8647   format %{
8648     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8649     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8650   %}
8651   ins_encode %{
8652     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8653                Assembler::word, /*acquire*/ true, /*release*/ true,
8654                /*weak*/ true, noreg);
8655     __ csetw($res$$Register, Assembler::EQ);
8656   %}
8657   ins_pipe(pipe_slow);
8658 %}
8659 
8660 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8661   predicate(needs_acquiring_load_exclusive(n));
8662   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8663   ins_cost(VOLATILE_REF_COST);
8664   effect(KILL cr);
8665   format %{
8666     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8667     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8668   %}
8669   ins_encode %{
8670     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8671                Assembler::xword, /*acquire*/ true, /*release*/ true,
8672                /*weak*/ true, noreg);
8673     __ csetw($res$$Register, Assembler::EQ);
8674   %}
8675   ins_pipe(pipe_slow);
8676 %}
8677 
8678 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8679   predicate(needs_acquiring_load_exclusive(n));
8680   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8681   ins_cost(VOLATILE_REF_COST);
8682   effect(KILL cr);
8683   format %{
8684     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8685     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8686   %}
8687   ins_encode %{
8688     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8689                Assembler::word, /*acquire*/ true, /*release*/ true,
8690                /*weak*/ true, noreg);
8691     __ csetw($res$$Register, Assembler::EQ);
8692   %}
8693   ins_pipe(pipe_slow);
8694 %}
8695 
8696 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8697   predicate(needs_acquiring_load_exclusive(n));
8698   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8699   ins_cost(VOLATILE_REF_COST);
8700   effect(KILL cr);
8701   format %{
8702     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8703     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8704   %}
8705   ins_encode %{
8706     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8707                Assembler::xword, /*acquire*/ true, /*release*/ true,
8708                /*weak*/ true, noreg);
8709     __ csetw($res$$Register, Assembler::EQ);
8710   %}
8711   ins_pipe(pipe_slow);
8712 %}
8713 
8714 // END This section of the file is automatically generated. Do not edit --------------
8715 // ---------------------------------------------------------------------
8716 
8717 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
8718   match(Set prev (GetAndSetI mem newv));
8719   ins_cost(2 * VOLATILE_REF_COST);
8720   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
8721   ins_encode %{
8722     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8723   %}
8724   ins_pipe(pipe_serial);
8725 %}
8726 
8727 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
8728   match(Set prev (GetAndSetL mem newv));
8729   ins_cost(2 * VOLATILE_REF_COST);
8730   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
8731   ins_encode %{
8732     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8733   %}
8734   ins_pipe(pipe_serial);
8735 %}
8736 
8737 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
8738   match(Set prev (GetAndSetN mem newv));
8739   ins_cost(2 * VOLATILE_REF_COST);
8740   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8741   ins_encode %{
8742     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8743   %}
8744   ins_pipe(pipe_serial);
8745 %}
8746 
8747 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
8748   match(Set prev (GetAndSetP mem newv));
8749   ins_cost(2 * VOLATILE_REF_COST);
8750   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
8751   ins_encode %{
8752     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8753   %}
8754   ins_pipe(pipe_serial);
8755 %}
8756 
8757 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
8758   predicate(needs_acquiring_load_exclusive(n));
8759   match(Set prev (GetAndSetI mem newv));
8760   ins_cost(VOLATILE_REF_COST);
8761   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
8762   ins_encode %{
8763     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8764   %}
8765   ins_pipe(pipe_serial);
8766 %}
8767 
8768 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
8769   predicate(needs_acquiring_load_exclusive(n));
8770   match(Set prev (GetAndSetL mem newv));
8771   ins_cost(VOLATILE_REF_COST);
8772   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
8773   ins_encode %{
8774     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8775   %}
8776   ins_pipe(pipe_serial);
8777 %}
8778 
8779 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
8780   predicate(needs_acquiring_load_exclusive(n));
8781   match(Set prev (GetAndSetN mem newv));
8782   ins_cost(VOLATILE_REF_COST);
8783   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8784   ins_encode %{
8785     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8786   %}
8787   ins_pipe(pipe_serial);
8788 %}
8789 
8790 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
8791   predicate(needs_acquiring_load_exclusive(n));
8792   match(Set prev (GetAndSetP mem newv));
8793   ins_cost(VOLATILE_REF_COST);
8794   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
8795   ins_encode %{
8796     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8797   %}
8798   ins_pipe(pipe_serial);
8799 %}
8800 
8801 
8802 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
8803   match(Set newval (GetAndAddL mem incr));
8804   ins_cost(2 * VOLATILE_REF_COST + 1);
8805   format %{ "get_and_addL $newval, [$mem], $incr" %}
8806   ins_encode %{
8807     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
8808   %}
8809   ins_pipe(pipe_serial);
8810 %}
8811 
8812 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
8813   predicate(n->as_LoadStore()->result_not_used());
8814   match(Set dummy (GetAndAddL mem incr));
8815   ins_cost(2 * VOLATILE_REF_COST);
8816   format %{ "get_and_addL [$mem], $incr" %}
8817   ins_encode %{
8818     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
8819   %}
8820   ins_pipe(pipe_serial);
8821 %}
8822 
8823 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
8824   match(Set newval (GetAndAddL mem incr));
8825   ins_cost(2 * VOLATILE_REF_COST + 1);
8826   format %{ "get_and_addL $newval, [$mem], $incr" %}
8827   ins_encode %{
8828     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
8829   %}
8830   ins_pipe(pipe_serial);
8831 %}
8832 
8833 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
8834   predicate(n->as_LoadStore()->result_not_used());
8835   match(Set dummy (GetAndAddL mem incr));
8836   ins_cost(2 * VOLATILE_REF_COST);
8837   format %{ "get_and_addL [$mem], $incr" %}
8838   ins_encode %{
8839     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
8840   %}
8841   ins_pipe(pipe_serial);
8842 %}
8843 
8844 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
8845   match(Set newval (GetAndAddI mem incr));
8846   ins_cost(2 * VOLATILE_REF_COST + 1);
8847   format %{ "get_and_addI $newval, [$mem], $incr" %}
8848   ins_encode %{
8849     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
8850   %}
8851   ins_pipe(pipe_serial);
8852 %}
8853 
8854 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
8855   predicate(n->as_LoadStore()->result_not_used());
8856   match(Set dummy (GetAndAddI mem incr));
8857   ins_cost(2 * VOLATILE_REF_COST);
8858   format %{ "get_and_addI [$mem], $incr" %}
8859   ins_encode %{
8860     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
8861   %}
8862   ins_pipe(pipe_serial);
8863 %}
8864 
8865 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
8866   match(Set newval (GetAndAddI mem incr));
8867   ins_cost(2 * VOLATILE_REF_COST + 1);
8868   format %{ "get_and_addI $newval, [$mem], $incr" %}
8869   ins_encode %{
8870     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
8871   %}
8872   ins_pipe(pipe_serial);
8873 %}
8874 
8875 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
8876   predicate(n->as_LoadStore()->result_not_used());
8877   match(Set dummy (GetAndAddI mem incr));
8878   ins_cost(2 * VOLATILE_REF_COST);
8879   format %{ "get_and_addI [$mem], $incr" %}
8880   ins_encode %{
8881     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
8882   %}
8883   ins_pipe(pipe_serial);
8884 %}
8885 
8886 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
8887   predicate(needs_acquiring_load_exclusive(n));
8888   match(Set newval (GetAndAddL mem incr));
8889   ins_cost(VOLATILE_REF_COST + 1);
8890   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
8891   ins_encode %{
8892     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
8893   %}
8894   ins_pipe(pipe_serial);
8895 %}
8896 
8897 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
8898   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
8899   match(Set dummy (GetAndAddL mem incr));
8900   ins_cost(VOLATILE_REF_COST);
8901   format %{ "get_and_addL_acq [$mem], $incr" %}
8902   ins_encode %{
8903     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
8904   %}
8905   ins_pipe(pipe_serial);
8906 %}
8907 
8908 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
8909   predicate(needs_acquiring_load_exclusive(n));
8910   match(Set newval (GetAndAddL mem incr));
8911   ins_cost(VOLATILE_REF_COST + 1);
8912   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
8913   ins_encode %{
8914     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
8915   %}
8916   ins_pipe(pipe_serial);
8917 %}
8918 
8919 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
8920   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
8921   match(Set dummy (GetAndAddL mem incr));
8922   ins_cost(VOLATILE_REF_COST);
8923   format %{ "get_and_addL_acq [$mem], $incr" %}
8924   ins_encode %{
8925     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
8926   %}
8927   ins_pipe(pipe_serial);
8928 %}
8929 
8930 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
8931   predicate(needs_acquiring_load_exclusive(n));
8932   match(Set newval (GetAndAddI mem incr));
8933   ins_cost(VOLATILE_REF_COST + 1);
8934   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
8935   ins_encode %{
8936     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
8937   %}
8938   ins_pipe(pipe_serial);
8939 %}
8940 
8941 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
8942   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
8943   match(Set dummy (GetAndAddI mem incr));
8944   ins_cost(VOLATILE_REF_COST);
8945   format %{ "get_and_addI_acq [$mem], $incr" %}
8946   ins_encode %{
8947     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
8948   %}
8949   ins_pipe(pipe_serial);
8950 %}
8951 
8952 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
8953   predicate(needs_acquiring_load_exclusive(n));
8954   match(Set newval (GetAndAddI mem incr));
8955   ins_cost(VOLATILE_REF_COST + 1);
8956   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
8957   ins_encode %{
8958     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
8959   %}
8960   ins_pipe(pipe_serial);
8961 %}
8962 
8963 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
8964   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
8965   match(Set dummy (GetAndAddI mem incr));
8966   ins_cost(VOLATILE_REF_COST);
8967   format %{ "get_and_addI_acq [$mem], $incr" %}
8968   ins_encode %{
8969     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
8970   %}
8971   ins_pipe(pipe_serial);
8972 %}
8973 
8974 // Manifest a CmpL result in an integer register.
8975 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8976 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8977 %{
8978   match(Set dst (CmpL3 src1 src2));
8979   effect(KILL flags);
8980 
8981   ins_cost(INSN_COST * 6);
8982   format %{
8983       "cmp $src1, $src2"
8984       "csetw $dst, ne"
8985       "cnegw $dst, lt"
8986   %}
8987   // format %{ "CmpL3 $dst, $src1, $src2" %}
8988   ins_encode %{
8989     __ cmp($src1$$Register, $src2$$Register);
8990     __ csetw($dst$$Register, Assembler::NE);
8991     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8992   %}
8993 
8994   ins_pipe(pipe_class_default);
8995 %}
8996 
8997 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8998 %{
8999   match(Set dst (CmpL3 src1 src2));
9000   effect(KILL flags);
9001 
9002   ins_cost(INSN_COST * 6);
9003   format %{
9004       "cmp $src1, $src2"
9005       "csetw $dst, ne"
9006       "cnegw $dst, lt"
9007   %}
9008   ins_encode %{
9009     int32_t con = (int32_t)$src2$$constant;
9010      if (con < 0) {
9011       __ adds(zr, $src1$$Register, -con);
9012     } else {
9013       __ subs(zr, $src1$$Register, con);
9014     }
9015     __ csetw($dst$$Register, Assembler::NE);
9016     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9017   %}
9018 
9019   ins_pipe(pipe_class_default);
9020 %}
9021 
9022 // ============================================================================
9023 // Conditional Move Instructions
9024 
9025 // n.b. we have identical rules for both a signed compare op (cmpOp)
9026 // and an unsigned compare op (cmpOpU). it would be nice if we could
9027 // define an op class which merged both inputs and use it to type the
9028 // argument to a single rule. unfortunatelyt his fails because the
9029 // opclass does not live up to the COND_INTER interface of its
9030 // component operands. When the generic code tries to negate the
9031 // operand it ends up running the generci Machoper::negate method
9032 // which throws a ShouldNotHappen. So, we have to provide two flavours
9033 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9034 
9035 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9036   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9037 
9038   ins_cost(INSN_COST * 2);
9039   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
9040 
9041   ins_encode %{
9042     __ cselw(as_Register($dst$$reg),
9043              as_Register($src2$$reg),
9044              as_Register($src1$$reg),
9045              (Assembler::Condition)$cmp$$cmpcode);
9046   %}
9047 
9048   ins_pipe(icond_reg_reg);
9049 %}
9050 
9051 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9052   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9053 
9054   ins_cost(INSN_COST * 2);
9055   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
9056 
9057   ins_encode %{
9058     __ cselw(as_Register($dst$$reg),
9059              as_Register($src2$$reg),
9060              as_Register($src1$$reg),
9061              (Assembler::Condition)$cmp$$cmpcode);
9062   %}
9063 
9064   ins_pipe(icond_reg_reg);
9065 %}
9066 
9067 // special cases where one arg is zero
9068 
9069 // n.b. this is selected in preference to the rule above because it
9070 // avoids loading constant 0 into a source register
9071 
9072 // TODO
9073 // we ought only to be able to cull one of these variants as the ideal
9074 // transforms ought always to order the zero consistently (to left/right?)
9075 
9076 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9077   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9078 
9079   ins_cost(INSN_COST * 2);
9080   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
9081 
9082   ins_encode %{
9083     __ cselw(as_Register($dst$$reg),
9084              as_Register($src$$reg),
9085              zr,
9086              (Assembler::Condition)$cmp$$cmpcode);
9087   %}
9088 
9089   ins_pipe(icond_reg);
9090 %}
9091 
9092 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9093   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9094 
9095   ins_cost(INSN_COST * 2);
9096   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
9097 
9098   ins_encode %{
9099     __ cselw(as_Register($dst$$reg),
9100              as_Register($src$$reg),
9101              zr,
9102              (Assembler::Condition)$cmp$$cmpcode);
9103   %}
9104 
9105   ins_pipe(icond_reg);
9106 %}
9107 
9108 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9109   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9110 
9111   ins_cost(INSN_COST * 2);
9112   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
9113 
9114   ins_encode %{
9115     __ cselw(as_Register($dst$$reg),
9116              zr,
9117              as_Register($src$$reg),
9118              (Assembler::Condition)$cmp$$cmpcode);
9119   %}
9120 
9121   ins_pipe(icond_reg);
9122 %}
9123 
9124 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9125   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9126 
9127   ins_cost(INSN_COST * 2);
9128   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
9129 
9130   ins_encode %{
9131     __ cselw(as_Register($dst$$reg),
9132              zr,
9133              as_Register($src$$reg),
9134              (Assembler::Condition)$cmp$$cmpcode);
9135   %}
9136 
9137   ins_pipe(icond_reg);
9138 %}
9139 
9140 // special case for creating a boolean 0 or 1
9141 
9142 // n.b. this is selected in preference to the rule above because it
9143 // avoids loading constants 0 and 1 into a source register
9144 
9145 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9146   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9147 
9148   ins_cost(INSN_COST * 2);
9149   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
9150 
9151   ins_encode %{
9152     // equivalently
9153     // cset(as_Register($dst$$reg),
9154     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
9155     __ csincw(as_Register($dst$$reg),
9156              zr,
9157              zr,
9158              (Assembler::Condition)$cmp$$cmpcode);
9159   %}
9160 
9161   ins_pipe(icond_none);
9162 %}
9163 
9164 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9165   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9166 
9167   ins_cost(INSN_COST * 2);
9168   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
9169 
9170   ins_encode %{
9171     // equivalently
9172     // cset(as_Register($dst$$reg),
9173     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
9174     __ csincw(as_Register($dst$$reg),
9175              zr,
9176              zr,
9177              (Assembler::Condition)$cmp$$cmpcode);
9178   %}
9179 
9180   ins_pipe(icond_none);
9181 %}
9182 
9183 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9184   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9185 
9186   ins_cost(INSN_COST * 2);
9187   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
9188 
9189   ins_encode %{
9190     __ csel(as_Register($dst$$reg),
9191             as_Register($src2$$reg),
9192             as_Register($src1$$reg),
9193             (Assembler::Condition)$cmp$$cmpcode);
9194   %}
9195 
9196   ins_pipe(icond_reg_reg);
9197 %}
9198 
9199 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9200   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9201 
9202   ins_cost(INSN_COST * 2);
9203   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
9204 
9205   ins_encode %{
9206     __ csel(as_Register($dst$$reg),
9207             as_Register($src2$$reg),
9208             as_Register($src1$$reg),
9209             (Assembler::Condition)$cmp$$cmpcode);
9210   %}
9211 
9212   ins_pipe(icond_reg_reg);
9213 %}
9214 
9215 // special cases where one arg is zero
9216 
9217 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9218   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9219 
9220   ins_cost(INSN_COST * 2);
9221   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
9222 
9223   ins_encode %{
9224     __ csel(as_Register($dst$$reg),
9225             zr,
9226             as_Register($src$$reg),
9227             (Assembler::Condition)$cmp$$cmpcode);
9228   %}
9229 
9230   ins_pipe(icond_reg);
9231 %}
9232 
9233 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9234   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9235 
9236   ins_cost(INSN_COST * 2);
9237   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
9238 
9239   ins_encode %{
9240     __ csel(as_Register($dst$$reg),
9241             zr,
9242             as_Register($src$$reg),
9243             (Assembler::Condition)$cmp$$cmpcode);
9244   %}
9245 
9246   ins_pipe(icond_reg);
9247 %}
9248 
9249 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9250   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9251 
9252   ins_cost(INSN_COST * 2);
9253   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
9254 
9255   ins_encode %{
9256     __ csel(as_Register($dst$$reg),
9257             as_Register($src$$reg),
9258             zr,
9259             (Assembler::Condition)$cmp$$cmpcode);
9260   %}
9261 
9262   ins_pipe(icond_reg);
9263 %}
9264 
9265 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9266   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9267 
9268   ins_cost(INSN_COST * 2);
9269   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
9270 
9271   ins_encode %{
9272     __ csel(as_Register($dst$$reg),
9273             as_Register($src$$reg),
9274             zr,
9275             (Assembler::Condition)$cmp$$cmpcode);
9276   %}
9277 
9278   ins_pipe(icond_reg);
9279 %}
9280 
9281 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9282   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9283 
9284   ins_cost(INSN_COST * 2);
9285   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
9286 
9287   ins_encode %{
9288     __ csel(as_Register($dst$$reg),
9289             as_Register($src2$$reg),
9290             as_Register($src1$$reg),
9291             (Assembler::Condition)$cmp$$cmpcode);
9292   %}
9293 
9294   ins_pipe(icond_reg_reg);
9295 %}
9296 
9297 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9298   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9299 
9300   ins_cost(INSN_COST * 2);
9301   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
9302 
9303   ins_encode %{
9304     __ csel(as_Register($dst$$reg),
9305             as_Register($src2$$reg),
9306             as_Register($src1$$reg),
9307             (Assembler::Condition)$cmp$$cmpcode);
9308   %}
9309 
9310   ins_pipe(icond_reg_reg);
9311 %}
9312 
9313 // special cases where one arg is zero
9314 
9315 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9316   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9317 
9318   ins_cost(INSN_COST * 2);
9319   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
9320 
9321   ins_encode %{
9322     __ csel(as_Register($dst$$reg),
9323             zr,
9324             as_Register($src$$reg),
9325             (Assembler::Condition)$cmp$$cmpcode);
9326   %}
9327 
9328   ins_pipe(icond_reg);
9329 %}
9330 
9331 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9332   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9333 
9334   ins_cost(INSN_COST * 2);
9335   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
9336 
9337   ins_encode %{
9338     __ csel(as_Register($dst$$reg),
9339             zr,
9340             as_Register($src$$reg),
9341             (Assembler::Condition)$cmp$$cmpcode);
9342   %}
9343 
9344   ins_pipe(icond_reg);
9345 %}
9346 
9347 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9348   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9349 
9350   ins_cost(INSN_COST * 2);
9351   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
9352 
9353   ins_encode %{
9354     __ csel(as_Register($dst$$reg),
9355             as_Register($src$$reg),
9356             zr,
9357             (Assembler::Condition)$cmp$$cmpcode);
9358   %}
9359 
9360   ins_pipe(icond_reg);
9361 %}
9362 
9363 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9364   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9365 
9366   ins_cost(INSN_COST * 2);
9367   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
9368 
9369   ins_encode %{
9370     __ csel(as_Register($dst$$reg),
9371             as_Register($src$$reg),
9372             zr,
9373             (Assembler::Condition)$cmp$$cmpcode);
9374   %}
9375 
9376   ins_pipe(icond_reg);
9377 %}
9378 
9379 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9380   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9381 
9382   ins_cost(INSN_COST * 2);
9383   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
9384 
9385   ins_encode %{
9386     __ cselw(as_Register($dst$$reg),
9387              as_Register($src2$$reg),
9388              as_Register($src1$$reg),
9389              (Assembler::Condition)$cmp$$cmpcode);
9390   %}
9391 
9392   ins_pipe(icond_reg_reg);
9393 %}
9394 
9395 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9396   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9397 
9398   ins_cost(INSN_COST * 2);
9399   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
9400 
9401   ins_encode %{
9402     __ cselw(as_Register($dst$$reg),
9403              as_Register($src2$$reg),
9404              as_Register($src1$$reg),
9405              (Assembler::Condition)$cmp$$cmpcode);
9406   %}
9407 
9408   ins_pipe(icond_reg_reg);
9409 %}
9410 
9411 // special cases where one arg is zero
9412 
9413 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9414   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9415 
9416   ins_cost(INSN_COST * 2);
9417   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
9418 
9419   ins_encode %{
9420     __ cselw(as_Register($dst$$reg),
9421              zr,
9422              as_Register($src$$reg),
9423              (Assembler::Condition)$cmp$$cmpcode);
9424   %}
9425 
9426   ins_pipe(icond_reg);
9427 %}
9428 
9429 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9430   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9431 
9432   ins_cost(INSN_COST * 2);
9433   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
9434 
9435   ins_encode %{
9436     __ cselw(as_Register($dst$$reg),
9437              zr,
9438              as_Register($src$$reg),
9439              (Assembler::Condition)$cmp$$cmpcode);
9440   %}
9441 
9442   ins_pipe(icond_reg);
9443 %}
9444 
9445 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9446   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9447 
9448   ins_cost(INSN_COST * 2);
9449   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
9450 
9451   ins_encode %{
9452     __ cselw(as_Register($dst$$reg),
9453              as_Register($src$$reg),
9454              zr,
9455              (Assembler::Condition)$cmp$$cmpcode);
9456   %}
9457 
9458   ins_pipe(icond_reg);
9459 %}
9460 
9461 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9462   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9463 
9464   ins_cost(INSN_COST * 2);
9465   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
9466 
9467   ins_encode %{
9468     __ cselw(as_Register($dst$$reg),
9469              as_Register($src$$reg),
9470              zr,
9471              (Assembler::Condition)$cmp$$cmpcode);
9472   %}
9473 
9474   ins_pipe(icond_reg);
9475 %}
9476 
9477 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
9478 %{
9479   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9480 
9481   ins_cost(INSN_COST * 3);
9482 
9483   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9484   ins_encode %{
9485     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9486     __ fcsels(as_FloatRegister($dst$$reg),
9487               as_FloatRegister($src2$$reg),
9488               as_FloatRegister($src1$$reg),
9489               cond);
9490   %}
9491 
9492   ins_pipe(fp_cond_reg_reg_s);
9493 %}
9494 
9495 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
9496 %{
9497   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9498 
9499   ins_cost(INSN_COST * 3);
9500 
9501   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9502   ins_encode %{
9503     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9504     __ fcsels(as_FloatRegister($dst$$reg),
9505               as_FloatRegister($src2$$reg),
9506               as_FloatRegister($src1$$reg),
9507               cond);
9508   %}
9509 
9510   ins_pipe(fp_cond_reg_reg_s);
9511 %}
9512 
9513 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
9514 %{
9515   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9516 
9517   ins_cost(INSN_COST * 3);
9518 
9519   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9520   ins_encode %{
9521     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9522     __ fcseld(as_FloatRegister($dst$$reg),
9523               as_FloatRegister($src2$$reg),
9524               as_FloatRegister($src1$$reg),
9525               cond);
9526   %}
9527 
9528   ins_pipe(fp_cond_reg_reg_d);
9529 %}
9530 
9531 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
9532 %{
9533   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9534 
9535   ins_cost(INSN_COST * 3);
9536 
9537   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9538   ins_encode %{
9539     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9540     __ fcseld(as_FloatRegister($dst$$reg),
9541               as_FloatRegister($src2$$reg),
9542               as_FloatRegister($src1$$reg),
9543               cond);
9544   %}
9545 
9546   ins_pipe(fp_cond_reg_reg_d);
9547 %}
9548 
9549 // ============================================================================
9550 // Arithmetic Instructions
9551 //
9552 
9553 // Integer Addition
9554 
9555 // TODO
9556 // these currently employ operations which do not set CR and hence are
9557 // not flagged as killing CR but we would like to isolate the cases
9558 // where we want to set flags from those where we don't. need to work
9559 // out how to do that.
9560 
9561 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9562   match(Set dst (AddI src1 src2));
9563 
9564   ins_cost(INSN_COST);
9565   format %{ "addw  $dst, $src1, $src2" %}
9566 
9567   ins_encode %{
9568     __ addw(as_Register($dst$$reg),
9569             as_Register($src1$$reg),
9570             as_Register($src2$$reg));
9571   %}
9572 
9573   ins_pipe(ialu_reg_reg);
9574 %}
9575 
9576 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9577   match(Set dst (AddI src1 src2));
9578 
9579   ins_cost(INSN_COST);
9580   format %{ "addw $dst, $src1, $src2" %}
9581 
9582   // use opcode to indicate that this is an add not a sub
9583   opcode(0x0);
9584 
9585   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9586 
9587   ins_pipe(ialu_reg_imm);
9588 %}
9589 
9590 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9591   match(Set dst (AddI (ConvL2I src1) src2));
9592 
9593   ins_cost(INSN_COST);
9594   format %{ "addw $dst, $src1, $src2" %}
9595 
9596   // use opcode to indicate that this is an add not a sub
9597   opcode(0x0);
9598 
9599   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9600 
9601   ins_pipe(ialu_reg_imm);
9602 %}
9603 
9604 // Pointer Addition
9605 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
9606   match(Set dst (AddP src1 src2));
9607 
9608   ins_cost(INSN_COST);
9609   format %{ "add $dst, $src1, $src2\t# ptr" %}
9610 
9611   ins_encode %{
9612     __ add(as_Register($dst$$reg),
9613            as_Register($src1$$reg),
9614            as_Register($src2$$reg));
9615   %}
9616 
9617   ins_pipe(ialu_reg_reg);
9618 %}
9619 
9620 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
9621   match(Set dst (AddP src1 (ConvI2L src2)));
9622 
9623   ins_cost(1.9 * INSN_COST);
9624   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9625 
9626   ins_encode %{
9627     __ add(as_Register($dst$$reg),
9628            as_Register($src1$$reg),
9629            as_Register($src2$$reg), ext::sxtw);
9630   %}
9631 
9632   ins_pipe(ialu_reg_reg);
9633 %}
9634 
9635 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
9636   match(Set dst (AddP src1 (LShiftL src2 scale)));
9637 
9638   ins_cost(1.9 * INSN_COST);
9639   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9640 
9641   ins_encode %{
9642     __ lea(as_Register($dst$$reg),
9643            Address(as_Register($src1$$reg), as_Register($src2$$reg),
9644                    Address::lsl($scale$$constant)));
9645   %}
9646 
9647   ins_pipe(ialu_reg_reg_shift);
9648 %}
9649 
9650 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
9651   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9652 
9653   ins_cost(1.9 * INSN_COST);
9654   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9655 
9656   ins_encode %{
9657     __ lea(as_Register($dst$$reg),
9658            Address(as_Register($src1$$reg), as_Register($src2$$reg),
9659                    Address::sxtw($scale$$constant)));
9660   %}
9661 
9662   ins_pipe(ialu_reg_reg_shift);
9663 %}
9664 
9665 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9666   match(Set dst (LShiftL (ConvI2L src) scale));
9667 
9668   ins_cost(INSN_COST);
9669   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9670 
9671   ins_encode %{
9672     __ sbfiz(as_Register($dst$$reg),
9673           as_Register($src$$reg),
9674           $scale$$constant & 63, MIN(32, (-$scale$$constant) & 63));
9675   %}
9676 
9677   ins_pipe(ialu_reg_shift);
9678 %}
9679 
9680 // Pointer Immediate Addition
9681 // n.b. this needs to be more expensive than using an indirect memory
9682 // operand
9683 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
9684   match(Set dst (AddP src1 src2));
9685 
9686   ins_cost(INSN_COST);
9687   format %{ "add $dst, $src1, $src2\t# ptr" %}
9688 
9689   // use opcode to indicate that this is an add not a sub
9690   opcode(0x0);
9691 
9692   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9693 
9694   ins_pipe(ialu_reg_imm);
9695 %}
9696 
9697 // Long Addition
9698 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9699 
9700   match(Set dst (AddL src1 src2));
9701 
9702   ins_cost(INSN_COST);
9703   format %{ "add  $dst, $src1, $src2" %}
9704 
9705   ins_encode %{
9706     __ add(as_Register($dst$$reg),
9707            as_Register($src1$$reg),
9708            as_Register($src2$$reg));
9709   %}
9710 
9711   ins_pipe(ialu_reg_reg);
9712 %}
9713 
9714 // No constant pool entries requiredLong Immediate Addition.
9715 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9716   match(Set dst (AddL src1 src2));
9717 
9718   ins_cost(INSN_COST);
9719   format %{ "add $dst, $src1, $src2" %}
9720 
9721   // use opcode to indicate that this is an add not a sub
9722   opcode(0x0);
9723 
9724   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9725 
9726   ins_pipe(ialu_reg_imm);
9727 %}
9728 
9729 // Integer Subtraction
9730 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9731   match(Set dst (SubI src1 src2));
9732 
9733   ins_cost(INSN_COST);
9734   format %{ "subw  $dst, $src1, $src2" %}
9735 
9736   ins_encode %{
9737     __ subw(as_Register($dst$$reg),
9738             as_Register($src1$$reg),
9739             as_Register($src2$$reg));
9740   %}
9741 
9742   ins_pipe(ialu_reg_reg);
9743 %}
9744 
9745 // Immediate Subtraction
9746 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9747   match(Set dst (SubI src1 src2));
9748 
9749   ins_cost(INSN_COST);
9750   format %{ "subw $dst, $src1, $src2" %}
9751 
9752   // use opcode to indicate that this is a sub not an add
9753   opcode(0x1);
9754 
9755   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9756 
9757   ins_pipe(ialu_reg_imm);
9758 %}
9759 
9760 // Long Subtraction
9761 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9762 
9763   match(Set dst (SubL src1 src2));
9764 
9765   ins_cost(INSN_COST);
9766   format %{ "sub  $dst, $src1, $src2" %}
9767 
9768   ins_encode %{
9769     __ sub(as_Register($dst$$reg),
9770            as_Register($src1$$reg),
9771            as_Register($src2$$reg));
9772   %}
9773 
9774   ins_pipe(ialu_reg_reg);
9775 %}
9776 
9777 // No constant pool entries requiredLong Immediate Subtraction.
9778 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9779   match(Set dst (SubL src1 src2));
9780 
9781   ins_cost(INSN_COST);
9782   format %{ "sub$dst, $src1, $src2" %}
9783 
9784   // use opcode to indicate that this is a sub not an add
9785   opcode(0x1);
9786 
9787   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9788 
9789   ins_pipe(ialu_reg_imm);
9790 %}
9791 
9792 // Integer Negation (special case for sub)
9793 
9794 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9795   match(Set dst (SubI zero src));
9796 
9797   ins_cost(INSN_COST);
9798   format %{ "negw $dst, $src\t# int" %}
9799 
9800   ins_encode %{
9801     __ negw(as_Register($dst$$reg),
9802             as_Register($src$$reg));
9803   %}
9804 
9805   ins_pipe(ialu_reg);
9806 %}
9807 
9808 // Long Negation
9809 
9810 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9811   match(Set dst (SubL zero src));
9812 
9813   ins_cost(INSN_COST);
9814   format %{ "neg $dst, $src\t# long" %}
9815 
9816   ins_encode %{
9817     __ neg(as_Register($dst$$reg),
9818            as_Register($src$$reg));
9819   %}
9820 
9821   ins_pipe(ialu_reg);
9822 %}
9823 
9824 // Integer Multiply
9825 
9826 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9827   match(Set dst (MulI src1 src2));
9828 
9829   ins_cost(INSN_COST * 3);
9830   format %{ "mulw  $dst, $src1, $src2" %}
9831 
9832   ins_encode %{
9833     __ mulw(as_Register($dst$$reg),
9834             as_Register($src1$$reg),
9835             as_Register($src2$$reg));
9836   %}
9837 
9838   ins_pipe(imul_reg_reg);
9839 %}
9840 
9841 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9842   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9843 
9844   ins_cost(INSN_COST * 3);
9845   format %{ "smull  $dst, $src1, $src2" %}
9846 
9847   ins_encode %{
9848     __ smull(as_Register($dst$$reg),
9849              as_Register($src1$$reg),
9850              as_Register($src2$$reg));
9851   %}
9852 
9853   ins_pipe(imul_reg_reg);
9854 %}
9855 
9856 // Long Multiply
9857 
9858 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9859   match(Set dst (MulL src1 src2));
9860 
9861   ins_cost(INSN_COST * 5);
9862   format %{ "mul  $dst, $src1, $src2" %}
9863 
9864   ins_encode %{
9865     __ mul(as_Register($dst$$reg),
9866            as_Register($src1$$reg),
9867            as_Register($src2$$reg));
9868   %}
9869 
9870   ins_pipe(lmul_reg_reg);
9871 %}
9872 
9873 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9874 %{
9875   match(Set dst (MulHiL src1 src2));
9876 
9877   ins_cost(INSN_COST * 7);
9878   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
9879 
9880   ins_encode %{
9881     __ smulh(as_Register($dst$$reg),
9882              as_Register($src1$$reg),
9883              as_Register($src2$$reg));
9884   %}
9885 
9886   ins_pipe(lmul_reg_reg);
9887 %}
9888 
9889 // Combined Integer Multiply & Add/Sub
9890 
9891 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9892   match(Set dst (AddI src3 (MulI src1 src2)));
9893 
9894   ins_cost(INSN_COST * 3);
9895   format %{ "madd  $dst, $src1, $src2, $src3" %}
9896 
9897   ins_encode %{
9898     __ maddw(as_Register($dst$$reg),
9899              as_Register($src1$$reg),
9900              as_Register($src2$$reg),
9901              as_Register($src3$$reg));
9902   %}
9903 
9904   ins_pipe(imac_reg_reg);
9905 %}
9906 
9907 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9908   match(Set dst (SubI src3 (MulI src1 src2)));
9909 
9910   ins_cost(INSN_COST * 3);
9911   format %{ "msub  $dst, $src1, $src2, $src3" %}
9912 
9913   ins_encode %{
9914     __ msubw(as_Register($dst$$reg),
9915              as_Register($src1$$reg),
9916              as_Register($src2$$reg),
9917              as_Register($src3$$reg));
9918   %}
9919 
9920   ins_pipe(imac_reg_reg);
9921 %}
9922 
9923 // Combined Integer Multiply & Neg
9924 
9925 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9926   match(Set dst (MulI (SubI zero src1) src2));
9927   match(Set dst (MulI src1 (SubI zero src2)));
9928 
9929   ins_cost(INSN_COST * 3);
9930   format %{ "mneg  $dst, $src1, $src2" %}
9931 
9932   ins_encode %{
9933     __ mnegw(as_Register($dst$$reg),
9934              as_Register($src1$$reg),
9935              as_Register($src2$$reg));
9936   %}
9937 
9938   ins_pipe(imac_reg_reg);
9939 %}
9940 
9941 // Combined Long Multiply & Add/Sub
9942 
9943 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9944   match(Set dst (AddL src3 (MulL src1 src2)));
9945 
9946   ins_cost(INSN_COST * 5);
9947   format %{ "madd  $dst, $src1, $src2, $src3" %}
9948 
9949   ins_encode %{
9950     __ madd(as_Register($dst$$reg),
9951             as_Register($src1$$reg),
9952             as_Register($src2$$reg),
9953             as_Register($src3$$reg));
9954   %}
9955 
9956   ins_pipe(lmac_reg_reg);
9957 %}
9958 
9959 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9960   match(Set dst (SubL src3 (MulL src1 src2)));
9961 
9962   ins_cost(INSN_COST * 5);
9963   format %{ "msub  $dst, $src1, $src2, $src3" %}
9964 
9965   ins_encode %{
9966     __ msub(as_Register($dst$$reg),
9967             as_Register($src1$$reg),
9968             as_Register($src2$$reg),
9969             as_Register($src3$$reg));
9970   %}
9971 
9972   ins_pipe(lmac_reg_reg);
9973 %}
9974 
9975 // Combined Long Multiply & Neg
9976 
9977 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9978   match(Set dst (MulL (SubL zero src1) src2));
9979   match(Set dst (MulL src1 (SubL zero src2)));
9980 
9981   ins_cost(INSN_COST * 5);
9982   format %{ "mneg  $dst, $src1, $src2" %}
9983 
9984   ins_encode %{
9985     __ mneg(as_Register($dst$$reg),
9986             as_Register($src1$$reg),
9987             as_Register($src2$$reg));
9988   %}
9989 
9990   ins_pipe(lmac_reg_reg);
9991 %}
9992 
9993 // Integer Divide
9994 
9995 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9996   match(Set dst (DivI src1 src2));
9997 
9998   ins_cost(INSN_COST * 19);
9999   format %{ "sdivw  $dst, $src1, $src2" %}
10000 
10001   ins_encode(aarch64_enc_divw(dst, src1, src2));
10002   ins_pipe(idiv_reg_reg);
10003 %}
10004 
10005 instruct signExtract(iRegINoSp dst, iRegIorL2I src1, immI_31 div1, immI_31 div2) %{
10006   match(Set dst (URShiftI (RShiftI src1 div1) div2));
10007   ins_cost(INSN_COST);
10008   format %{ "lsrw $dst, $src1, $div1" %}
10009   ins_encode %{
10010     __ lsrw(as_Register($dst$$reg), as_Register($src1$$reg), 31);
10011   %}
10012   ins_pipe(ialu_reg_shift);
10013 %}
10014 
10015 instruct div2Round(iRegINoSp dst, iRegIorL2I src, immI_31 div1, immI_31 div2) %{
10016   match(Set dst (AddI src (URShiftI (RShiftI src div1) div2)));
10017   ins_cost(INSN_COST);
10018   format %{ "addw $dst, $src, LSR $div1" %}
10019 
10020   ins_encode %{
10021     __ addw(as_Register($dst$$reg),
10022               as_Register($src$$reg),
10023               as_Register($src$$reg),
10024               Assembler::LSR, 31);
10025   %}
10026   ins_pipe(ialu_reg);
10027 %}
10028 
10029 // Long Divide
10030 
10031 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10032   match(Set dst (DivL src1 src2));
10033 
10034   ins_cost(INSN_COST * 35);
10035   format %{ "sdiv   $dst, $src1, $src2" %}
10036 
10037   ins_encode(aarch64_enc_div(dst, src1, src2));
10038   ins_pipe(ldiv_reg_reg);
10039 %}
10040 
10041 instruct signExtractL(iRegLNoSp dst, iRegL src1, immI_63 div1, immI_63 div2) %{
10042   match(Set dst (URShiftL (RShiftL src1 div1) div2));
10043   ins_cost(INSN_COST);
10044   format %{ "lsr $dst, $src1, $div1" %}
10045   ins_encode %{
10046     __ lsr(as_Register($dst$$reg), as_Register($src1$$reg), 63);
10047   %}
10048   ins_pipe(ialu_reg_shift);
10049 %}
10050 
10051 instruct div2RoundL(iRegLNoSp dst, iRegL src, immI_63 div1, immI_63 div2) %{
10052   match(Set dst (AddL src (URShiftL (RShiftL src div1) div2)));
10053   ins_cost(INSN_COST);
10054   format %{ "add $dst, $src, $div1" %}
10055 
10056   ins_encode %{
10057     __ add(as_Register($dst$$reg),
10058               as_Register($src$$reg),
10059               as_Register($src$$reg),
10060               Assembler::LSR, 63);
10061   %}
10062   ins_pipe(ialu_reg);
10063 %}
10064 
10065 // Integer Remainder
10066 
10067 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10068   match(Set dst (ModI src1 src2));
10069 
10070   ins_cost(INSN_COST * 22);
10071   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
10072             "msubw($dst, rscratch1, $src2, $src1" %}
10073 
10074   ins_encode(aarch64_enc_modw(dst, src1, src2));
10075   ins_pipe(idiv_reg_reg);
10076 %}
10077 
10078 // Long Remainder
10079 
10080 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10081   match(Set dst (ModL src1 src2));
10082 
10083   ins_cost(INSN_COST * 38);
10084   format %{ "sdiv   rscratch1, $src1, $src2\n"
10085             "msub($dst, rscratch1, $src2, $src1" %}
10086 
10087   ins_encode(aarch64_enc_mod(dst, src1, src2));
10088   ins_pipe(ldiv_reg_reg);
10089 %}
10090 
10091 // Integer Shifts
10092 
10093 // Shift Left Register
10094 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10095   match(Set dst (LShiftI src1 src2));
10096 
10097   ins_cost(INSN_COST * 2);
10098   format %{ "lslvw  $dst, $src1, $src2" %}
10099 
10100   ins_encode %{
10101     __ lslvw(as_Register($dst$$reg),
10102              as_Register($src1$$reg),
10103              as_Register($src2$$reg));
10104   %}
10105 
10106   ins_pipe(ialu_reg_reg_vshift);
10107 %}
10108 
10109 // Shift Left Immediate
10110 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10111   match(Set dst (LShiftI src1 src2));
10112 
10113   ins_cost(INSN_COST);
10114   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10115 
10116   ins_encode %{
10117     __ lslw(as_Register($dst$$reg),
10118             as_Register($src1$$reg),
10119             $src2$$constant & 0x1f);
10120   %}
10121 
10122   ins_pipe(ialu_reg_shift);
10123 %}
10124 
10125 // Shift Right Logical Register
10126 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10127   match(Set dst (URShiftI src1 src2));
10128 
10129   ins_cost(INSN_COST * 2);
10130   format %{ "lsrvw  $dst, $src1, $src2" %}
10131 
10132   ins_encode %{
10133     __ lsrvw(as_Register($dst$$reg),
10134              as_Register($src1$$reg),
10135              as_Register($src2$$reg));
10136   %}
10137 
10138   ins_pipe(ialu_reg_reg_vshift);
10139 %}
10140 
10141 // Shift Right Logical Immediate
10142 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10143   match(Set dst (URShiftI src1 src2));
10144 
10145   ins_cost(INSN_COST);
10146   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10147 
10148   ins_encode %{
10149     __ lsrw(as_Register($dst$$reg),
10150             as_Register($src1$$reg),
10151             $src2$$constant & 0x1f);
10152   %}
10153 
10154   ins_pipe(ialu_reg_shift);
10155 %}
10156 
10157 // Shift Right Arithmetic Register
10158 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10159   match(Set dst (RShiftI src1 src2));
10160 
10161   ins_cost(INSN_COST * 2);
10162   format %{ "asrvw  $dst, $src1, $src2" %}
10163 
10164   ins_encode %{
10165     __ asrvw(as_Register($dst$$reg),
10166              as_Register($src1$$reg),
10167              as_Register($src2$$reg));
10168   %}
10169 
10170   ins_pipe(ialu_reg_reg_vshift);
10171 %}
10172 
10173 // Shift Right Arithmetic Immediate
10174 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10175   match(Set dst (RShiftI src1 src2));
10176 
10177   ins_cost(INSN_COST);
10178   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10179 
10180   ins_encode %{
10181     __ asrw(as_Register($dst$$reg),
10182             as_Register($src1$$reg),
10183             $src2$$constant & 0x1f);
10184   %}
10185 
10186   ins_pipe(ialu_reg_shift);
10187 %}
10188 
10189 // Combined Int Mask and Right Shift (using UBFM)
10190 // TODO
10191 
10192 // Long Shifts
10193 
10194 // Shift Left Register
10195 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10196   match(Set dst (LShiftL src1 src2));
10197 
10198   ins_cost(INSN_COST * 2);
10199   format %{ "lslv  $dst, $src1, $src2" %}
10200 
10201   ins_encode %{
10202     __ lslv(as_Register($dst$$reg),
10203             as_Register($src1$$reg),
10204             as_Register($src2$$reg));
10205   %}
10206 
10207   ins_pipe(ialu_reg_reg_vshift);
10208 %}
10209 
10210 // Shift Left Immediate
10211 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10212   match(Set dst (LShiftL src1 src2));
10213 
10214   ins_cost(INSN_COST);
10215   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10216 
10217   ins_encode %{
10218     __ lsl(as_Register($dst$$reg),
10219             as_Register($src1$$reg),
10220             $src2$$constant & 0x3f);
10221   %}
10222 
10223   ins_pipe(ialu_reg_shift);
10224 %}
10225 
10226 // Shift Right Logical Register
10227 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10228   match(Set dst (URShiftL src1 src2));
10229 
10230   ins_cost(INSN_COST * 2);
10231   format %{ "lsrv  $dst, $src1, $src2" %}
10232 
10233   ins_encode %{
10234     __ lsrv(as_Register($dst$$reg),
10235             as_Register($src1$$reg),
10236             as_Register($src2$$reg));
10237   %}
10238 
10239   ins_pipe(ialu_reg_reg_vshift);
10240 %}
10241 
10242 // Shift Right Logical Immediate
10243 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10244   match(Set dst (URShiftL src1 src2));
10245 
10246   ins_cost(INSN_COST);
10247   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10248 
10249   ins_encode %{
10250     __ lsr(as_Register($dst$$reg),
10251            as_Register($src1$$reg),
10252            $src2$$constant & 0x3f);
10253   %}
10254 
10255   ins_pipe(ialu_reg_shift);
10256 %}
10257 
10258 // A special-case pattern for card table stores.
10259 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10260   match(Set dst (URShiftL (CastP2X src1) src2));
10261 
10262   ins_cost(INSN_COST);
10263   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10264 
10265   ins_encode %{
10266     __ lsr(as_Register($dst$$reg),
10267            as_Register($src1$$reg),
10268            $src2$$constant & 0x3f);
10269   %}
10270 
10271   ins_pipe(ialu_reg_shift);
10272 %}
10273 
10274 // Shift Right Arithmetic Register
10275 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10276   match(Set dst (RShiftL src1 src2));
10277 
10278   ins_cost(INSN_COST * 2);
10279   format %{ "asrv  $dst, $src1, $src2" %}
10280 
10281   ins_encode %{
10282     __ asrv(as_Register($dst$$reg),
10283             as_Register($src1$$reg),
10284             as_Register($src2$$reg));
10285   %}
10286 
10287   ins_pipe(ialu_reg_reg_vshift);
10288 %}
10289 
10290 // Shift Right Arithmetic Immediate
10291 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10292   match(Set dst (RShiftL src1 src2));
10293 
10294   ins_cost(INSN_COST);
10295   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10296 
10297   ins_encode %{
10298     __ asr(as_Register($dst$$reg),
10299            as_Register($src1$$reg),
10300            $src2$$constant & 0x3f);
10301   %}
10302 
10303   ins_pipe(ialu_reg_shift);
10304 %}
10305 
10306 // BEGIN This section of the file is automatically generated. Do not edit --------------
10307 
10308 instruct regL_not_reg(iRegLNoSp dst,
10309                          iRegL src1, immL_M1 m1,
10310                          rFlagsReg cr) %{
10311   match(Set dst (XorL src1 m1));
10312   ins_cost(INSN_COST);
10313   format %{ "eon  $dst, $src1, zr" %}
10314 
10315   ins_encode %{
10316     __ eon(as_Register($dst$$reg),
10317               as_Register($src1$$reg),
10318               zr,
10319               Assembler::LSL, 0);
10320   %}
10321 
10322   ins_pipe(ialu_reg);
10323 %}
10324 instruct regI_not_reg(iRegINoSp dst,
10325                          iRegIorL2I src1, immI_M1 m1,
10326                          rFlagsReg cr) %{
10327   match(Set dst (XorI src1 m1));
10328   ins_cost(INSN_COST);
10329   format %{ "eonw  $dst, $src1, zr" %}
10330 
10331   ins_encode %{
10332     __ eonw(as_Register($dst$$reg),
10333               as_Register($src1$$reg),
10334               zr,
10335               Assembler::LSL, 0);
10336   %}
10337 
10338   ins_pipe(ialu_reg);
10339 %}
10340 
10341 instruct AndI_reg_not_reg(iRegINoSp dst,
10342                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
10343                          rFlagsReg cr) %{
10344   match(Set dst (AndI src1 (XorI src2 m1)));
10345   ins_cost(INSN_COST);
10346   format %{ "bicw  $dst, $src1, $src2" %}
10347 
10348   ins_encode %{
10349     __ bicw(as_Register($dst$$reg),
10350               as_Register($src1$$reg),
10351               as_Register($src2$$reg),
10352               Assembler::LSL, 0);
10353   %}
10354 
10355   ins_pipe(ialu_reg_reg);
10356 %}
10357 
10358 instruct AndL_reg_not_reg(iRegLNoSp dst,
10359                          iRegL src1, iRegL src2, immL_M1 m1,
10360                          rFlagsReg cr) %{
10361   match(Set dst (AndL src1 (XorL src2 m1)));
10362   ins_cost(INSN_COST);
10363   format %{ "bic  $dst, $src1, $src2" %}
10364 
10365   ins_encode %{
10366     __ bic(as_Register($dst$$reg),
10367               as_Register($src1$$reg),
10368               as_Register($src2$$reg),
10369               Assembler::LSL, 0);
10370   %}
10371 
10372   ins_pipe(ialu_reg_reg);
10373 %}
10374 
10375 instruct OrI_reg_not_reg(iRegINoSp dst,
10376                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
10377                          rFlagsReg cr) %{
10378   match(Set dst (OrI src1 (XorI src2 m1)));
10379   ins_cost(INSN_COST);
10380   format %{ "ornw  $dst, $src1, $src2" %}
10381 
10382   ins_encode %{
10383     __ ornw(as_Register($dst$$reg),
10384               as_Register($src1$$reg),
10385               as_Register($src2$$reg),
10386               Assembler::LSL, 0);
10387   %}
10388 
10389   ins_pipe(ialu_reg_reg);
10390 %}
10391 
10392 instruct OrL_reg_not_reg(iRegLNoSp dst,
10393                          iRegL src1, iRegL src2, immL_M1 m1,
10394                          rFlagsReg cr) %{
10395   match(Set dst (OrL src1 (XorL src2 m1)));
10396   ins_cost(INSN_COST);
10397   format %{ "orn  $dst, $src1, $src2" %}
10398 
10399   ins_encode %{
10400     __ orn(as_Register($dst$$reg),
10401               as_Register($src1$$reg),
10402               as_Register($src2$$reg),
10403               Assembler::LSL, 0);
10404   %}
10405 
10406   ins_pipe(ialu_reg_reg);
10407 %}
10408 
10409 instruct XorI_reg_not_reg(iRegINoSp dst,
10410                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
10411                          rFlagsReg cr) %{
10412   match(Set dst (XorI m1 (XorI src2 src1)));
10413   ins_cost(INSN_COST);
10414   format %{ "eonw  $dst, $src1, $src2" %}
10415 
10416   ins_encode %{
10417     __ eonw(as_Register($dst$$reg),
10418               as_Register($src1$$reg),
10419               as_Register($src2$$reg),
10420               Assembler::LSL, 0);
10421   %}
10422 
10423   ins_pipe(ialu_reg_reg);
10424 %}
10425 
10426 instruct XorL_reg_not_reg(iRegLNoSp dst,
10427                          iRegL src1, iRegL src2, immL_M1 m1,
10428                          rFlagsReg cr) %{
10429   match(Set dst (XorL m1 (XorL src2 src1)));
10430   ins_cost(INSN_COST);
10431   format %{ "eon  $dst, $src1, $src2" %}
10432 
10433   ins_encode %{
10434     __ eon(as_Register($dst$$reg),
10435               as_Register($src1$$reg),
10436               as_Register($src2$$reg),
10437               Assembler::LSL, 0);
10438   %}
10439 
10440   ins_pipe(ialu_reg_reg);
10441 %}
10442 
10443 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10444                          iRegIorL2I src1, iRegIorL2I src2,
10445                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10446   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10447   ins_cost(1.9 * INSN_COST);
10448   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
10449 
10450   ins_encode %{
10451     __ bicw(as_Register($dst$$reg),
10452               as_Register($src1$$reg),
10453               as_Register($src2$$reg),
10454               Assembler::LSR,
10455               $src3$$constant & 0x1f);
10456   %}
10457 
10458   ins_pipe(ialu_reg_reg_shift);
10459 %}
10460 
10461 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10462                          iRegL src1, iRegL src2,
10463                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10464   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10465   ins_cost(1.9 * INSN_COST);
10466   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
10467 
10468   ins_encode %{
10469     __ bic(as_Register($dst$$reg),
10470               as_Register($src1$$reg),
10471               as_Register($src2$$reg),
10472               Assembler::LSR,
10473               $src3$$constant & 0x3f);
10474   %}
10475 
10476   ins_pipe(ialu_reg_reg_shift);
10477 %}
10478 
10479 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10480                          iRegIorL2I src1, iRegIorL2I src2,
10481                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10482   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10483   ins_cost(1.9 * INSN_COST);
10484   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
10485 
10486   ins_encode %{
10487     __ bicw(as_Register($dst$$reg),
10488               as_Register($src1$$reg),
10489               as_Register($src2$$reg),
10490               Assembler::ASR,
10491               $src3$$constant & 0x1f);
10492   %}
10493 
10494   ins_pipe(ialu_reg_reg_shift);
10495 %}
10496 
10497 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10498                          iRegL src1, iRegL src2,
10499                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10500   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10501   ins_cost(1.9 * INSN_COST);
10502   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
10503 
10504   ins_encode %{
10505     __ bic(as_Register($dst$$reg),
10506               as_Register($src1$$reg),
10507               as_Register($src2$$reg),
10508               Assembler::ASR,
10509               $src3$$constant & 0x3f);
10510   %}
10511 
10512   ins_pipe(ialu_reg_reg_shift);
10513 %}
10514 
10515 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10516                          iRegIorL2I src1, iRegIorL2I src2,
10517                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10518   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10519   ins_cost(1.9 * INSN_COST);
10520   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
10521 
10522   ins_encode %{
10523     __ bicw(as_Register($dst$$reg),
10524               as_Register($src1$$reg),
10525               as_Register($src2$$reg),
10526               Assembler::LSL,
10527               $src3$$constant & 0x1f);
10528   %}
10529 
10530   ins_pipe(ialu_reg_reg_shift);
10531 %}
10532 
10533 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10534                          iRegL src1, iRegL src2,
10535                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10536   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10537   ins_cost(1.9 * INSN_COST);
10538   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
10539 
10540   ins_encode %{
10541     __ bic(as_Register($dst$$reg),
10542               as_Register($src1$$reg),
10543               as_Register($src2$$reg),
10544               Assembler::LSL,
10545               $src3$$constant & 0x3f);
10546   %}
10547 
10548   ins_pipe(ialu_reg_reg_shift);
10549 %}
10550 
10551 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10552                          iRegIorL2I src1, iRegIorL2I src2,
10553                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10554   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10555   ins_cost(1.9 * INSN_COST);
10556   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
10557 
10558   ins_encode %{
10559     __ eonw(as_Register($dst$$reg),
10560               as_Register($src1$$reg),
10561               as_Register($src2$$reg),
10562               Assembler::LSR,
10563               $src3$$constant & 0x1f);
10564   %}
10565 
10566   ins_pipe(ialu_reg_reg_shift);
10567 %}
10568 
10569 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10570                          iRegL src1, iRegL src2,
10571                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10572   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10573   ins_cost(1.9 * INSN_COST);
10574   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
10575 
10576   ins_encode %{
10577     __ eon(as_Register($dst$$reg),
10578               as_Register($src1$$reg),
10579               as_Register($src2$$reg),
10580               Assembler::LSR,
10581               $src3$$constant & 0x3f);
10582   %}
10583 
10584   ins_pipe(ialu_reg_reg_shift);
10585 %}
10586 
10587 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10588                          iRegIorL2I src1, iRegIorL2I src2,
10589                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10590   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10591   ins_cost(1.9 * INSN_COST);
10592   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
10593 
10594   ins_encode %{
10595     __ eonw(as_Register($dst$$reg),
10596               as_Register($src1$$reg),
10597               as_Register($src2$$reg),
10598               Assembler::ASR,
10599               $src3$$constant & 0x1f);
10600   %}
10601 
10602   ins_pipe(ialu_reg_reg_shift);
10603 %}
10604 
10605 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10606                          iRegL src1, iRegL src2,
10607                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10608   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10609   ins_cost(1.9 * INSN_COST);
10610   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
10611 
10612   ins_encode %{
10613     __ eon(as_Register($dst$$reg),
10614               as_Register($src1$$reg),
10615               as_Register($src2$$reg),
10616               Assembler::ASR,
10617               $src3$$constant & 0x3f);
10618   %}
10619 
10620   ins_pipe(ialu_reg_reg_shift);
10621 %}
10622 
10623 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10624                          iRegIorL2I src1, iRegIorL2I src2,
10625                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10626   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10627   ins_cost(1.9 * INSN_COST);
10628   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
10629 
10630   ins_encode %{
10631     __ eonw(as_Register($dst$$reg),
10632               as_Register($src1$$reg),
10633               as_Register($src2$$reg),
10634               Assembler::LSL,
10635               $src3$$constant & 0x1f);
10636   %}
10637 
10638   ins_pipe(ialu_reg_reg_shift);
10639 %}
10640 
10641 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10642                          iRegL src1, iRegL src2,
10643                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10644   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10645   ins_cost(1.9 * INSN_COST);
10646   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
10647 
10648   ins_encode %{
10649     __ eon(as_Register($dst$$reg),
10650               as_Register($src1$$reg),
10651               as_Register($src2$$reg),
10652               Assembler::LSL,
10653               $src3$$constant & 0x3f);
10654   %}
10655 
10656   ins_pipe(ialu_reg_reg_shift);
10657 %}
10658 
10659 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10660                          iRegIorL2I src1, iRegIorL2I src2,
10661                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10662   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10663   ins_cost(1.9 * INSN_COST);
10664   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
10665 
10666   ins_encode %{
10667     __ ornw(as_Register($dst$$reg),
10668               as_Register($src1$$reg),
10669               as_Register($src2$$reg),
10670               Assembler::LSR,
10671               $src3$$constant & 0x1f);
10672   %}
10673 
10674   ins_pipe(ialu_reg_reg_shift);
10675 %}
10676 
10677 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10678                          iRegL src1, iRegL src2,
10679                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10680   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10681   ins_cost(1.9 * INSN_COST);
10682   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
10683 
10684   ins_encode %{
10685     __ orn(as_Register($dst$$reg),
10686               as_Register($src1$$reg),
10687               as_Register($src2$$reg),
10688               Assembler::LSR,
10689               $src3$$constant & 0x3f);
10690   %}
10691 
10692   ins_pipe(ialu_reg_reg_shift);
10693 %}
10694 
10695 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10696                          iRegIorL2I src1, iRegIorL2I src2,
10697                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10698   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10699   ins_cost(1.9 * INSN_COST);
10700   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
10701 
10702   ins_encode %{
10703     __ ornw(as_Register($dst$$reg),
10704               as_Register($src1$$reg),
10705               as_Register($src2$$reg),
10706               Assembler::ASR,
10707               $src3$$constant & 0x1f);
10708   %}
10709 
10710   ins_pipe(ialu_reg_reg_shift);
10711 %}
10712 
10713 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10714                          iRegL src1, iRegL src2,
10715                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10716   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10717   ins_cost(1.9 * INSN_COST);
10718   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
10719 
10720   ins_encode %{
10721     __ orn(as_Register($dst$$reg),
10722               as_Register($src1$$reg),
10723               as_Register($src2$$reg),
10724               Assembler::ASR,
10725               $src3$$constant & 0x3f);
10726   %}
10727 
10728   ins_pipe(ialu_reg_reg_shift);
10729 %}
10730 
10731 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10732                          iRegIorL2I src1, iRegIorL2I src2,
10733                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10734   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10735   ins_cost(1.9 * INSN_COST);
10736   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
10737 
10738   ins_encode %{
10739     __ ornw(as_Register($dst$$reg),
10740               as_Register($src1$$reg),
10741               as_Register($src2$$reg),
10742               Assembler::LSL,
10743               $src3$$constant & 0x1f);
10744   %}
10745 
10746   ins_pipe(ialu_reg_reg_shift);
10747 %}
10748 
10749 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10750                          iRegL src1, iRegL src2,
10751                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10752   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10753   ins_cost(1.9 * INSN_COST);
10754   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
10755 
10756   ins_encode %{
10757     __ orn(as_Register($dst$$reg),
10758               as_Register($src1$$reg),
10759               as_Register($src2$$reg),
10760               Assembler::LSL,
10761               $src3$$constant & 0x3f);
10762   %}
10763 
10764   ins_pipe(ialu_reg_reg_shift);
10765 %}
10766 
10767 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10768                          iRegIorL2I src1, iRegIorL2I src2,
10769                          immI src3, rFlagsReg cr) %{
10770   match(Set dst (AndI src1 (URShiftI src2 src3)));
10771 
10772   ins_cost(1.9 * INSN_COST);
10773   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
10774 
10775   ins_encode %{
10776     __ andw(as_Register($dst$$reg),
10777               as_Register($src1$$reg),
10778               as_Register($src2$$reg),
10779               Assembler::LSR,
10780               $src3$$constant & 0x1f);
10781   %}
10782 
10783   ins_pipe(ialu_reg_reg_shift);
10784 %}
10785 
10786 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10787                          iRegL src1, iRegL src2,
10788                          immI src3, rFlagsReg cr) %{
10789   match(Set dst (AndL src1 (URShiftL src2 src3)));
10790 
10791   ins_cost(1.9 * INSN_COST);
10792   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
10793 
10794   ins_encode %{
10795     __ andr(as_Register($dst$$reg),
10796               as_Register($src1$$reg),
10797               as_Register($src2$$reg),
10798               Assembler::LSR,
10799               $src3$$constant & 0x3f);
10800   %}
10801 
10802   ins_pipe(ialu_reg_reg_shift);
10803 %}
10804 
10805 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10806                          iRegIorL2I src1, iRegIorL2I src2,
10807                          immI src3, rFlagsReg cr) %{
10808   match(Set dst (AndI src1 (RShiftI src2 src3)));
10809 
10810   ins_cost(1.9 * INSN_COST);
10811   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
10812 
10813   ins_encode %{
10814     __ andw(as_Register($dst$$reg),
10815               as_Register($src1$$reg),
10816               as_Register($src2$$reg),
10817               Assembler::ASR,
10818               $src3$$constant & 0x1f);
10819   %}
10820 
10821   ins_pipe(ialu_reg_reg_shift);
10822 %}
10823 
10824 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10825                          iRegL src1, iRegL src2,
10826                          immI src3, rFlagsReg cr) %{
10827   match(Set dst (AndL src1 (RShiftL src2 src3)));
10828 
10829   ins_cost(1.9 * INSN_COST);
10830   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
10831 
10832   ins_encode %{
10833     __ andr(as_Register($dst$$reg),
10834               as_Register($src1$$reg),
10835               as_Register($src2$$reg),
10836               Assembler::ASR,
10837               $src3$$constant & 0x3f);
10838   %}
10839 
10840   ins_pipe(ialu_reg_reg_shift);
10841 %}
10842 
10843 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10844                          iRegIorL2I src1, iRegIorL2I src2,
10845                          immI src3, rFlagsReg cr) %{
10846   match(Set dst (AndI src1 (LShiftI src2 src3)));
10847 
10848   ins_cost(1.9 * INSN_COST);
10849   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
10850 
10851   ins_encode %{
10852     __ andw(as_Register($dst$$reg),
10853               as_Register($src1$$reg),
10854               as_Register($src2$$reg),
10855               Assembler::LSL,
10856               $src3$$constant & 0x1f);
10857   %}
10858 
10859   ins_pipe(ialu_reg_reg_shift);
10860 %}
10861 
10862 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10863                          iRegL src1, iRegL src2,
10864                          immI src3, rFlagsReg cr) %{
10865   match(Set dst (AndL src1 (LShiftL src2 src3)));
10866 
10867   ins_cost(1.9 * INSN_COST);
10868   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
10869 
10870   ins_encode %{
10871     __ andr(as_Register($dst$$reg),
10872               as_Register($src1$$reg),
10873               as_Register($src2$$reg),
10874               Assembler::LSL,
10875               $src3$$constant & 0x3f);
10876   %}
10877 
10878   ins_pipe(ialu_reg_reg_shift);
10879 %}
10880 
10881 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10882                          iRegIorL2I src1, iRegIorL2I src2,
10883                          immI src3, rFlagsReg cr) %{
10884   match(Set dst (XorI src1 (URShiftI src2 src3)));
10885 
10886   ins_cost(1.9 * INSN_COST);
10887   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
10888 
10889   ins_encode %{
10890     __ eorw(as_Register($dst$$reg),
10891               as_Register($src1$$reg),
10892               as_Register($src2$$reg),
10893               Assembler::LSR,
10894               $src3$$constant & 0x1f);
10895   %}
10896 
10897   ins_pipe(ialu_reg_reg_shift);
10898 %}
10899 
10900 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10901                          iRegL src1, iRegL src2,
10902                          immI src3, rFlagsReg cr) %{
10903   match(Set dst (XorL src1 (URShiftL src2 src3)));
10904 
10905   ins_cost(1.9 * INSN_COST);
10906   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
10907 
10908   ins_encode %{
10909     __ eor(as_Register($dst$$reg),
10910               as_Register($src1$$reg),
10911               as_Register($src2$$reg),
10912               Assembler::LSR,
10913               $src3$$constant & 0x3f);
10914   %}
10915 
10916   ins_pipe(ialu_reg_reg_shift);
10917 %}
10918 
10919 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10920                          iRegIorL2I src1, iRegIorL2I src2,
10921                          immI src3, rFlagsReg cr) %{
10922   match(Set dst (XorI src1 (RShiftI src2 src3)));
10923 
10924   ins_cost(1.9 * INSN_COST);
10925   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
10926 
10927   ins_encode %{
10928     __ eorw(as_Register($dst$$reg),
10929               as_Register($src1$$reg),
10930               as_Register($src2$$reg),
10931               Assembler::ASR,
10932               $src3$$constant & 0x1f);
10933   %}
10934 
10935   ins_pipe(ialu_reg_reg_shift);
10936 %}
10937 
10938 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10939                          iRegL src1, iRegL src2,
10940                          immI src3, rFlagsReg cr) %{
10941   match(Set dst (XorL src1 (RShiftL src2 src3)));
10942 
10943   ins_cost(1.9 * INSN_COST);
10944   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
10945 
10946   ins_encode %{
10947     __ eor(as_Register($dst$$reg),
10948               as_Register($src1$$reg),
10949               as_Register($src2$$reg),
10950               Assembler::ASR,
10951               $src3$$constant & 0x3f);
10952   %}
10953 
10954   ins_pipe(ialu_reg_reg_shift);
10955 %}
10956 
10957 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10958                          iRegIorL2I src1, iRegIorL2I src2,
10959                          immI src3, rFlagsReg cr) %{
10960   match(Set dst (XorI src1 (LShiftI src2 src3)));
10961 
10962   ins_cost(1.9 * INSN_COST);
10963   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
10964 
10965   ins_encode %{
10966     __ eorw(as_Register($dst$$reg),
10967               as_Register($src1$$reg),
10968               as_Register($src2$$reg),
10969               Assembler::LSL,
10970               $src3$$constant & 0x1f);
10971   %}
10972 
10973   ins_pipe(ialu_reg_reg_shift);
10974 %}
10975 
10976 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10977                          iRegL src1, iRegL src2,
10978                          immI src3, rFlagsReg cr) %{
10979   match(Set dst (XorL src1 (LShiftL src2 src3)));
10980 
10981   ins_cost(1.9 * INSN_COST);
10982   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
10983 
10984   ins_encode %{
10985     __ eor(as_Register($dst$$reg),
10986               as_Register($src1$$reg),
10987               as_Register($src2$$reg),
10988               Assembler::LSL,
10989               $src3$$constant & 0x3f);
10990   %}
10991 
10992   ins_pipe(ialu_reg_reg_shift);
10993 %}
10994 
10995 instruct OrI_reg_URShift_reg(iRegINoSp dst,
10996                          iRegIorL2I src1, iRegIorL2I src2,
10997                          immI src3, rFlagsReg cr) %{
10998   match(Set dst (OrI src1 (URShiftI src2 src3)));
10999 
11000   ins_cost(1.9 * INSN_COST);
11001   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
11002 
11003   ins_encode %{
11004     __ orrw(as_Register($dst$$reg),
11005               as_Register($src1$$reg),
11006               as_Register($src2$$reg),
11007               Assembler::LSR,
11008               $src3$$constant & 0x1f);
11009   %}
11010 
11011   ins_pipe(ialu_reg_reg_shift);
11012 %}
11013 
11014 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11015                          iRegL src1, iRegL src2,
11016                          immI src3, rFlagsReg cr) %{
11017   match(Set dst (OrL src1 (URShiftL src2 src3)));
11018 
11019   ins_cost(1.9 * INSN_COST);
11020   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
11021 
11022   ins_encode %{
11023     __ orr(as_Register($dst$$reg),
11024               as_Register($src1$$reg),
11025               as_Register($src2$$reg),
11026               Assembler::LSR,
11027               $src3$$constant & 0x3f);
11028   %}
11029 
11030   ins_pipe(ialu_reg_reg_shift);
11031 %}
11032 
11033 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11034                          iRegIorL2I src1, iRegIorL2I src2,
11035                          immI src3, rFlagsReg cr) %{
11036   match(Set dst (OrI src1 (RShiftI src2 src3)));
11037 
11038   ins_cost(1.9 * INSN_COST);
11039   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
11040 
11041   ins_encode %{
11042     __ orrw(as_Register($dst$$reg),
11043               as_Register($src1$$reg),
11044               as_Register($src2$$reg),
11045               Assembler::ASR,
11046               $src3$$constant & 0x1f);
11047   %}
11048 
11049   ins_pipe(ialu_reg_reg_shift);
11050 %}
11051 
11052 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11053                          iRegL src1, iRegL src2,
11054                          immI src3, rFlagsReg cr) %{
11055   match(Set dst (OrL src1 (RShiftL src2 src3)));
11056 
11057   ins_cost(1.9 * INSN_COST);
11058   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
11059 
11060   ins_encode %{
11061     __ orr(as_Register($dst$$reg),
11062               as_Register($src1$$reg),
11063               as_Register($src2$$reg),
11064               Assembler::ASR,
11065               $src3$$constant & 0x3f);
11066   %}
11067 
11068   ins_pipe(ialu_reg_reg_shift);
11069 %}
11070 
11071 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11072                          iRegIorL2I src1, iRegIorL2I src2,
11073                          immI src3, rFlagsReg cr) %{
11074   match(Set dst (OrI src1 (LShiftI src2 src3)));
11075 
11076   ins_cost(1.9 * INSN_COST);
11077   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
11078 
11079   ins_encode %{
11080     __ orrw(as_Register($dst$$reg),
11081               as_Register($src1$$reg),
11082               as_Register($src2$$reg),
11083               Assembler::LSL,
11084               $src3$$constant & 0x1f);
11085   %}
11086 
11087   ins_pipe(ialu_reg_reg_shift);
11088 %}
11089 
11090 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11091                          iRegL src1, iRegL src2,
11092                          immI src3, rFlagsReg cr) %{
11093   match(Set dst (OrL src1 (LShiftL src2 src3)));
11094 
11095   ins_cost(1.9 * INSN_COST);
11096   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
11097 
11098   ins_encode %{
11099     __ orr(as_Register($dst$$reg),
11100               as_Register($src1$$reg),
11101               as_Register($src2$$reg),
11102               Assembler::LSL,
11103               $src3$$constant & 0x3f);
11104   %}
11105 
11106   ins_pipe(ialu_reg_reg_shift);
11107 %}
11108 
11109 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11110                          iRegIorL2I src1, iRegIorL2I src2,
11111                          immI src3, rFlagsReg cr) %{
11112   match(Set dst (AddI src1 (URShiftI src2 src3)));
11113 
11114   ins_cost(1.9 * INSN_COST);
11115   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
11116 
11117   ins_encode %{
11118     __ addw(as_Register($dst$$reg),
11119               as_Register($src1$$reg),
11120               as_Register($src2$$reg),
11121               Assembler::LSR,
11122               $src3$$constant & 0x1f);
11123   %}
11124 
11125   ins_pipe(ialu_reg_reg_shift);
11126 %}
11127 
11128 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11129                          iRegL src1, iRegL src2,
11130                          immI src3, rFlagsReg cr) %{
11131   match(Set dst (AddL src1 (URShiftL src2 src3)));
11132 
11133   ins_cost(1.9 * INSN_COST);
11134   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
11135 
11136   ins_encode %{
11137     __ add(as_Register($dst$$reg),
11138               as_Register($src1$$reg),
11139               as_Register($src2$$reg),
11140               Assembler::LSR,
11141               $src3$$constant & 0x3f);
11142   %}
11143 
11144   ins_pipe(ialu_reg_reg_shift);
11145 %}
11146 
11147 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11148                          iRegIorL2I src1, iRegIorL2I src2,
11149                          immI src3, rFlagsReg cr) %{
11150   match(Set dst (AddI src1 (RShiftI src2 src3)));
11151 
11152   ins_cost(1.9 * INSN_COST);
11153   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
11154 
11155   ins_encode %{
11156     __ addw(as_Register($dst$$reg),
11157               as_Register($src1$$reg),
11158               as_Register($src2$$reg),
11159               Assembler::ASR,
11160               $src3$$constant & 0x1f);
11161   %}
11162 
11163   ins_pipe(ialu_reg_reg_shift);
11164 %}
11165 
11166 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11167                          iRegL src1, iRegL src2,
11168                          immI src3, rFlagsReg cr) %{
11169   match(Set dst (AddL src1 (RShiftL src2 src3)));
11170 
11171   ins_cost(1.9 * INSN_COST);
11172   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
11173 
11174   ins_encode %{
11175     __ add(as_Register($dst$$reg),
11176               as_Register($src1$$reg),
11177               as_Register($src2$$reg),
11178               Assembler::ASR,
11179               $src3$$constant & 0x3f);
11180   %}
11181 
11182   ins_pipe(ialu_reg_reg_shift);
11183 %}
11184 
11185 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11186                          iRegIorL2I src1, iRegIorL2I src2,
11187                          immI src3, rFlagsReg cr) %{
11188   match(Set dst (AddI src1 (LShiftI src2 src3)));
11189 
11190   ins_cost(1.9 * INSN_COST);
11191   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
11192 
11193   ins_encode %{
11194     __ addw(as_Register($dst$$reg),
11195               as_Register($src1$$reg),
11196               as_Register($src2$$reg),
11197               Assembler::LSL,
11198               $src3$$constant & 0x1f);
11199   %}
11200 
11201   ins_pipe(ialu_reg_reg_shift);
11202 %}
11203 
11204 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11205                          iRegL src1, iRegL src2,
11206                          immI src3, rFlagsReg cr) %{
11207   match(Set dst (AddL src1 (LShiftL src2 src3)));
11208 
11209   ins_cost(1.9 * INSN_COST);
11210   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
11211 
11212   ins_encode %{
11213     __ add(as_Register($dst$$reg),
11214               as_Register($src1$$reg),
11215               as_Register($src2$$reg),
11216               Assembler::LSL,
11217               $src3$$constant & 0x3f);
11218   %}
11219 
11220   ins_pipe(ialu_reg_reg_shift);
11221 %}
11222 
11223 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11224                          iRegIorL2I src1, iRegIorL2I src2,
11225                          immI src3, rFlagsReg cr) %{
11226   match(Set dst (SubI src1 (URShiftI src2 src3)));
11227 
11228   ins_cost(1.9 * INSN_COST);
11229   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
11230 
11231   ins_encode %{
11232     __ subw(as_Register($dst$$reg),
11233               as_Register($src1$$reg),
11234               as_Register($src2$$reg),
11235               Assembler::LSR,
11236               $src3$$constant & 0x1f);
11237   %}
11238 
11239   ins_pipe(ialu_reg_reg_shift);
11240 %}
11241 
11242 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11243                          iRegL src1, iRegL src2,
11244                          immI src3, rFlagsReg cr) %{
11245   match(Set dst (SubL src1 (URShiftL src2 src3)));
11246 
11247   ins_cost(1.9 * INSN_COST);
11248   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
11249 
11250   ins_encode %{
11251     __ sub(as_Register($dst$$reg),
11252               as_Register($src1$$reg),
11253               as_Register($src2$$reg),
11254               Assembler::LSR,
11255               $src3$$constant & 0x3f);
11256   %}
11257 
11258   ins_pipe(ialu_reg_reg_shift);
11259 %}
11260 
11261 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11262                          iRegIorL2I src1, iRegIorL2I src2,
11263                          immI src3, rFlagsReg cr) %{
11264   match(Set dst (SubI src1 (RShiftI src2 src3)));
11265 
11266   ins_cost(1.9 * INSN_COST);
11267   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
11268 
11269   ins_encode %{
11270     __ subw(as_Register($dst$$reg),
11271               as_Register($src1$$reg),
11272               as_Register($src2$$reg),
11273               Assembler::ASR,
11274               $src3$$constant & 0x1f);
11275   %}
11276 
11277   ins_pipe(ialu_reg_reg_shift);
11278 %}
11279 
11280 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11281                          iRegL src1, iRegL src2,
11282                          immI src3, rFlagsReg cr) %{
11283   match(Set dst (SubL src1 (RShiftL src2 src3)));
11284 
11285   ins_cost(1.9 * INSN_COST);
11286   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
11287 
11288   ins_encode %{
11289     __ sub(as_Register($dst$$reg),
11290               as_Register($src1$$reg),
11291               as_Register($src2$$reg),
11292               Assembler::ASR,
11293               $src3$$constant & 0x3f);
11294   %}
11295 
11296   ins_pipe(ialu_reg_reg_shift);
11297 %}
11298 
11299 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11300                          iRegIorL2I src1, iRegIorL2I src2,
11301                          immI src3, rFlagsReg cr) %{
11302   match(Set dst (SubI src1 (LShiftI src2 src3)));
11303 
11304   ins_cost(1.9 * INSN_COST);
11305   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
11306 
11307   ins_encode %{
11308     __ subw(as_Register($dst$$reg),
11309               as_Register($src1$$reg),
11310               as_Register($src2$$reg),
11311               Assembler::LSL,
11312               $src3$$constant & 0x1f);
11313   %}
11314 
11315   ins_pipe(ialu_reg_reg_shift);
11316 %}
11317 
11318 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11319                          iRegL src1, iRegL src2,
11320                          immI src3, rFlagsReg cr) %{
11321   match(Set dst (SubL src1 (LShiftL src2 src3)));
11322 
11323   ins_cost(1.9 * INSN_COST);
11324   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
11325 
11326   ins_encode %{
11327     __ sub(as_Register($dst$$reg),
11328               as_Register($src1$$reg),
11329               as_Register($src2$$reg),
11330               Assembler::LSL,
11331               $src3$$constant & 0x3f);
11332   %}
11333 
11334   ins_pipe(ialu_reg_reg_shift);
11335 %}
11336 
11337 
11338 
11339 // Shift Left followed by Shift Right.
11340 // This idiom is used by the compiler for the i2b bytecode etc.
11341 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11342 %{
11343   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11344   // Make sure we are not going to exceed what sbfm can do.
11345   predicate((unsigned int)n->in(2)->get_int() <= 63
11346             && (unsigned int)n->in(1)->in(2)->get_int() <= 63);
11347 
11348   ins_cost(INSN_COST * 2);
11349   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11350   ins_encode %{
11351     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
11352     int s = 63 - lshift;
11353     int r = (rshift - lshift) & 63;
11354     __ sbfm(as_Register($dst$$reg),
11355             as_Register($src$$reg),
11356             r, s);
11357   %}
11358 
11359   ins_pipe(ialu_reg_shift);
11360 %}
11361 
11362 // Shift Left followed by Shift Right.
11363 // This idiom is used by the compiler for the i2b bytecode etc.
11364 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11365 %{
11366   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11367   // Make sure we are not going to exceed what sbfmw can do.
11368   predicate((unsigned int)n->in(2)->get_int() <= 31
11369             && (unsigned int)n->in(1)->in(2)->get_int() <= 31);
11370 
11371   ins_cost(INSN_COST * 2);
11372   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11373   ins_encode %{
11374     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
11375     int s = 31 - lshift;
11376     int r = (rshift - lshift) & 31;
11377     __ sbfmw(as_Register($dst$$reg),
11378             as_Register($src$$reg),
11379             r, s);
11380   %}
11381 
11382   ins_pipe(ialu_reg_shift);
11383 %}
11384 
11385 // Shift Left followed by Shift Right.
11386 // This idiom is used by the compiler for the i2b bytecode etc.
11387 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11388 %{
11389   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11390   // Make sure we are not going to exceed what ubfm can do.
11391   predicate((unsigned int)n->in(2)->get_int() <= 63
11392             && (unsigned int)n->in(1)->in(2)->get_int() <= 63);
11393 
11394   ins_cost(INSN_COST * 2);
11395   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11396   ins_encode %{
11397     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
11398     int s = 63 - lshift;
11399     int r = (rshift - lshift) & 63;
11400     __ ubfm(as_Register($dst$$reg),
11401             as_Register($src$$reg),
11402             r, s);
11403   %}
11404 
11405   ins_pipe(ialu_reg_shift);
11406 %}
11407 
11408 // Shift Left followed by Shift Right.
11409 // This idiom is used by the compiler for the i2b bytecode etc.
11410 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11411 %{
11412   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11413   // Make sure we are not going to exceed what ubfmw can do.
11414   predicate((unsigned int)n->in(2)->get_int() <= 31
11415             && (unsigned int)n->in(1)->in(2)->get_int() <= 31);
11416 
11417   ins_cost(INSN_COST * 2);
11418   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11419   ins_encode %{
11420     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
11421     int s = 31 - lshift;
11422     int r = (rshift - lshift) & 31;
11423     __ ubfmw(as_Register($dst$$reg),
11424             as_Register($src$$reg),
11425             r, s);
11426   %}
11427 
11428   ins_pipe(ialu_reg_shift);
11429 %}
11430 // Bitfield extract with shift & mask
11431 
11432 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11433 %{
11434   match(Set dst (AndI (URShiftI src rshift) mask));
11435 
11436   ins_cost(INSN_COST);
11437   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11438   ins_encode %{
11439     int rshift = $rshift$$constant;
11440     long mask = $mask$$constant;
11441     int width = exact_log2(mask+1);
11442     __ ubfxw(as_Register($dst$$reg),
11443             as_Register($src$$reg), rshift, width);
11444   %}
11445   ins_pipe(ialu_reg_shift);
11446 %}
11447 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11448 %{
11449   match(Set dst (AndL (URShiftL src rshift) mask));
11450 
11451   ins_cost(INSN_COST);
11452   format %{ "ubfx $dst, $src, $rshift, $mask" %}
11453   ins_encode %{
11454     int rshift = $rshift$$constant;
11455     long mask = $mask$$constant;
11456     int width = exact_log2(mask+1);
11457     __ ubfx(as_Register($dst$$reg),
11458             as_Register($src$$reg), rshift, width);
11459   %}
11460   ins_pipe(ialu_reg_shift);
11461 %}
11462 
11463 // We can use ubfx when extending an And with a mask when we know mask
11464 // is positive.  We know that because immI_bitmask guarantees it.
11465 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11466 %{
11467   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11468 
11469   ins_cost(INSN_COST * 2);
11470   format %{ "ubfx $dst, $src, $rshift, $mask" %}
11471   ins_encode %{
11472     int rshift = $rshift$$constant;
11473     long mask = $mask$$constant;
11474     int width = exact_log2(mask+1);
11475     __ ubfx(as_Register($dst$$reg),
11476             as_Register($src$$reg), rshift, width);
11477   %}
11478   ins_pipe(ialu_reg_shift);
11479 %}
11480 
11481 // We can use ubfiz when masking by a positive number and then left shifting the result.
11482 // We know that the mask is positive because immI_bitmask guarantees it.
11483 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11484 %{
11485   match(Set dst (LShiftI (AndI src mask) lshift));
11486   predicate((unsigned int)n->in(2)->get_int() <= 31 &&
11487     (exact_log2(n->in(1)->in(2)->get_int()+1) + (unsigned int)n->in(2)->get_int()) <= (31+1));
11488 
11489   ins_cost(INSN_COST);
11490   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11491   ins_encode %{
11492     int lshift = $lshift$$constant;
11493     long mask = $mask$$constant;
11494     int width = exact_log2(mask+1);
11495     __ ubfizw(as_Register($dst$$reg),
11496           as_Register($src$$reg), lshift, width);
11497   %}
11498   ins_pipe(ialu_reg_shift);
11499 %}
11500 // We can use ubfiz when masking by a positive number and then left shifting the result.
11501 // We know that the mask is positive because immL_bitmask guarantees it.
11502 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11503 %{
11504   match(Set dst (LShiftL (AndL src mask) lshift));
11505   predicate((unsigned int)n->in(2)->get_int() <= 63 &&
11506     (exact_log2_long(n->in(1)->in(2)->get_long()+1) + (unsigned int)n->in(2)->get_int()) <= (63+1));
11507 
11508   ins_cost(INSN_COST);
11509   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11510   ins_encode %{
11511     int lshift = $lshift$$constant;
11512     long mask = $mask$$constant;
11513     int width = exact_log2(mask+1);
11514     __ ubfiz(as_Register($dst$$reg),
11515           as_Register($src$$reg), lshift, width);
11516   %}
11517   ins_pipe(ialu_reg_shift);
11518 %}
11519 
11520 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11521 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11522 %{
11523   match(Set dst (LShiftL (ConvI2L(AndI src mask)) lshift));
11524   predicate((unsigned int)n->in(2)->get_int() <= 31 &&
11525     (exact_log2((unsigned int)n->in(1)->in(1)->in(2)->get_int()+1) + (unsigned int)n->in(2)->get_int()) <= 32);
11526 
11527   ins_cost(INSN_COST);
11528   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11529   ins_encode %{
11530     int lshift = $lshift$$constant;
11531     long mask = $mask$$constant;
11532     int width = exact_log2(mask+1);
11533     __ ubfiz(as_Register($dst$$reg),
11534              as_Register($src$$reg), lshift, width);
11535   %}
11536   ins_pipe(ialu_reg_shift);
11537 %}
11538 
11539 // Rotations
11540 
11541 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11542 %{
11543   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11544   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 63));
11545 
11546   ins_cost(INSN_COST);
11547   format %{ "extr $dst, $src1, $src2, #$rshift" %}
11548 
11549   ins_encode %{
11550     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11551             $rshift$$constant & 63);
11552   %}
11553   ins_pipe(ialu_reg_reg_extr);
11554 %}
11555 
11556 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11557 %{
11558   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11559   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 31));
11560 
11561   ins_cost(INSN_COST);
11562   format %{ "extr $dst, $src1, $src2, #$rshift" %}
11563 
11564   ins_encode %{
11565     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11566             $rshift$$constant & 31);
11567   %}
11568   ins_pipe(ialu_reg_reg_extr);
11569 %}
11570 
11571 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11572 %{
11573   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11574   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 63));
11575 
11576   ins_cost(INSN_COST);
11577   format %{ "extr $dst, $src1, $src2, #$rshift" %}
11578 
11579   ins_encode %{
11580     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11581             $rshift$$constant & 63);
11582   %}
11583   ins_pipe(ialu_reg_reg_extr);
11584 %}
11585 
11586 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11587 %{
11588   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11589   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 31));
11590 
11591   ins_cost(INSN_COST);
11592   format %{ "extr $dst, $src1, $src2, #$rshift" %}
11593 
11594   ins_encode %{
11595     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11596             $rshift$$constant & 31);
11597   %}
11598   ins_pipe(ialu_reg_reg_extr);
11599 %}
11600 
11601 
11602 // rol expander
11603 
11604 instruct rolL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
11605 %{
11606   effect(DEF dst, USE src, USE shift);
11607 
11608   format %{ "rol    $dst, $src, $shift" %}
11609   ins_cost(INSN_COST * 3);
11610   ins_encode %{
11611     __ subw(rscratch1, zr, as_Register($shift$$reg));
11612     __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
11613             rscratch1);
11614     %}
11615   ins_pipe(ialu_reg_reg_vshift);
11616 %}
11617 
11618 // rol expander
11619 
11620 instruct rolI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
11621 %{
11622   effect(DEF dst, USE src, USE shift);
11623 
11624   format %{ "rol    $dst, $src, $shift" %}
11625   ins_cost(INSN_COST * 3);
11626   ins_encode %{
11627     __ subw(rscratch1, zr, as_Register($shift$$reg));
11628     __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
11629             rscratch1);
11630     %}
11631   ins_pipe(ialu_reg_reg_vshift);
11632 %}
11633 
11634 instruct rolL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
11635 %{
11636   match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c_64 shift))));
11637 
11638   expand %{
11639     rolL_rReg(dst, src, shift, cr);
11640   %}
11641 %}
11642 
11643 instruct rolL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
11644 %{
11645   match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c0 shift))));
11646 
11647   expand %{
11648     rolL_rReg(dst, src, shift, cr);
11649   %}
11650 %}
11651 
11652 instruct rolI_rReg_Var_C_32(iRegINoSp dst, iRegI src, iRegI shift, immI_32 c_32, rFlagsReg cr)
11653 %{
11654   match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c_32 shift))));
11655 
11656   expand %{
11657     rolI_rReg(dst, src, shift, cr);
11658   %}
11659 %}
11660 
11661 instruct rolI_rReg_Var_C0(iRegINoSp dst, iRegI src, iRegI shift, immI0 c0, rFlagsReg cr)
11662 %{
11663   match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c0 shift))));
11664 
11665   expand %{
11666     rolI_rReg(dst, src, shift, cr);
11667   %}
11668 %}
11669 
11670 // ror expander
11671 
11672 instruct rorL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
11673 %{
11674   effect(DEF dst, USE src, USE shift);
11675 
11676   format %{ "ror    $dst, $src, $shift" %}
11677   ins_cost(INSN_COST);
11678   ins_encode %{
11679     __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
11680             as_Register($shift$$reg));
11681     %}
11682   ins_pipe(ialu_reg_reg_vshift);
11683 %}
11684 
11685 // ror expander
11686 
11687 instruct rorI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
11688 %{
11689   effect(DEF dst, USE src, USE shift);
11690 
11691   format %{ "ror    $dst, $src, $shift" %}
11692   ins_cost(INSN_COST);
11693   ins_encode %{
11694     __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
11695             as_Register($shift$$reg));
11696     %}
11697   ins_pipe(ialu_reg_reg_vshift);
11698 %}
11699 
11700 instruct rorL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
11701 %{
11702   match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c_64 shift))));
11703 
11704   expand %{
11705     rorL_rReg(dst, src, shift, cr);
11706   %}
11707 %}
11708 
11709 instruct rorL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
11710 %{
11711   match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c0 shift))));
11712 
11713   expand %{
11714     rorL_rReg(dst, src, shift, cr);
11715   %}
11716 %}
11717 
11718 instruct rorI_rReg_Var_C_32(iRegINoSp dst, iRegI src, iRegI shift, immI_32 c_32, rFlagsReg cr)
11719 %{
11720   match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c_32 shift))));
11721 
11722   expand %{
11723     rorI_rReg(dst, src, shift, cr);
11724   %}
11725 %}
11726 
11727 instruct rorI_rReg_Var_C0(iRegINoSp dst, iRegI src, iRegI shift, immI0 c0, rFlagsReg cr)
11728 %{
11729   match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c0 shift))));
11730 
11731   expand %{
11732     rorI_rReg(dst, src, shift, cr);
11733   %}
11734 %}
11735 
11736 // Add/subtract (extended)
11737 
11738 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11739 %{
11740   match(Set dst (AddL src1 (ConvI2L src2)));
11741   ins_cost(INSN_COST);
11742   format %{ "add  $dst, $src1, $src2, sxtw" %}
11743 
11744    ins_encode %{
11745      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11746             as_Register($src2$$reg), ext::sxtw);
11747    %}
11748   ins_pipe(ialu_reg_reg);
11749 %};
11750 
11751 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11752 %{
11753   match(Set dst (SubL src1 (ConvI2L src2)));
11754   ins_cost(INSN_COST);
11755   format %{ "sub  $dst, $src1, $src2, sxtw" %}
11756 
11757    ins_encode %{
11758      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11759             as_Register($src2$$reg), ext::sxtw);
11760    %}
11761   ins_pipe(ialu_reg_reg);
11762 %};
11763 
11764 
11765 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11766 %{
11767   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11768   ins_cost(INSN_COST);
11769   format %{ "add  $dst, $src1, $src2, sxth" %}
11770 
11771    ins_encode %{
11772      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11773             as_Register($src2$$reg), ext::sxth);
11774    %}
11775   ins_pipe(ialu_reg_reg);
11776 %}
11777 
11778 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11779 %{
11780   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11781   ins_cost(INSN_COST);
11782   format %{ "add  $dst, $src1, $src2, sxtb" %}
11783 
11784    ins_encode %{
11785      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11786             as_Register($src2$$reg), ext::sxtb);
11787    %}
11788   ins_pipe(ialu_reg_reg);
11789 %}
11790 
11791 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11792 %{
11793   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11794   ins_cost(INSN_COST);
11795   format %{ "add  $dst, $src1, $src2, uxtb" %}
11796 
11797    ins_encode %{
11798      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11799             as_Register($src2$$reg), ext::uxtb);
11800    %}
11801   ins_pipe(ialu_reg_reg);
11802 %}
11803 
11804 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11805 %{
11806   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11807   ins_cost(INSN_COST);
11808   format %{ "add  $dst, $src1, $src2, sxth" %}
11809 
11810    ins_encode %{
11811      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11812             as_Register($src2$$reg), ext::sxth);
11813    %}
11814   ins_pipe(ialu_reg_reg);
11815 %}
11816 
11817 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11818 %{
11819   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11820   ins_cost(INSN_COST);
11821   format %{ "add  $dst, $src1, $src2, sxtw" %}
11822 
11823    ins_encode %{
11824      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11825             as_Register($src2$$reg), ext::sxtw);
11826    %}
11827   ins_pipe(ialu_reg_reg);
11828 %}
11829 
11830 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11831 %{
11832   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11833   ins_cost(INSN_COST);
11834   format %{ "add  $dst, $src1, $src2, sxtb" %}
11835 
11836    ins_encode %{
11837      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11838             as_Register($src2$$reg), ext::sxtb);
11839    %}
11840   ins_pipe(ialu_reg_reg);
11841 %}
11842 
11843 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11844 %{
11845   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
11846   ins_cost(INSN_COST);
11847   format %{ "add  $dst, $src1, $src2, uxtb" %}
11848 
11849    ins_encode %{
11850      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11851             as_Register($src2$$reg), ext::uxtb);
11852    %}
11853   ins_pipe(ialu_reg_reg);
11854 %}
11855 
11856 
11857 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11858 %{
11859   match(Set dst (AddI src1 (AndI src2 mask)));
11860   ins_cost(INSN_COST);
11861   format %{ "addw  $dst, $src1, $src2, uxtb" %}
11862 
11863    ins_encode %{
11864      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11865             as_Register($src2$$reg), ext::uxtb);
11866    %}
11867   ins_pipe(ialu_reg_reg);
11868 %}
11869 
11870 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
11871 %{
11872   match(Set dst (AddI src1 (AndI src2 mask)));
11873   ins_cost(INSN_COST);
11874   format %{ "addw  $dst, $src1, $src2, uxth" %}
11875 
11876    ins_encode %{
11877      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11878             as_Register($src2$$reg), ext::uxth);
11879    %}
11880   ins_pipe(ialu_reg_reg);
11881 %}
11882 
11883 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
11884 %{
11885   match(Set dst (AddL src1 (AndL src2 mask)));
11886   ins_cost(INSN_COST);
11887   format %{ "add  $dst, $src1, $src2, uxtb" %}
11888 
11889    ins_encode %{
11890      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11891             as_Register($src2$$reg), ext::uxtb);
11892    %}
11893   ins_pipe(ialu_reg_reg);
11894 %}
11895 
11896 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
11897 %{
11898   match(Set dst (AddL src1 (AndL src2 mask)));
11899   ins_cost(INSN_COST);
11900   format %{ "add  $dst, $src1, $src2, uxth" %}
11901 
11902    ins_encode %{
11903      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11904             as_Register($src2$$reg), ext::uxth);
11905    %}
11906   ins_pipe(ialu_reg_reg);
11907 %}
11908 
11909 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
11910 %{
11911   match(Set dst (AddL src1 (AndL src2 mask)));
11912   ins_cost(INSN_COST);
11913   format %{ "add  $dst, $src1, $src2, uxtw" %}
11914 
11915    ins_encode %{
11916      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11917             as_Register($src2$$reg), ext::uxtw);
11918    %}
11919   ins_pipe(ialu_reg_reg);
11920 %}
11921 
11922 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11923 %{
11924   match(Set dst (SubI src1 (AndI src2 mask)));
11925   ins_cost(INSN_COST);
11926   format %{ "subw  $dst, $src1, $src2, uxtb" %}
11927 
11928    ins_encode %{
11929      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
11930             as_Register($src2$$reg), ext::uxtb);
11931    %}
11932   ins_pipe(ialu_reg_reg);
11933 %}
11934 
11935 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
11936 %{
11937   match(Set dst (SubI src1 (AndI src2 mask)));
11938   ins_cost(INSN_COST);
11939   format %{ "subw  $dst, $src1, $src2, uxth" %}
11940 
11941    ins_encode %{
11942      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
11943             as_Register($src2$$reg), ext::uxth);
11944    %}
11945   ins_pipe(ialu_reg_reg);
11946 %}
11947 
11948 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
11949 %{
11950   match(Set dst (SubL src1 (AndL src2 mask)));
11951   ins_cost(INSN_COST);
11952   format %{ "sub  $dst, $src1, $src2, uxtb" %}
11953 
11954    ins_encode %{
11955      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11956             as_Register($src2$$reg), ext::uxtb);
11957    %}
11958   ins_pipe(ialu_reg_reg);
11959 %}
11960 
11961 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
11962 %{
11963   match(Set dst (SubL src1 (AndL src2 mask)));
11964   ins_cost(INSN_COST);
11965   format %{ "sub  $dst, $src1, $src2, uxth" %}
11966 
11967    ins_encode %{
11968      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11969             as_Register($src2$$reg), ext::uxth);
11970    %}
11971   ins_pipe(ialu_reg_reg);
11972 %}
11973 
11974 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
11975 %{
11976   match(Set dst (SubL src1 (AndL src2 mask)));
11977   ins_cost(INSN_COST);
11978   format %{ "sub  $dst, $src1, $src2, uxtw" %}
11979 
11980    ins_encode %{
11981      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11982             as_Register($src2$$reg), ext::uxtw);
11983    %}
11984   ins_pipe(ialu_reg_reg);
11985 %}
11986 
11987 
11988 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
11989 %{
11990   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
11991   ins_cost(1.9 * INSN_COST);
11992   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
11993 
11994    ins_encode %{
11995      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11996             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
11997    %}
11998   ins_pipe(ialu_reg_reg_shift);
11999 %}
12000 
12001 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12002 %{
12003   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12004   ins_cost(1.9 * INSN_COST);
12005   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
12006 
12007    ins_encode %{
12008      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12009             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12010    %}
12011   ins_pipe(ialu_reg_reg_shift);
12012 %}
12013 
12014 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12015 %{
12016   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12017   ins_cost(1.9 * INSN_COST);
12018   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
12019 
12020    ins_encode %{
12021      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12022             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12023    %}
12024   ins_pipe(ialu_reg_reg_shift);
12025 %}
12026 
12027 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12028 %{
12029   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12030   ins_cost(1.9 * INSN_COST);
12031   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
12032 
12033    ins_encode %{
12034      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12035             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12036    %}
12037   ins_pipe(ialu_reg_reg_shift);
12038 %}
12039 
12040 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12041 %{
12042   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12043   ins_cost(1.9 * INSN_COST);
12044   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
12045 
12046    ins_encode %{
12047      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12048             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12049    %}
12050   ins_pipe(ialu_reg_reg_shift);
12051 %}
12052 
12053 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12054 %{
12055   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12056   ins_cost(1.9 * INSN_COST);
12057   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
12058 
12059    ins_encode %{
12060      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12061             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12062    %}
12063   ins_pipe(ialu_reg_reg_shift);
12064 %}
12065 
12066 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12067 %{
12068   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12069   ins_cost(1.9 * INSN_COST);
12070   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
12071 
12072    ins_encode %{
12073      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12074             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12075    %}
12076   ins_pipe(ialu_reg_reg_shift);
12077 %}
12078 
12079 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12080 %{
12081   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12082   ins_cost(1.9 * INSN_COST);
12083   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
12084 
12085    ins_encode %{
12086      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12087             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12088    %}
12089   ins_pipe(ialu_reg_reg_shift);
12090 %}
12091 
12092 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12093 %{
12094   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12095   ins_cost(1.9 * INSN_COST);
12096   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
12097 
12098    ins_encode %{
12099      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12100             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12101    %}
12102   ins_pipe(ialu_reg_reg_shift);
12103 %}
12104 
12105 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12106 %{
12107   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12108   ins_cost(1.9 * INSN_COST);
12109   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
12110 
12111    ins_encode %{
12112      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12113             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12114    %}
12115   ins_pipe(ialu_reg_reg_shift);
12116 %}
12117 
12118 
12119 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12120 %{
12121   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12122   ins_cost(1.9 * INSN_COST);
12123   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
12124 
12125    ins_encode %{
12126      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12127             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12128    %}
12129   ins_pipe(ialu_reg_reg_shift);
12130 %};
12131 
12132 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12133 %{
12134   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12135   ins_cost(1.9 * INSN_COST);
12136   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
12137 
12138    ins_encode %{
12139      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12140             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12141    %}
12142   ins_pipe(ialu_reg_reg_shift);
12143 %};
12144 
12145 
12146 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12147 %{
12148   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12149   ins_cost(1.9 * INSN_COST);
12150   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
12151 
12152    ins_encode %{
12153      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12154             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12155    %}
12156   ins_pipe(ialu_reg_reg_shift);
12157 %}
12158 
12159 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12160 %{
12161   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12162   ins_cost(1.9 * INSN_COST);
12163   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
12164 
12165    ins_encode %{
12166      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12167             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12168    %}
12169   ins_pipe(ialu_reg_reg_shift);
12170 %}
12171 
12172 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12173 %{
12174   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12175   ins_cost(1.9 * INSN_COST);
12176   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
12177 
12178    ins_encode %{
12179      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12180             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12181    %}
12182   ins_pipe(ialu_reg_reg_shift);
12183 %}
12184 
12185 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12186 %{
12187   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12188   ins_cost(1.9 * INSN_COST);
12189   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
12190 
12191    ins_encode %{
12192      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12193             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12194    %}
12195   ins_pipe(ialu_reg_reg_shift);
12196 %}
12197 
12198 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12199 %{
12200   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12201   ins_cost(1.9 * INSN_COST);
12202   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
12203 
12204    ins_encode %{
12205      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12206             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12207    %}
12208   ins_pipe(ialu_reg_reg_shift);
12209 %}
12210 
12211 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12212 %{
12213   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12214   ins_cost(1.9 * INSN_COST);
12215   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
12216 
12217    ins_encode %{
12218      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12219             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12220    %}
12221   ins_pipe(ialu_reg_reg_shift);
12222 %}
12223 
12224 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12225 %{
12226   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12227   ins_cost(1.9 * INSN_COST);
12228   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
12229 
12230    ins_encode %{
12231      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12232             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12233    %}
12234   ins_pipe(ialu_reg_reg_shift);
12235 %}
12236 
12237 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12238 %{
12239   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12240   ins_cost(1.9 * INSN_COST);
12241   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
12242 
12243    ins_encode %{
12244      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12245             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12246    %}
12247   ins_pipe(ialu_reg_reg_shift);
12248 %}
12249 
12250 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12251 %{
12252   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12253   ins_cost(1.9 * INSN_COST);
12254   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
12255 
12256    ins_encode %{
12257      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12258             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12259    %}
12260   ins_pipe(ialu_reg_reg_shift);
12261 %}
12262 
12263 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12264 %{
12265   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12266   ins_cost(1.9 * INSN_COST);
12267   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
12268 
12269    ins_encode %{
12270      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12271             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12272    %}
12273   ins_pipe(ialu_reg_reg_shift);
12274 %}
12275 // END This section of the file is automatically generated. Do not edit --------------
12276 
12277 // ============================================================================
12278 // Floating Point Arithmetic Instructions
12279 
12280 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12281   match(Set dst (AddF src1 src2));
12282 
12283   ins_cost(INSN_COST * 5);
12284   format %{ "fadds   $dst, $src1, $src2" %}
12285 
12286   ins_encode %{
12287     __ fadds(as_FloatRegister($dst$$reg),
12288              as_FloatRegister($src1$$reg),
12289              as_FloatRegister($src2$$reg));
12290   %}
12291 
12292   ins_pipe(fp_dop_reg_reg_s);
12293 %}
12294 
12295 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12296   match(Set dst (AddD src1 src2));
12297 
12298   ins_cost(INSN_COST * 5);
12299   format %{ "faddd   $dst, $src1, $src2" %}
12300 
12301   ins_encode %{
12302     __ faddd(as_FloatRegister($dst$$reg),
12303              as_FloatRegister($src1$$reg),
12304              as_FloatRegister($src2$$reg));
12305   %}
12306 
12307   ins_pipe(fp_dop_reg_reg_d);
12308 %}
12309 
12310 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12311   match(Set dst (SubF src1 src2));
12312 
12313   ins_cost(INSN_COST * 5);
12314   format %{ "fsubs   $dst, $src1, $src2" %}
12315 
12316   ins_encode %{
12317     __ fsubs(as_FloatRegister($dst$$reg),
12318              as_FloatRegister($src1$$reg),
12319              as_FloatRegister($src2$$reg));
12320   %}
12321 
12322   ins_pipe(fp_dop_reg_reg_s);
12323 %}
12324 
12325 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12326   match(Set dst (SubD src1 src2));
12327 
12328   ins_cost(INSN_COST * 5);
12329   format %{ "fsubd   $dst, $src1, $src2" %}
12330 
12331   ins_encode %{
12332     __ fsubd(as_FloatRegister($dst$$reg),
12333              as_FloatRegister($src1$$reg),
12334              as_FloatRegister($src2$$reg));
12335   %}
12336 
12337   ins_pipe(fp_dop_reg_reg_d);
12338 %}
12339 
12340 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12341   match(Set dst (MulF src1 src2));
12342 
12343   ins_cost(INSN_COST * 6);
12344   format %{ "fmuls   $dst, $src1, $src2" %}
12345 
12346   ins_encode %{
12347     __ fmuls(as_FloatRegister($dst$$reg),
12348              as_FloatRegister($src1$$reg),
12349              as_FloatRegister($src2$$reg));
12350   %}
12351 
12352   ins_pipe(fp_dop_reg_reg_s);
12353 %}
12354 
12355 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12356   match(Set dst (MulD src1 src2));
12357 
12358   ins_cost(INSN_COST * 6);
12359   format %{ "fmuld   $dst, $src1, $src2" %}
12360 
12361   ins_encode %{
12362     __ fmuld(as_FloatRegister($dst$$reg),
12363              as_FloatRegister($src1$$reg),
12364              as_FloatRegister($src2$$reg));
12365   %}
12366 
12367   ins_pipe(fp_dop_reg_reg_d);
12368 %}
12369 
12370 // src1 * src2 + src3
12371 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12372   predicate(UseFMA);
12373   match(Set dst (FmaF src3 (Binary src1 src2)));
12374 
12375   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
12376 
12377   ins_encode %{
12378     __ fmadds(as_FloatRegister($dst$$reg),
12379              as_FloatRegister($src1$$reg),
12380              as_FloatRegister($src2$$reg),
12381              as_FloatRegister($src3$$reg));
12382   %}
12383 
12384   ins_pipe(pipe_class_default);
12385 %}
12386 
12387 // src1 * src2 + src3
12388 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12389   predicate(UseFMA);
12390   match(Set dst (FmaD src3 (Binary src1 src2)));
12391 
12392   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
12393 
12394   ins_encode %{
12395     __ fmaddd(as_FloatRegister($dst$$reg),
12396              as_FloatRegister($src1$$reg),
12397              as_FloatRegister($src2$$reg),
12398              as_FloatRegister($src3$$reg));
12399   %}
12400 
12401   ins_pipe(pipe_class_default);
12402 %}
12403 
12404 // -src1 * src2 + src3
12405 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12406   predicate(UseFMA);
12407   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
12408   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
12409 
12410   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
12411 
12412   ins_encode %{
12413     __ fmsubs(as_FloatRegister($dst$$reg),
12414               as_FloatRegister($src1$$reg),
12415               as_FloatRegister($src2$$reg),
12416               as_FloatRegister($src3$$reg));
12417   %}
12418 
12419   ins_pipe(pipe_class_default);
12420 %}
12421 
12422 // -src1 * src2 + src3
12423 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12424   predicate(UseFMA);
12425   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
12426   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
12427 
12428   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
12429 
12430   ins_encode %{
12431     __ fmsubd(as_FloatRegister($dst$$reg),
12432               as_FloatRegister($src1$$reg),
12433               as_FloatRegister($src2$$reg),
12434               as_FloatRegister($src3$$reg));
12435   %}
12436 
12437   ins_pipe(pipe_class_default);
12438 %}
12439 
12440 // -src1 * src2 - src3
12441 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12442   predicate(UseFMA);
12443   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
12444   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
12445 
12446   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
12447 
12448   ins_encode %{
12449     __ fnmadds(as_FloatRegister($dst$$reg),
12450                as_FloatRegister($src1$$reg),
12451                as_FloatRegister($src2$$reg),
12452                as_FloatRegister($src3$$reg));
12453   %}
12454 
12455   ins_pipe(pipe_class_default);
12456 %}
12457 
12458 // -src1 * src2 - src3
12459 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12460   predicate(UseFMA);
12461   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
12462   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
12463 
12464   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
12465 
12466   ins_encode %{
12467     __ fnmaddd(as_FloatRegister($dst$$reg),
12468                as_FloatRegister($src1$$reg),
12469                as_FloatRegister($src2$$reg),
12470                as_FloatRegister($src3$$reg));
12471   %}
12472 
12473   ins_pipe(pipe_class_default);
12474 %}
12475 
12476 // src1 * src2 - src3
12477 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
12478   predicate(UseFMA);
12479   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
12480 
12481   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
12482 
12483   ins_encode %{
12484     __ fnmsubs(as_FloatRegister($dst$$reg),
12485                as_FloatRegister($src1$$reg),
12486                as_FloatRegister($src2$$reg),
12487                as_FloatRegister($src3$$reg));
12488   %}
12489 
12490   ins_pipe(pipe_class_default);
12491 %}
12492 
12493 // src1 * src2 - src3
12494 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
12495   predicate(UseFMA);
12496   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
12497 
12498   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
12499 
12500   ins_encode %{
12501   // n.b. insn name should be fnmsubd
12502     __ fnmsub(as_FloatRegister($dst$$reg),
12503               as_FloatRegister($src1$$reg),
12504               as_FloatRegister($src2$$reg),
12505               as_FloatRegister($src3$$reg));
12506   %}
12507 
12508   ins_pipe(pipe_class_default);
12509 %}
12510 
12511 
12512 // Math.max(FF)F
12513 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12514   match(Set dst (MaxF src1 src2));
12515 
12516   format %{ "fmaxs   $dst, $src1, $src2" %}
12517   ins_encode %{
12518     __ fmaxs(as_FloatRegister($dst$$reg),
12519              as_FloatRegister($src1$$reg),
12520              as_FloatRegister($src2$$reg));
12521   %}
12522 
12523   ins_pipe(fp_dop_reg_reg_s);
12524 %}
12525 
12526 // Math.min(FF)F
12527 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12528   match(Set dst (MinF src1 src2));
12529 
12530   format %{ "fmins   $dst, $src1, $src2" %}
12531   ins_encode %{
12532     __ fmins(as_FloatRegister($dst$$reg),
12533              as_FloatRegister($src1$$reg),
12534              as_FloatRegister($src2$$reg));
12535   %}
12536 
12537   ins_pipe(fp_dop_reg_reg_s);
12538 %}
12539 
12540 // Math.max(DD)D
12541 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12542   match(Set dst (MaxD src1 src2));
12543 
12544   format %{ "fmaxd   $dst, $src1, $src2" %}
12545   ins_encode %{
12546     __ fmaxd(as_FloatRegister($dst$$reg),
12547              as_FloatRegister($src1$$reg),
12548              as_FloatRegister($src2$$reg));
12549   %}
12550 
12551   ins_pipe(fp_dop_reg_reg_d);
12552 %}
12553 
12554 // Math.min(DD)D
12555 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12556   match(Set dst (MinD src1 src2));
12557 
12558   format %{ "fmind   $dst, $src1, $src2" %}
12559   ins_encode %{
12560     __ fmind(as_FloatRegister($dst$$reg),
12561              as_FloatRegister($src1$$reg),
12562              as_FloatRegister($src2$$reg));
12563   %}
12564 
12565   ins_pipe(fp_dop_reg_reg_d);
12566 %}
12567 
12568 
12569 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12570   match(Set dst (DivF src1  src2));
12571 
12572   ins_cost(INSN_COST * 18);
12573   format %{ "fdivs   $dst, $src1, $src2" %}
12574 
12575   ins_encode %{
12576     __ fdivs(as_FloatRegister($dst$$reg),
12577              as_FloatRegister($src1$$reg),
12578              as_FloatRegister($src2$$reg));
12579   %}
12580 
12581   ins_pipe(fp_div_s);
12582 %}
12583 
12584 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12585   match(Set dst (DivD src1  src2));
12586 
12587   ins_cost(INSN_COST * 32);
12588   format %{ "fdivd   $dst, $src1, $src2" %}
12589 
12590   ins_encode %{
12591     __ fdivd(as_FloatRegister($dst$$reg),
12592              as_FloatRegister($src1$$reg),
12593              as_FloatRegister($src2$$reg));
12594   %}
12595 
12596   ins_pipe(fp_div_d);
12597 %}
12598 
12599 instruct negF_reg_reg(vRegF dst, vRegF src) %{
12600   match(Set dst (NegF src));
12601 
12602   ins_cost(INSN_COST * 3);
12603   format %{ "fneg   $dst, $src" %}
12604 
12605   ins_encode %{
12606     __ fnegs(as_FloatRegister($dst$$reg),
12607              as_FloatRegister($src$$reg));
12608   %}
12609 
12610   ins_pipe(fp_uop_s);
12611 %}
12612 
12613 instruct negD_reg_reg(vRegD dst, vRegD src) %{
12614   match(Set dst (NegD src));
12615 
12616   ins_cost(INSN_COST * 3);
12617   format %{ "fnegd   $dst, $src" %}
12618 
12619   ins_encode %{
12620     __ fnegd(as_FloatRegister($dst$$reg),
12621              as_FloatRegister($src$$reg));
12622   %}
12623 
12624   ins_pipe(fp_uop_d);
12625 %}
12626 
12627 instruct absF_reg(vRegF dst, vRegF src) %{
12628   match(Set dst (AbsF src));
12629 
12630   ins_cost(INSN_COST * 3);
12631   format %{ "fabss   $dst, $src" %}
12632   ins_encode %{
12633     __ fabss(as_FloatRegister($dst$$reg),
12634              as_FloatRegister($src$$reg));
12635   %}
12636 
12637   ins_pipe(fp_uop_s);
12638 %}
12639 
12640 instruct absD_reg(vRegD dst, vRegD src) %{
12641   match(Set dst (AbsD src));
12642 
12643   ins_cost(INSN_COST * 3);
12644   format %{ "fabsd   $dst, $src" %}
12645   ins_encode %{
12646     __ fabsd(as_FloatRegister($dst$$reg),
12647              as_FloatRegister($src$$reg));
12648   %}
12649 
12650   ins_pipe(fp_uop_d);
12651 %}
12652 
12653 instruct sqrtD_reg(vRegD dst, vRegD src) %{
12654   match(Set dst (SqrtD src));
12655 
12656   ins_cost(INSN_COST * 50);
12657   format %{ "fsqrtd  $dst, $src" %}
12658   ins_encode %{
12659     __ fsqrtd(as_FloatRegister($dst$$reg),
12660              as_FloatRegister($src$$reg));
12661   %}
12662 
12663   ins_pipe(fp_div_s);
12664 %}
12665 
12666 instruct sqrtF_reg(vRegF dst, vRegF src) %{
12667   match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
12668 
12669   ins_cost(INSN_COST * 50);
12670   format %{ "fsqrts  $dst, $src" %}
12671   ins_encode %{
12672     __ fsqrts(as_FloatRegister($dst$$reg),
12673              as_FloatRegister($src$$reg));
12674   %}
12675 
12676   ins_pipe(fp_div_d);
12677 %}
12678 
12679 // ============================================================================
12680 // Logical Instructions
12681 
12682 // Integer Logical Instructions
12683 
12684 // And Instructions
12685 
12686 
12687 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
12688   match(Set dst (AndI src1 src2));
12689 
12690   format %{ "andw  $dst, $src1, $src2\t# int" %}
12691 
12692   ins_cost(INSN_COST);
12693   ins_encode %{
12694     __ andw(as_Register($dst$$reg),
12695             as_Register($src1$$reg),
12696             as_Register($src2$$reg));
12697   %}
12698 
12699   ins_pipe(ialu_reg_reg);
12700 %}
12701 
12702 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
12703   match(Set dst (AndI src1 src2));
12704 
12705   format %{ "andsw  $dst, $src1, $src2\t# int" %}
12706 
12707   ins_cost(INSN_COST);
12708   ins_encode %{
12709     __ andw(as_Register($dst$$reg),
12710             as_Register($src1$$reg),
12711             (unsigned long)($src2$$constant));
12712   %}
12713 
12714   ins_pipe(ialu_reg_imm);
12715 %}
12716 
12717 // Or Instructions
12718 
12719 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
12720   match(Set dst (OrI src1 src2));
12721 
12722   format %{ "orrw  $dst, $src1, $src2\t# int" %}
12723 
12724   ins_cost(INSN_COST);
12725   ins_encode %{
12726     __ orrw(as_Register($dst$$reg),
12727             as_Register($src1$$reg),
12728             as_Register($src2$$reg));
12729   %}
12730 
12731   ins_pipe(ialu_reg_reg);
12732 %}
12733 
12734 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
12735   match(Set dst (OrI src1 src2));
12736 
12737   format %{ "orrw  $dst, $src1, $src2\t# int" %}
12738 
12739   ins_cost(INSN_COST);
12740   ins_encode %{
12741     __ orrw(as_Register($dst$$reg),
12742             as_Register($src1$$reg),
12743             (unsigned long)($src2$$constant));
12744   %}
12745 
12746   ins_pipe(ialu_reg_imm);
12747 %}
12748 
12749 // Xor Instructions
12750 
12751 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
12752   match(Set dst (XorI src1 src2));
12753 
12754   format %{ "eorw  $dst, $src1, $src2\t# int" %}
12755 
12756   ins_cost(INSN_COST);
12757   ins_encode %{
12758     __ eorw(as_Register($dst$$reg),
12759             as_Register($src1$$reg),
12760             as_Register($src2$$reg));
12761   %}
12762 
12763   ins_pipe(ialu_reg_reg);
12764 %}
12765 
12766 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
12767   match(Set dst (XorI src1 src2));
12768 
12769   format %{ "eorw  $dst, $src1, $src2\t# int" %}
12770 
12771   ins_cost(INSN_COST);
12772   ins_encode %{
12773     __ eorw(as_Register($dst$$reg),
12774             as_Register($src1$$reg),
12775             (unsigned long)($src2$$constant));
12776   %}
12777 
12778   ins_pipe(ialu_reg_imm);
12779 %}
12780 
12781 // Long Logical Instructions
12782 // TODO
12783 
12784 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
12785   match(Set dst (AndL src1 src2));
12786 
12787   format %{ "and  $dst, $src1, $src2\t# int" %}
12788 
12789   ins_cost(INSN_COST);
12790   ins_encode %{
12791     __ andr(as_Register($dst$$reg),
12792             as_Register($src1$$reg),
12793             as_Register($src2$$reg));
12794   %}
12795 
12796   ins_pipe(ialu_reg_reg);
12797 %}
12798 
12799 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
12800   match(Set dst (AndL src1 src2));
12801 
12802   format %{ "and  $dst, $src1, $src2\t# int" %}
12803 
12804   ins_cost(INSN_COST);
12805   ins_encode %{
12806     __ andr(as_Register($dst$$reg),
12807             as_Register($src1$$reg),
12808             (unsigned long)($src2$$constant));
12809   %}
12810 
12811   ins_pipe(ialu_reg_imm);
12812 %}
12813 
12814 // Or Instructions
12815 
12816 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
12817   match(Set dst (OrL src1 src2));
12818 
12819   format %{ "orr  $dst, $src1, $src2\t# int" %}
12820 
12821   ins_cost(INSN_COST);
12822   ins_encode %{
12823     __ orr(as_Register($dst$$reg),
12824            as_Register($src1$$reg),
12825            as_Register($src2$$reg));
12826   %}
12827 
12828   ins_pipe(ialu_reg_reg);
12829 %}
12830 
12831 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
12832   match(Set dst (OrL src1 src2));
12833 
12834   format %{ "orr  $dst, $src1, $src2\t# int" %}
12835 
12836   ins_cost(INSN_COST);
12837   ins_encode %{
12838     __ orr(as_Register($dst$$reg),
12839            as_Register($src1$$reg),
12840            (unsigned long)($src2$$constant));
12841   %}
12842 
12843   ins_pipe(ialu_reg_imm);
12844 %}
12845 
12846 // Xor Instructions
12847 
12848 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
12849   match(Set dst (XorL src1 src2));
12850 
12851   format %{ "eor  $dst, $src1, $src2\t# int" %}
12852 
12853   ins_cost(INSN_COST);
12854   ins_encode %{
12855     __ eor(as_Register($dst$$reg),
12856            as_Register($src1$$reg),
12857            as_Register($src2$$reg));
12858   %}
12859 
12860   ins_pipe(ialu_reg_reg);
12861 %}
12862 
12863 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
12864   match(Set dst (XorL src1 src2));
12865 
12866   ins_cost(INSN_COST);
12867   format %{ "eor  $dst, $src1, $src2\t# int" %}
12868 
12869   ins_encode %{
12870     __ eor(as_Register($dst$$reg),
12871            as_Register($src1$$reg),
12872            (unsigned long)($src2$$constant));
12873   %}
12874 
12875   ins_pipe(ialu_reg_imm);
12876 %}
12877 
12878 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
12879 %{
12880   match(Set dst (ConvI2L src));
12881 
12882   ins_cost(INSN_COST);
12883   format %{ "sxtw  $dst, $src\t# i2l" %}
12884   ins_encode %{
12885     __ sbfm($dst$$Register, $src$$Register, 0, 31);
12886   %}
12887   ins_pipe(ialu_reg_shift);
12888 %}
12889 
12890 // this pattern occurs in bigmath arithmetic
12891 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
12892 %{
12893   match(Set dst (AndL (ConvI2L src) mask));
12894 
12895   ins_cost(INSN_COST);
12896   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
12897   ins_encode %{
12898     __ ubfm($dst$$Register, $src$$Register, 0, 31);
12899   %}
12900 
12901   ins_pipe(ialu_reg_shift);
12902 %}
12903 
12904 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
12905   match(Set dst (ConvL2I src));
12906 
12907   ins_cost(INSN_COST);
12908   format %{ "movw  $dst, $src \t// l2i" %}
12909 
12910   ins_encode %{
12911     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
12912   %}
12913 
12914   ins_pipe(ialu_reg);
12915 %}
12916 
12917 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
12918 %{
12919   match(Set dst (Conv2B src));
12920   effect(KILL cr);
12921 
12922   format %{
12923     "cmpw $src, zr\n\t"
12924     "cset $dst, ne"
12925   %}
12926 
12927   ins_encode %{
12928     __ cmpw(as_Register($src$$reg), zr);
12929     __ cset(as_Register($dst$$reg), Assembler::NE);
12930   %}
12931 
12932   ins_pipe(ialu_reg);
12933 %}
12934 
12935 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
12936 %{
12937   match(Set dst (Conv2B src));
12938   effect(KILL cr);
12939 
12940   format %{
12941     "cmp  $src, zr\n\t"
12942     "cset $dst, ne"
12943   %}
12944 
12945   ins_encode %{
12946     __ cmp(as_Register($src$$reg), zr);
12947     __ cset(as_Register($dst$$reg), Assembler::NE);
12948   %}
12949 
12950   ins_pipe(ialu_reg);
12951 %}
12952 
12953 instruct convD2F_reg(vRegF dst, vRegD src) %{
12954   match(Set dst (ConvD2F src));
12955 
12956   ins_cost(INSN_COST * 5);
12957   format %{ "fcvtd  $dst, $src \t// d2f" %}
12958 
12959   ins_encode %{
12960     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
12961   %}
12962 
12963   ins_pipe(fp_d2f);
12964 %}
12965 
12966 instruct convF2D_reg(vRegD dst, vRegF src) %{
12967   match(Set dst (ConvF2D src));
12968 
12969   ins_cost(INSN_COST * 5);
12970   format %{ "fcvts  $dst, $src \t// f2d" %}
12971 
12972   ins_encode %{
12973     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
12974   %}
12975 
12976   ins_pipe(fp_f2d);
12977 %}
12978 
12979 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
12980   match(Set dst (ConvF2I src));
12981 
12982   ins_cost(INSN_COST * 5);
12983   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
12984 
12985   ins_encode %{
12986     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
12987   %}
12988 
12989   ins_pipe(fp_f2i);
12990 %}
12991 
12992 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
12993   match(Set dst (ConvF2L src));
12994 
12995   ins_cost(INSN_COST * 5);
12996   format %{ "fcvtzs  $dst, $src \t// f2l" %}
12997 
12998   ins_encode %{
12999     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13000   %}
13001 
13002   ins_pipe(fp_f2l);
13003 %}
13004 
13005 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13006   match(Set dst (ConvI2F src));
13007 
13008   ins_cost(INSN_COST * 5);
13009   format %{ "scvtfws  $dst, $src \t// i2f" %}
13010 
13011   ins_encode %{
13012     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13013   %}
13014 
13015   ins_pipe(fp_i2f);
13016 %}
13017 
13018 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13019   match(Set dst (ConvL2F src));
13020 
13021   ins_cost(INSN_COST * 5);
13022   format %{ "scvtfs  $dst, $src \t// l2f" %}
13023 
13024   ins_encode %{
13025     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13026   %}
13027 
13028   ins_pipe(fp_l2f);
13029 %}
13030 
13031 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13032   match(Set dst (ConvD2I src));
13033 
13034   ins_cost(INSN_COST * 5);
13035   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
13036 
13037   ins_encode %{
13038     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13039   %}
13040 
13041   ins_pipe(fp_d2i);
13042 %}
13043 
13044 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13045   match(Set dst (ConvD2L src));
13046 
13047   ins_cost(INSN_COST * 5);
13048   format %{ "fcvtzd  $dst, $src \t// d2l" %}
13049 
13050   ins_encode %{
13051     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13052   %}
13053 
13054   ins_pipe(fp_d2l);
13055 %}
13056 
13057 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13058   match(Set dst (ConvI2D src));
13059 
13060   ins_cost(INSN_COST * 5);
13061   format %{ "scvtfwd  $dst, $src \t// i2d" %}
13062 
13063   ins_encode %{
13064     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13065   %}
13066 
13067   ins_pipe(fp_i2d);
13068 %}
13069 
13070 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13071   match(Set dst (ConvL2D src));
13072 
13073   ins_cost(INSN_COST * 5);
13074   format %{ "scvtfd  $dst, $src \t// l2d" %}
13075 
13076   ins_encode %{
13077     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13078   %}
13079 
13080   ins_pipe(fp_l2d);
13081 %}
13082 
13083 // stack <-> reg and reg <-> reg shuffles with no conversion
13084 
13085 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13086 
13087   match(Set dst (MoveF2I src));
13088 
13089   effect(DEF dst, USE src);
13090 
13091   ins_cost(4 * INSN_COST);
13092 
13093   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13094 
13095   ins_encode %{
13096     __ ldrw($dst$$Register, Address(sp, $src$$disp));
13097   %}
13098 
13099   ins_pipe(iload_reg_reg);
13100 
13101 %}
13102 
13103 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13104 
13105   match(Set dst (MoveI2F src));
13106 
13107   effect(DEF dst, USE src);
13108 
13109   ins_cost(4 * INSN_COST);
13110 
13111   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13112 
13113   ins_encode %{
13114     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13115   %}
13116 
13117   ins_pipe(pipe_class_memory);
13118 
13119 %}
13120 
13121 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13122 
13123   match(Set dst (MoveD2L src));
13124 
13125   effect(DEF dst, USE src);
13126 
13127   ins_cost(4 * INSN_COST);
13128 
13129   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13130 
13131   ins_encode %{
13132     __ ldr($dst$$Register, Address(sp, $src$$disp));
13133   %}
13134 
13135   ins_pipe(iload_reg_reg);
13136 
13137 %}
13138 
13139 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13140 
13141   match(Set dst (MoveL2D src));
13142 
13143   effect(DEF dst, USE src);
13144 
13145   ins_cost(4 * INSN_COST);
13146 
13147   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13148 
13149   ins_encode %{
13150     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13151   %}
13152 
13153   ins_pipe(pipe_class_memory);
13154 
13155 %}
13156 
13157 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13158 
13159   match(Set dst (MoveF2I src));
13160 
13161   effect(DEF dst, USE src);
13162 
13163   ins_cost(INSN_COST);
13164 
13165   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13166 
13167   ins_encode %{
13168     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13169   %}
13170 
13171   ins_pipe(pipe_class_memory);
13172 
13173 %}
13174 
13175 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
13176 
13177   match(Set dst (MoveI2F src));
13178 
13179   effect(DEF dst, USE src);
13180 
13181   ins_cost(INSN_COST);
13182 
13183   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
13184 
13185   ins_encode %{
13186     __ strw($src$$Register, Address(sp, $dst$$disp));
13187   %}
13188 
13189   ins_pipe(istore_reg_reg);
13190 
13191 %}
13192 
13193 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
13194 
13195   match(Set dst (MoveD2L src));
13196 
13197   effect(DEF dst, USE src);
13198 
13199   ins_cost(INSN_COST);
13200 
13201   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
13202 
13203   ins_encode %{
13204     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13205   %}
13206 
13207   ins_pipe(pipe_class_memory);
13208 
13209 %}
13210 
13211 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
13212 
13213   match(Set dst (MoveL2D src));
13214 
13215   effect(DEF dst, USE src);
13216 
13217   ins_cost(INSN_COST);
13218 
13219   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
13220 
13221   ins_encode %{
13222     __ str($src$$Register, Address(sp, $dst$$disp));
13223   %}
13224 
13225   ins_pipe(istore_reg_reg);
13226 
13227 %}
13228 
13229 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13230 
13231   match(Set dst (MoveF2I src));
13232 
13233   effect(DEF dst, USE src);
13234 
13235   ins_cost(INSN_COST);
13236 
13237   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
13238 
13239   ins_encode %{
13240     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
13241   %}
13242 
13243   ins_pipe(fp_f2i);
13244 
13245 %}
13246 
13247 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
13248 
13249   match(Set dst (MoveI2F src));
13250 
13251   effect(DEF dst, USE src);
13252 
13253   ins_cost(INSN_COST);
13254 
13255   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
13256 
13257   ins_encode %{
13258     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
13259   %}
13260 
13261   ins_pipe(fp_i2f);
13262 
13263 %}
13264 
13265 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13266 
13267   match(Set dst (MoveD2L src));
13268 
13269   effect(DEF dst, USE src);
13270 
13271   ins_cost(INSN_COST);
13272 
13273   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
13274 
13275   ins_encode %{
13276     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
13277   %}
13278 
13279   ins_pipe(fp_d2l);
13280 
13281 %}
13282 
13283 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
13284 
13285   match(Set dst (MoveL2D src));
13286 
13287   effect(DEF dst, USE src);
13288 
13289   ins_cost(INSN_COST);
13290 
13291   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
13292 
13293   ins_encode %{
13294     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
13295   %}
13296 
13297   ins_pipe(fp_l2d);
13298 
13299 %}
13300 
13301 // ============================================================================
13302 // clearing of an array
13303 
13304 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
13305 %{
13306   match(Set dummy (ClearArray cnt base));
13307   effect(USE_KILL cnt, USE_KILL base);
13308 
13309   ins_cost(4 * INSN_COST);
13310   format %{ "ClearArray $cnt, $base" %}
13311 
13312   ins_encode %{
13313     __ zero_words($base$$Register, $cnt$$Register);
13314   %}
13315 
13316   ins_pipe(pipe_class_memory);
13317 %}
13318 
13319 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
13320 %{
13321   predicate((u_int64_t)n->in(2)->get_long()
13322             < (u_int64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
13323   match(Set dummy (ClearArray cnt base));
13324   effect(USE_KILL base);
13325 
13326   ins_cost(4 * INSN_COST);
13327   format %{ "ClearArray $cnt, $base" %}
13328 
13329   ins_encode %{
13330     __ zero_words($base$$Register, (u_int64_t)$cnt$$constant);
13331   %}
13332 
13333   ins_pipe(pipe_class_memory);
13334 %}
13335 
13336 // ============================================================================
13337 // Overflow Math Instructions
13338 
13339 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
13340 %{
13341   match(Set cr (OverflowAddI op1 op2));
13342 
13343   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
13344   ins_cost(INSN_COST);
13345   ins_encode %{
13346     __ cmnw($op1$$Register, $op2$$Register);
13347   %}
13348 
13349   ins_pipe(icmp_reg_reg);
13350 %}
13351 
13352 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
13353 %{
13354   match(Set cr (OverflowAddI op1 op2));
13355 
13356   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
13357   ins_cost(INSN_COST);
13358   ins_encode %{
13359     __ cmnw($op1$$Register, $op2$$constant);
13360   %}
13361 
13362   ins_pipe(icmp_reg_imm);
13363 %}
13364 
13365 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
13366 %{
13367   match(Set cr (OverflowAddL op1 op2));
13368 
13369   format %{ "cmn   $op1, $op2\t# overflow check long" %}
13370   ins_cost(INSN_COST);
13371   ins_encode %{
13372     __ cmn($op1$$Register, $op2$$Register);
13373   %}
13374 
13375   ins_pipe(icmp_reg_reg);
13376 %}
13377 
13378 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
13379 %{
13380   match(Set cr (OverflowAddL op1 op2));
13381 
13382   format %{ "cmn   $op1, $op2\t# overflow check long" %}
13383   ins_cost(INSN_COST);
13384   ins_encode %{
13385     __ cmn($op1$$Register, $op2$$constant);
13386   %}
13387 
13388   ins_pipe(icmp_reg_imm);
13389 %}
13390 
13391 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
13392 %{
13393   match(Set cr (OverflowSubI op1 op2));
13394 
13395   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
13396   ins_cost(INSN_COST);
13397   ins_encode %{
13398     __ cmpw($op1$$Register, $op2$$Register);
13399   %}
13400 
13401   ins_pipe(icmp_reg_reg);
13402 %}
13403 
13404 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
13405 %{
13406   match(Set cr (OverflowSubI op1 op2));
13407 
13408   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
13409   ins_cost(INSN_COST);
13410   ins_encode %{
13411     __ cmpw($op1$$Register, $op2$$constant);
13412   %}
13413 
13414   ins_pipe(icmp_reg_imm);
13415 %}
13416 
13417 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
13418 %{
13419   match(Set cr (OverflowSubL op1 op2));
13420 
13421   format %{ "cmp   $op1, $op2\t# overflow check long" %}
13422   ins_cost(INSN_COST);
13423   ins_encode %{
13424     __ cmp($op1$$Register, $op2$$Register);
13425   %}
13426 
13427   ins_pipe(icmp_reg_reg);
13428 %}
13429 
13430 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
13431 %{
13432   match(Set cr (OverflowSubL op1 op2));
13433 
13434   format %{ "cmp   $op1, $op2\t# overflow check long" %}
13435   ins_cost(INSN_COST);
13436   ins_encode %{
13437     __ subs(zr, $op1$$Register, $op2$$constant);
13438   %}
13439 
13440   ins_pipe(icmp_reg_imm);
13441 %}
13442 
13443 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
13444 %{
13445   match(Set cr (OverflowSubI zero op1));
13446 
13447   format %{ "cmpw  zr, $op1\t# overflow check int" %}
13448   ins_cost(INSN_COST);
13449   ins_encode %{
13450     __ cmpw(zr, $op1$$Register);
13451   %}
13452 
13453   ins_pipe(icmp_reg_imm);
13454 %}
13455 
13456 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
13457 %{
13458   match(Set cr (OverflowSubL zero op1));
13459 
13460   format %{ "cmp   zr, $op1\t# overflow check long" %}
13461   ins_cost(INSN_COST);
13462   ins_encode %{
13463     __ cmp(zr, $op1$$Register);
13464   %}
13465 
13466   ins_pipe(icmp_reg_imm);
13467 %}
13468 
13469 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
13470 %{
13471   match(Set cr (OverflowMulI op1 op2));
13472 
13473   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
13474             "cmp   rscratch1, rscratch1, sxtw\n\t"
13475             "movw  rscratch1, #0x80000000\n\t"
13476             "cselw rscratch1, rscratch1, zr, NE\n\t"
13477             "cmpw  rscratch1, #1" %}
13478   ins_cost(5 * INSN_COST);
13479   ins_encode %{
13480     __ smull(rscratch1, $op1$$Register, $op2$$Register);
13481     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
13482     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
13483     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
13484     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
13485   %}
13486 
13487   ins_pipe(pipe_slow);
13488 %}
13489 
13490 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
13491 %{
13492   match(If cmp (OverflowMulI op1 op2));
13493   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
13494             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
13495   effect(USE labl, KILL cr);
13496 
13497   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
13498             "cmp   rscratch1, rscratch1, sxtw\n\t"
13499             "b$cmp   $labl" %}
13500   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
13501   ins_encode %{
13502     Label* L = $labl$$label;
13503     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13504     __ smull(rscratch1, $op1$$Register, $op2$$Register);
13505     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
13506     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
13507   %}
13508 
13509   ins_pipe(pipe_serial);
13510 %}
13511 
13512 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
13513 %{
13514   match(Set cr (OverflowMulL op1 op2));
13515 
13516   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
13517             "smulh rscratch2, $op1, $op2\n\t"
13518             "cmp   rscratch2, rscratch1, ASR #63\n\t"
13519             "movw  rscratch1, #0x80000000\n\t"
13520             "cselw rscratch1, rscratch1, zr, NE\n\t"
13521             "cmpw  rscratch1, #1" %}
13522   ins_cost(6 * INSN_COST);
13523   ins_encode %{
13524     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
13525     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
13526     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
13527     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
13528     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
13529     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
13530   %}
13531 
13532   ins_pipe(pipe_slow);
13533 %}
13534 
13535 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
13536 %{
13537   match(If cmp (OverflowMulL op1 op2));
13538   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
13539             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
13540   effect(USE labl, KILL cr);
13541 
13542   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
13543             "smulh rscratch2, $op1, $op2\n\t"
13544             "cmp   rscratch2, rscratch1, ASR #63\n\t"
13545             "b$cmp $labl" %}
13546   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
13547   ins_encode %{
13548     Label* L = $labl$$label;
13549     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13550     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
13551     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
13552     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
13553     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
13554   %}
13555 
13556   ins_pipe(pipe_serial);
13557 %}
13558 
13559 // ============================================================================
13560 // Compare Instructions
13561 
13562 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
13563 %{
13564   match(Set cr (CmpI op1 op2));
13565 
13566   effect(DEF cr, USE op1, USE op2);
13567 
13568   ins_cost(INSN_COST);
13569   format %{ "cmpw  $op1, $op2" %}
13570 
13571   ins_encode(aarch64_enc_cmpw(op1, op2));
13572 
13573   ins_pipe(icmp_reg_reg);
13574 %}
13575 
13576 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
13577 %{
13578   match(Set cr (CmpI op1 zero));
13579 
13580   effect(DEF cr, USE op1);
13581 
13582   ins_cost(INSN_COST);
13583   format %{ "cmpw $op1, 0" %}
13584 
13585   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
13586 
13587   ins_pipe(icmp_reg_imm);
13588 %}
13589 
13590 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
13591 %{
13592   match(Set cr (CmpI op1 op2));
13593 
13594   effect(DEF cr, USE op1);
13595 
13596   ins_cost(INSN_COST);
13597   format %{ "cmpw  $op1, $op2" %}
13598 
13599   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
13600 
13601   ins_pipe(icmp_reg_imm);
13602 %}
13603 
13604 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
13605 %{
13606   match(Set cr (CmpI op1 op2));
13607 
13608   effect(DEF cr, USE op1);
13609 
13610   ins_cost(INSN_COST * 2);
13611   format %{ "cmpw  $op1, $op2" %}
13612 
13613   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
13614 
13615   ins_pipe(icmp_reg_imm);
13616 %}
13617 
13618 // Unsigned compare Instructions; really, same as signed compare
13619 // except it should only be used to feed an If or a CMovI which takes a
13620 // cmpOpU.
13621 
13622 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
13623 %{
13624   match(Set cr (CmpU op1 op2));
13625 
13626   effect(DEF cr, USE op1, USE op2);
13627 
13628   ins_cost(INSN_COST);
13629   format %{ "cmpw  $op1, $op2\t# unsigned" %}
13630 
13631   ins_encode(aarch64_enc_cmpw(op1, op2));
13632 
13633   ins_pipe(icmp_reg_reg);
13634 %}
13635 
13636 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
13637 %{
13638   match(Set cr (CmpU op1 zero));
13639 
13640   effect(DEF cr, USE op1);
13641 
13642   ins_cost(INSN_COST);
13643   format %{ "cmpw $op1, #0\t# unsigned" %}
13644 
13645   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
13646 
13647   ins_pipe(icmp_reg_imm);
13648 %}
13649 
13650 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
13651 %{
13652   match(Set cr (CmpU op1 op2));
13653 
13654   effect(DEF cr, USE op1);
13655 
13656   ins_cost(INSN_COST);
13657   format %{ "cmpw  $op1, $op2\t# unsigned" %}
13658 
13659   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
13660 
13661   ins_pipe(icmp_reg_imm);
13662 %}
13663 
13664 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
13665 %{
13666   match(Set cr (CmpU op1 op2));
13667 
13668   effect(DEF cr, USE op1);
13669 
13670   ins_cost(INSN_COST * 2);
13671   format %{ "cmpw  $op1, $op2\t# unsigned" %}
13672 
13673   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
13674 
13675   ins_pipe(icmp_reg_imm);
13676 %}
13677 
13678 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
13679 %{
13680   match(Set cr (CmpL op1 op2));
13681 
13682   effect(DEF cr, USE op1, USE op2);
13683 
13684   ins_cost(INSN_COST);
13685   format %{ "cmp  $op1, $op2" %}
13686 
13687   ins_encode(aarch64_enc_cmp(op1, op2));
13688 
13689   ins_pipe(icmp_reg_reg);
13690 %}
13691 
13692 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
13693 %{
13694   match(Set cr (CmpL op1 zero));
13695 
13696   effect(DEF cr, USE op1);
13697 
13698   ins_cost(INSN_COST);
13699   format %{ "tst  $op1" %}
13700 
13701   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
13702 
13703   ins_pipe(icmp_reg_imm);
13704 %}
13705 
13706 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
13707 %{
13708   match(Set cr (CmpL op1 op2));
13709 
13710   effect(DEF cr, USE op1);
13711 
13712   ins_cost(INSN_COST);
13713   format %{ "cmp  $op1, $op2" %}
13714 
13715   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
13716 
13717   ins_pipe(icmp_reg_imm);
13718 %}
13719 
13720 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
13721 %{
13722   match(Set cr (CmpL op1 op2));
13723 
13724   effect(DEF cr, USE op1);
13725 
13726   ins_cost(INSN_COST * 2);
13727   format %{ "cmp  $op1, $op2" %}
13728 
13729   ins_encode(aarch64_enc_cmp_imm(op1, op2));
13730 
13731   ins_pipe(icmp_reg_imm);
13732 %}
13733 
13734 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
13735 %{
13736   match(Set cr (CmpUL op1 op2));
13737 
13738   effect(DEF cr, USE op1, USE op2);
13739 
13740   ins_cost(INSN_COST);
13741   format %{ "cmp  $op1, $op2" %}
13742 
13743   ins_encode(aarch64_enc_cmp(op1, op2));
13744 
13745   ins_pipe(icmp_reg_reg);
13746 %}
13747 
13748 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
13749 %{
13750   match(Set cr (CmpUL op1 zero));
13751 
13752   effect(DEF cr, USE op1);
13753 
13754   ins_cost(INSN_COST);
13755   format %{ "tst  $op1" %}
13756 
13757   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
13758 
13759   ins_pipe(icmp_reg_imm);
13760 %}
13761 
13762 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
13763 %{
13764   match(Set cr (CmpUL op1 op2));
13765 
13766   effect(DEF cr, USE op1);
13767 
13768   ins_cost(INSN_COST);
13769   format %{ "cmp  $op1, $op2" %}
13770 
13771   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
13772 
13773   ins_pipe(icmp_reg_imm);
13774 %}
13775 
13776 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
13777 %{
13778   match(Set cr (CmpUL op1 op2));
13779 
13780   effect(DEF cr, USE op1);
13781 
13782   ins_cost(INSN_COST * 2);
13783   format %{ "cmp  $op1, $op2" %}
13784 
13785   ins_encode(aarch64_enc_cmp_imm(op1, op2));
13786 
13787   ins_pipe(icmp_reg_imm);
13788 %}
13789 
13790 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
13791 %{
13792   match(Set cr (CmpP op1 op2));
13793 
13794   effect(DEF cr, USE op1, USE op2);
13795 
13796   ins_cost(INSN_COST);
13797   format %{ "cmp  $op1, $op2\t // ptr" %}
13798 
13799   ins_encode(aarch64_enc_cmpp(op1, op2));
13800 
13801   ins_pipe(icmp_reg_reg);
13802 %}
13803 
13804 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
13805 %{
13806   match(Set cr (CmpN op1 op2));
13807 
13808   effect(DEF cr, USE op1, USE op2);
13809 
13810   ins_cost(INSN_COST);
13811   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
13812 
13813   ins_encode(aarch64_enc_cmpn(op1, op2));
13814 
13815   ins_pipe(icmp_reg_reg);
13816 %}
13817 
13818 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
13819 %{
13820   match(Set cr (CmpP op1 zero));
13821 
13822   effect(DEF cr, USE op1, USE zero);
13823 
13824   ins_cost(INSN_COST);
13825   format %{ "cmp  $op1, 0\t // ptr" %}
13826 
13827   ins_encode(aarch64_enc_testp(op1));
13828 
13829   ins_pipe(icmp_reg_imm);
13830 %}
13831 
13832 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
13833 %{
13834   match(Set cr (CmpN op1 zero));
13835 
13836   effect(DEF cr, USE op1, USE zero);
13837 
13838   ins_cost(INSN_COST);
13839   format %{ "cmp  $op1, 0\t // compressed ptr" %}
13840 
13841   ins_encode(aarch64_enc_testn(op1));
13842 
13843   ins_pipe(icmp_reg_imm);
13844 %}
13845 
13846 // FP comparisons
13847 //
13848 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
13849 // using normal cmpOp. See declaration of rFlagsReg for details.
13850 
13851 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
13852 %{
13853   match(Set cr (CmpF src1 src2));
13854 
13855   ins_cost(3 * INSN_COST);
13856   format %{ "fcmps $src1, $src2" %}
13857 
13858   ins_encode %{
13859     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
13860   %}
13861 
13862   ins_pipe(pipe_class_compare);
13863 %}
13864 
13865 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
13866 %{
13867   match(Set cr (CmpF src1 src2));
13868 
13869   ins_cost(3 * INSN_COST);
13870   format %{ "fcmps $src1, 0.0" %}
13871 
13872   ins_encode %{
13873     __ fcmps(as_FloatRegister($src1$$reg), 0.0D);
13874   %}
13875 
13876   ins_pipe(pipe_class_compare);
13877 %}
13878 // FROM HERE
13879 
13880 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
13881 %{
13882   match(Set cr (CmpD src1 src2));
13883 
13884   ins_cost(3 * INSN_COST);
13885   format %{ "fcmpd $src1, $src2" %}
13886 
13887   ins_encode %{
13888     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
13889   %}
13890 
13891   ins_pipe(pipe_class_compare);
13892 %}
13893 
13894 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
13895 %{
13896   match(Set cr (CmpD src1 src2));
13897 
13898   ins_cost(3 * INSN_COST);
13899   format %{ "fcmpd $src1, 0.0" %}
13900 
13901   ins_encode %{
13902     __ fcmpd(as_FloatRegister($src1$$reg), 0.0D);
13903   %}
13904 
13905   ins_pipe(pipe_class_compare);
13906 %}
13907 
13908 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
13909 %{
13910   match(Set dst (CmpF3 src1 src2));
13911   effect(KILL cr);
13912 
13913   ins_cost(5 * INSN_COST);
13914   format %{ "fcmps $src1, $src2\n\t"
13915             "csinvw($dst, zr, zr, eq\n\t"
13916             "csnegw($dst, $dst, $dst, lt)"
13917   %}
13918 
13919   ins_encode %{
13920     Label done;
13921     FloatRegister s1 = as_FloatRegister($src1$$reg);
13922     FloatRegister s2 = as_FloatRegister($src2$$reg);
13923     Register d = as_Register($dst$$reg);
13924     __ fcmps(s1, s2);
13925     // installs 0 if EQ else -1
13926     __ csinvw(d, zr, zr, Assembler::EQ);
13927     // keeps -1 if less or unordered else installs 1
13928     __ csnegw(d, d, d, Assembler::LT);
13929     __ bind(done);
13930   %}
13931 
13932   ins_pipe(pipe_class_default);
13933 
13934 %}
13935 
13936 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
13937 %{
13938   match(Set dst (CmpD3 src1 src2));
13939   effect(KILL cr);
13940 
13941   ins_cost(5 * INSN_COST);
13942   format %{ "fcmpd $src1, $src2\n\t"
13943             "csinvw($dst, zr, zr, eq\n\t"
13944             "csnegw($dst, $dst, $dst, lt)"
13945   %}
13946 
13947   ins_encode %{
13948     Label done;
13949     FloatRegister s1 = as_FloatRegister($src1$$reg);
13950     FloatRegister s2 = as_FloatRegister($src2$$reg);
13951     Register d = as_Register($dst$$reg);
13952     __ fcmpd(s1, s2);
13953     // installs 0 if EQ else -1
13954     __ csinvw(d, zr, zr, Assembler::EQ);
13955     // keeps -1 if less or unordered else installs 1
13956     __ csnegw(d, d, d, Assembler::LT);
13957     __ bind(done);
13958   %}
13959   ins_pipe(pipe_class_default);
13960 
13961 %}
13962 
13963 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
13964 %{
13965   match(Set dst (CmpF3 src1 zero));
13966   effect(KILL cr);
13967 
13968   ins_cost(5 * INSN_COST);
13969   format %{ "fcmps $src1, 0.0\n\t"
13970             "csinvw($dst, zr, zr, eq\n\t"
13971             "csnegw($dst, $dst, $dst, lt)"
13972   %}
13973 
13974   ins_encode %{
13975     Label done;
13976     FloatRegister s1 = as_FloatRegister($src1$$reg);
13977     Register d = as_Register($dst$$reg);
13978     __ fcmps(s1, 0.0D);
13979     // installs 0 if EQ else -1
13980     __ csinvw(d, zr, zr, Assembler::EQ);
13981     // keeps -1 if less or unordered else installs 1
13982     __ csnegw(d, d, d, Assembler::LT);
13983     __ bind(done);
13984   %}
13985 
13986   ins_pipe(pipe_class_default);
13987 
13988 %}
13989 
13990 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
13991 %{
13992   match(Set dst (CmpD3 src1 zero));
13993   effect(KILL cr);
13994 
13995   ins_cost(5 * INSN_COST);
13996   format %{ "fcmpd $src1, 0.0\n\t"
13997             "csinvw($dst, zr, zr, eq\n\t"
13998             "csnegw($dst, $dst, $dst, lt)"
13999   %}
14000 
14001   ins_encode %{
14002     Label done;
14003     FloatRegister s1 = as_FloatRegister($src1$$reg);
14004     Register d = as_Register($dst$$reg);
14005     __ fcmpd(s1, 0.0D);
14006     // installs 0 if EQ else -1
14007     __ csinvw(d, zr, zr, Assembler::EQ);
14008     // keeps -1 if less or unordered else installs 1
14009     __ csnegw(d, d, d, Assembler::LT);
14010     __ bind(done);
14011   %}
14012   ins_pipe(pipe_class_default);
14013 
14014 %}
14015 
14016 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14017 %{
14018   match(Set dst (CmpLTMask p q));
14019   effect(KILL cr);
14020 
14021   ins_cost(3 * INSN_COST);
14022 
14023   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14024             "csetw $dst, lt\n\t"
14025             "subw $dst, zr, $dst"
14026   %}
14027 
14028   ins_encode %{
14029     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14030     __ csetw(as_Register($dst$$reg), Assembler::LT);
14031     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14032   %}
14033 
14034   ins_pipe(ialu_reg_reg);
14035 %}
14036 
14037 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14038 %{
14039   match(Set dst (CmpLTMask src zero));
14040   effect(KILL cr);
14041 
14042   ins_cost(INSN_COST);
14043 
14044   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14045 
14046   ins_encode %{
14047     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14048   %}
14049 
14050   ins_pipe(ialu_reg_shift);
14051 %}
14052 
14053 // ============================================================================
14054 // Max and Min
14055 
14056 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
14057 %{
14058   match(Set dst (MinI src1 src2));
14059 
14060   effect(DEF dst, USE src1, USE src2, KILL cr);
14061   size(8);
14062 
14063   ins_cost(INSN_COST * 3);
14064   format %{
14065     "cmpw $src1 $src2\t signed int\n\t"
14066     "cselw $dst, $src1, $src2 lt\t"
14067   %}
14068 
14069   ins_encode %{
14070     __ cmpw(as_Register($src1$$reg),
14071             as_Register($src2$$reg));
14072     __ cselw(as_Register($dst$$reg),
14073              as_Register($src1$$reg),
14074              as_Register($src2$$reg),
14075              Assembler::LT);
14076   %}
14077 
14078   ins_pipe(ialu_reg_reg);
14079 %}
14080 // FROM HERE
14081 
14082 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
14083 %{
14084   match(Set dst (MaxI src1 src2));
14085 
14086   effect(DEF dst, USE src1, USE src2, KILL cr);
14087   size(8);
14088 
14089   ins_cost(INSN_COST * 3);
14090   format %{
14091     "cmpw $src1 $src2\t signed int\n\t"
14092     "cselw $dst, $src1, $src2 gt\t"
14093   %}
14094 
14095   ins_encode %{
14096     __ cmpw(as_Register($src1$$reg),
14097             as_Register($src2$$reg));
14098     __ cselw(as_Register($dst$$reg),
14099              as_Register($src1$$reg),
14100              as_Register($src2$$reg),
14101              Assembler::GT);
14102   %}
14103 
14104   ins_pipe(ialu_reg_reg);
14105 %}
14106 
14107 // ============================================================================
14108 // Branch Instructions
14109 
14110 // Direct Branch.
14111 instruct branch(label lbl)
14112 %{
14113   match(Goto);
14114 
14115   effect(USE lbl);
14116 
14117   ins_cost(BRANCH_COST);
14118   format %{ "b  $lbl" %}
14119 
14120   ins_encode(aarch64_enc_b(lbl));
14121 
14122   ins_pipe(pipe_branch);
14123 %}
14124 
14125 // Conditional Near Branch
14126 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14127 %{
14128   // Same match rule as `branchConFar'.
14129   match(If cmp cr);
14130 
14131   effect(USE lbl);
14132 
14133   ins_cost(BRANCH_COST);
14134   // If set to 1 this indicates that the current instruction is a
14135   // short variant of a long branch. This avoids using this
14136   // instruction in first-pass matching. It will then only be used in
14137   // the `Shorten_branches' pass.
14138   // ins_short_branch(1);
14139   format %{ "b$cmp  $lbl" %}
14140 
14141   ins_encode(aarch64_enc_br_con(cmp, lbl));
14142 
14143   ins_pipe(pipe_branch_cond);
14144 %}
14145 
14146 // Conditional Near Branch Unsigned
14147 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14148 %{
14149   // Same match rule as `branchConFar'.
14150   match(If cmp cr);
14151 
14152   effect(USE lbl);
14153 
14154   ins_cost(BRANCH_COST);
14155   // If set to 1 this indicates that the current instruction is a
14156   // short variant of a long branch. This avoids using this
14157   // instruction in first-pass matching. It will then only be used in
14158   // the `Shorten_branches' pass.
14159   // ins_short_branch(1);
14160   format %{ "b$cmp  $lbl\t# unsigned" %}
14161 
14162   ins_encode(aarch64_enc_br_conU(cmp, lbl));
14163 
14164   ins_pipe(pipe_branch_cond);
14165 %}
14166 
14167 // Make use of CBZ and CBNZ.  These instructions, as well as being
14168 // shorter than (cmp; branch), have the additional benefit of not
14169 // killing the flags.
14170 
14171 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14172   match(If cmp (CmpI op1 op2));
14173   effect(USE labl);
14174 
14175   ins_cost(BRANCH_COST);
14176   format %{ "cbw$cmp   $op1, $labl" %}
14177   ins_encode %{
14178     Label* L = $labl$$label;
14179     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14180     if (cond == Assembler::EQ)
14181       __ cbzw($op1$$Register, *L);
14182     else
14183       __ cbnzw($op1$$Register, *L);
14184   %}
14185   ins_pipe(pipe_cmp_branch);
14186 %}
14187 
14188 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
14189   match(If cmp (CmpL op1 op2));
14190   effect(USE labl);
14191 
14192   ins_cost(BRANCH_COST);
14193   format %{ "cb$cmp   $op1, $labl" %}
14194   ins_encode %{
14195     Label* L = $labl$$label;
14196     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14197     if (cond == Assembler::EQ)
14198       __ cbz($op1$$Register, *L);
14199     else
14200       __ cbnz($op1$$Register, *L);
14201   %}
14202   ins_pipe(pipe_cmp_branch);
14203 %}
14204 
14205 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
14206   match(If cmp (CmpP op1 op2));
14207   effect(USE labl);
14208 
14209   ins_cost(BRANCH_COST);
14210   format %{ "cb$cmp   $op1, $labl" %}
14211   ins_encode %{
14212     Label* L = $labl$$label;
14213     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14214     if (cond == Assembler::EQ)
14215       __ cbz($op1$$Register, *L);
14216     else
14217       __ cbnz($op1$$Register, *L);
14218   %}
14219   ins_pipe(pipe_cmp_branch);
14220 %}
14221 
14222 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
14223   match(If cmp (CmpN op1 op2));
14224   effect(USE labl);
14225 
14226   ins_cost(BRANCH_COST);
14227   format %{ "cbw$cmp   $op1, $labl" %}
14228   ins_encode %{
14229     Label* L = $labl$$label;
14230     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14231     if (cond == Assembler::EQ)
14232       __ cbzw($op1$$Register, *L);
14233     else
14234       __ cbnzw($op1$$Register, *L);
14235   %}
14236   ins_pipe(pipe_cmp_branch);
14237 %}
14238 
14239 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
14240   match(If cmp (CmpP (DecodeN oop) zero));
14241   effect(USE labl);
14242 
14243   ins_cost(BRANCH_COST);
14244   format %{ "cb$cmp   $oop, $labl" %}
14245   ins_encode %{
14246     Label* L = $labl$$label;
14247     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14248     if (cond == Assembler::EQ)
14249       __ cbzw($oop$$Register, *L);
14250     else
14251       __ cbnzw($oop$$Register, *L);
14252   %}
14253   ins_pipe(pipe_cmp_branch);
14254 %}
14255 
14256 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
14257   match(If cmp (CmpU op1 op2));
14258   effect(USE labl);
14259 
14260   ins_cost(BRANCH_COST);
14261   format %{ "cbw$cmp   $op1, $labl" %}
14262   ins_encode %{
14263     Label* L = $labl$$label;
14264     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14265     if (cond == Assembler::EQ || cond == Assembler::LS)
14266       __ cbzw($op1$$Register, *L);
14267     else
14268       __ cbnzw($op1$$Register, *L);
14269   %}
14270   ins_pipe(pipe_cmp_branch);
14271 %}
14272 
14273 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
14274   match(If cmp (CmpUL op1 op2));
14275   effect(USE labl);
14276 
14277   ins_cost(BRANCH_COST);
14278   format %{ "cb$cmp   $op1, $labl" %}
14279   ins_encode %{
14280     Label* L = $labl$$label;
14281     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14282     if (cond == Assembler::EQ || cond == Assembler::LS)
14283       __ cbz($op1$$Register, *L);
14284     else
14285       __ cbnz($op1$$Register, *L);
14286   %}
14287   ins_pipe(pipe_cmp_branch);
14288 %}
14289 
14290 // Test bit and Branch
14291 
14292 // Patterns for short (< 32KiB) variants
14293 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
14294   match(If cmp (CmpL op1 op2));
14295   effect(USE labl);
14296 
14297   ins_cost(BRANCH_COST);
14298   format %{ "cb$cmp   $op1, $labl # long" %}
14299   ins_encode %{
14300     Label* L = $labl$$label;
14301     Assembler::Condition cond =
14302       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
14303     __ tbr(cond, $op1$$Register, 63, *L);
14304   %}
14305   ins_pipe(pipe_cmp_branch);
14306   ins_short_branch(1);
14307 %}
14308 
14309 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
14310   match(If cmp (CmpI op1 op2));
14311   effect(USE labl);
14312 
14313   ins_cost(BRANCH_COST);
14314   format %{ "cb$cmp   $op1, $labl # int" %}
14315   ins_encode %{
14316     Label* L = $labl$$label;
14317     Assembler::Condition cond =
14318       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
14319     __ tbr(cond, $op1$$Register, 31, *L);
14320   %}
14321   ins_pipe(pipe_cmp_branch);
14322   ins_short_branch(1);
14323 %}
14324 
14325 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
14326   match(If cmp (CmpL (AndL op1 op2) op3));
14327   predicate(is_power_of_2(n->in(2)->in(1)->in(2)->get_long()));
14328   effect(USE labl);
14329 
14330   ins_cost(BRANCH_COST);
14331   format %{ "tb$cmp   $op1, $op2, $labl" %}
14332   ins_encode %{
14333     Label* L = $labl$$label;
14334     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14335     int bit = exact_log2($op2$$constant);
14336     __ tbr(cond, $op1$$Register, bit, *L);
14337   %}
14338   ins_pipe(pipe_cmp_branch);
14339   ins_short_branch(1);
14340 %}
14341 
14342 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
14343   match(If cmp (CmpI (AndI op1 op2) op3));
14344   predicate(is_power_of_2(n->in(2)->in(1)->in(2)->get_int()));
14345   effect(USE labl);
14346 
14347   ins_cost(BRANCH_COST);
14348   format %{ "tb$cmp   $op1, $op2, $labl" %}
14349   ins_encode %{
14350     Label* L = $labl$$label;
14351     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14352     int bit = exact_log2($op2$$constant);
14353     __ tbr(cond, $op1$$Register, bit, *L);
14354   %}
14355   ins_pipe(pipe_cmp_branch);
14356   ins_short_branch(1);
14357 %}
14358 
14359 // And far variants
14360 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
14361   match(If cmp (CmpL op1 op2));
14362   effect(USE labl);
14363 
14364   ins_cost(BRANCH_COST);
14365   format %{ "cb$cmp   $op1, $labl # long" %}
14366   ins_encode %{
14367     Label* L = $labl$$label;
14368     Assembler::Condition cond =
14369       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
14370     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
14371   %}
14372   ins_pipe(pipe_cmp_branch);
14373 %}
14374 
14375 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
14376   match(If cmp (CmpI op1 op2));
14377   effect(USE labl);
14378 
14379   ins_cost(BRANCH_COST);
14380   format %{ "cb$cmp   $op1, $labl # int" %}
14381   ins_encode %{
14382     Label* L = $labl$$label;
14383     Assembler::Condition cond =
14384       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
14385     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
14386   %}
14387   ins_pipe(pipe_cmp_branch);
14388 %}
14389 
14390 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
14391   match(If cmp (CmpL (AndL op1 op2) op3));
14392   predicate(is_power_of_2(n->in(2)->in(1)->in(2)->get_long()));
14393   effect(USE labl);
14394 
14395   ins_cost(BRANCH_COST);
14396   format %{ "tb$cmp   $op1, $op2, $labl" %}
14397   ins_encode %{
14398     Label* L = $labl$$label;
14399     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14400     int bit = exact_log2($op2$$constant);
14401     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
14402   %}
14403   ins_pipe(pipe_cmp_branch);
14404 %}
14405 
14406 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
14407   match(If cmp (CmpI (AndI op1 op2) op3));
14408   predicate(is_power_of_2(n->in(2)->in(1)->in(2)->get_int()));
14409   effect(USE labl);
14410 
14411   ins_cost(BRANCH_COST);
14412   format %{ "tb$cmp   $op1, $op2, $labl" %}
14413   ins_encode %{
14414     Label* L = $labl$$label;
14415     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14416     int bit = exact_log2($op2$$constant);
14417     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
14418   %}
14419   ins_pipe(pipe_cmp_branch);
14420 %}
14421 
14422 // Test bits
14423 
14424 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
14425   match(Set cr (CmpL (AndL op1 op2) op3));
14426   predicate(Assembler::operand_valid_for_logical_immediate
14427             (/*is_32*/false, n->in(1)->in(2)->get_long()));
14428 
14429   ins_cost(INSN_COST);
14430   format %{ "tst $op1, $op2 # long" %}
14431   ins_encode %{
14432     __ tst($op1$$Register, $op2$$constant);
14433   %}
14434   ins_pipe(ialu_reg_reg);
14435 %}
14436 
14437 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
14438   match(Set cr (CmpI (AndI op1 op2) op3));
14439   predicate(Assembler::operand_valid_for_logical_immediate
14440             (/*is_32*/true, n->in(1)->in(2)->get_int()));
14441 
14442   ins_cost(INSN_COST);
14443   format %{ "tst $op1, $op2 # int" %}
14444   ins_encode %{
14445     __ tstw($op1$$Register, $op2$$constant);
14446   %}
14447   ins_pipe(ialu_reg_reg);
14448 %}
14449 
14450 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
14451   match(Set cr (CmpL (AndL op1 op2) op3));
14452 
14453   ins_cost(INSN_COST);
14454   format %{ "tst $op1, $op2 # long" %}
14455   ins_encode %{
14456     __ tst($op1$$Register, $op2$$Register);
14457   %}
14458   ins_pipe(ialu_reg_reg);
14459 %}
14460 
14461 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
14462   match(Set cr (CmpI (AndI op1 op2) op3));
14463 
14464   ins_cost(INSN_COST);
14465   format %{ "tstw $op1, $op2 # int" %}
14466   ins_encode %{
14467     __ tstw($op1$$Register, $op2$$Register);
14468   %}
14469   ins_pipe(ialu_reg_reg);
14470 %}
14471 
14472 
14473 // Conditional Far Branch
14474 // Conditional Far Branch Unsigned
14475 // TODO: fixme
14476 
14477 // counted loop end branch near
14478 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
14479 %{
14480   match(CountedLoopEnd cmp cr);
14481 
14482   effect(USE lbl);
14483 
14484   ins_cost(BRANCH_COST);
14485   // short variant.
14486   // ins_short_branch(1);
14487   format %{ "b$cmp $lbl \t// counted loop end" %}
14488 
14489   ins_encode(aarch64_enc_br_con(cmp, lbl));
14490 
14491   ins_pipe(pipe_branch);
14492 %}
14493 
14494 // counted loop end branch near Unsigned
14495 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14496 %{
14497   match(CountedLoopEnd cmp cr);
14498 
14499   effect(USE lbl);
14500 
14501   ins_cost(BRANCH_COST);
14502   // short variant.
14503   // ins_short_branch(1);
14504   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
14505 
14506   ins_encode(aarch64_enc_br_conU(cmp, lbl));
14507 
14508   ins_pipe(pipe_branch);
14509 %}
14510 
14511 // counted loop end branch far
14512 // counted loop end branch far unsigned
14513 // TODO: fixme
14514 
14515 // ============================================================================
14516 // inlined locking and unlocking
14517 
14518 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
14519 %{
14520   match(Set cr (FastLock object box));
14521   effect(TEMP tmp, TEMP tmp2);
14522 
14523   // TODO
14524   // identify correct cost
14525   ins_cost(5 * INSN_COST);
14526   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
14527 
14528   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
14529 
14530   ins_pipe(pipe_serial);
14531 %}
14532 
14533 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
14534 %{
14535   match(Set cr (FastUnlock object box));
14536   effect(TEMP tmp, TEMP tmp2);
14537 
14538   ins_cost(5 * INSN_COST);
14539   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
14540 
14541   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
14542 
14543   ins_pipe(pipe_serial);
14544 %}
14545 
14546 
14547 // ============================================================================
14548 // Safepoint Instructions
14549 
14550 // TODO
14551 // provide a near and far version of this code
14552 
14553 instruct safePoint(iRegP poll)
14554 %{
14555   match(SafePoint poll);
14556 
14557   format %{
14558     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
14559   %}
14560   ins_encode %{
14561     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
14562   %}
14563   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
14564 %}
14565 
14566 
14567 // ============================================================================
14568 // Procedure Call/Return Instructions
14569 
14570 // Call Java Static Instruction
14571 
14572 instruct CallStaticJavaDirect(method meth)
14573 %{
14574   match(CallStaticJava);
14575 
14576   effect(USE meth);
14577 
14578   ins_cost(CALL_COST);
14579 
14580   format %{ "call,static $meth \t// ==> " %}
14581 
14582   ins_encode( aarch64_enc_java_static_call(meth),
14583               aarch64_enc_call_epilog );
14584 
14585   ins_pipe(pipe_class_call);
14586 %}
14587 
14588 // TO HERE
14589 
14590 // Call Java Dynamic Instruction
14591 instruct CallDynamicJavaDirect(method meth)
14592 %{
14593   match(CallDynamicJava);
14594 
14595   effect(USE meth);
14596 
14597   ins_cost(CALL_COST);
14598 
14599   format %{ "CALL,dynamic $meth \t// ==> " %}
14600 
14601   ins_encode( aarch64_enc_java_dynamic_call(meth),
14602                aarch64_enc_call_epilog );
14603 
14604   ins_pipe(pipe_class_call);
14605 %}
14606 
14607 // Call Runtime Instruction
14608 
14609 instruct CallRuntimeDirect(method meth)
14610 %{
14611   match(CallRuntime);
14612 
14613   effect(USE meth);
14614 
14615   ins_cost(CALL_COST);
14616 
14617   format %{ "CALL, runtime $meth" %}
14618 
14619   ins_encode( aarch64_enc_java_to_runtime(meth) );
14620 
14621   ins_pipe(pipe_class_call);
14622 %}
14623 
14624 // Call Runtime Instruction
14625 
14626 instruct CallLeafDirect(method meth)
14627 %{
14628   match(CallLeaf);
14629 
14630   effect(USE meth);
14631 
14632   ins_cost(CALL_COST);
14633 
14634   format %{ "CALL, runtime leaf $meth" %}
14635 
14636   ins_encode( aarch64_enc_java_to_runtime(meth) );
14637 
14638   ins_pipe(pipe_class_call);
14639 %}
14640 
14641 // Call Runtime Instruction
14642 
14643 instruct CallLeafNoFPDirect(method meth)
14644 %{
14645   match(CallLeafNoFP);
14646 
14647   effect(USE meth);
14648 
14649   ins_cost(CALL_COST);
14650 
14651   format %{ "CALL, runtime leaf nofp $meth" %}
14652 
14653   ins_encode( aarch64_enc_java_to_runtime(meth) );
14654 
14655   ins_pipe(pipe_class_call);
14656 %}
14657 
14658 // Tail Call; Jump from runtime stub to Java code.
14659 // Also known as an 'interprocedural jump'.
14660 // Target of jump will eventually return to caller.
14661 // TailJump below removes the return address.
14662 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_oop)
14663 %{
14664   match(TailCall jump_target method_oop);
14665 
14666   ins_cost(CALL_COST);
14667 
14668   format %{ "br $jump_target\t# $method_oop holds method oop" %}
14669 
14670   ins_encode(aarch64_enc_tail_call(jump_target));
14671 
14672   ins_pipe(pipe_class_call);
14673 %}
14674 
14675 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
14676 %{
14677   match(TailJump jump_target ex_oop);
14678 
14679   ins_cost(CALL_COST);
14680 
14681   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
14682 
14683   ins_encode(aarch64_enc_tail_jmp(jump_target));
14684 
14685   ins_pipe(pipe_class_call);
14686 %}
14687 
14688 // Create exception oop: created by stack-crawling runtime code.
14689 // Created exception is now available to this handler, and is setup
14690 // just prior to jumping to this handler. No code emitted.
14691 // TODO check
14692 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
14693 instruct CreateException(iRegP_R0 ex_oop)
14694 %{
14695   match(Set ex_oop (CreateEx));
14696 
14697   format %{ " -- \t// exception oop; no code emitted" %}
14698 
14699   size(0);
14700 
14701   ins_encode( /*empty*/ );
14702 
14703   ins_pipe(pipe_class_empty);
14704 %}
14705 
14706 // Rethrow exception: The exception oop will come in the first
14707 // argument position. Then JUMP (not call) to the rethrow stub code.
14708 instruct RethrowException() %{
14709   match(Rethrow);
14710   ins_cost(CALL_COST);
14711 
14712   format %{ "b rethrow_stub" %}
14713 
14714   ins_encode( aarch64_enc_rethrow() );
14715 
14716   ins_pipe(pipe_class_call);
14717 %}
14718 
14719 
14720 // Return Instruction
14721 // epilog node loads ret address into lr as part of frame pop
14722 instruct Ret()
14723 %{
14724   match(Return);
14725 
14726   format %{ "ret\t// return register" %}
14727 
14728   ins_encode( aarch64_enc_ret() );
14729 
14730   ins_pipe(pipe_branch);
14731 %}
14732 
14733 // Die now.
14734 instruct ShouldNotReachHere() %{
14735   match(Halt);
14736 
14737   ins_cost(CALL_COST);
14738   format %{ "ShouldNotReachHere" %}
14739 
14740   ins_encode %{
14741     // +1 so NativeInstruction::is_sigill_zombie_not_entrant() doesn't
14742     // return true
14743     __ dpcs1(0xdead + 1);
14744   %}
14745 
14746   ins_pipe(pipe_class_default);
14747 %}
14748 
14749 // ============================================================================
14750 // Partial Subtype Check
14751 //
14752 // superklass array for an instance of the superklass.  Set a hidden
14753 // internal cache on a hit (cache is checked with exposed code in
14754 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
14755 // encoding ALSO sets flags.
14756 
14757 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
14758 %{
14759   match(Set result (PartialSubtypeCheck sub super));
14760   effect(KILL cr, KILL temp);
14761 
14762   ins_cost(1100);  // slightly larger than the next version
14763   format %{ "partialSubtypeCheck $result, $sub, $super" %}
14764 
14765   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
14766 
14767   opcode(0x1); // Force zero of result reg on hit
14768 
14769   ins_pipe(pipe_class_memory);
14770 %}
14771 
14772 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
14773 %{
14774   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
14775   effect(KILL temp, KILL result);
14776 
14777   ins_cost(1100);  // slightly larger than the next version
14778   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
14779 
14780   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
14781 
14782   opcode(0x0); // Don't zero result reg on hit
14783 
14784   ins_pipe(pipe_class_memory);
14785 %}
14786 
14787 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
14788                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
14789 %{
14790   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
14791   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14792   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14793 
14794   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
14795   ins_encode %{
14796     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
14797     __ string_compare($str1$$Register, $str2$$Register,
14798                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
14799                       $tmp1$$Register, $tmp2$$Register,
14800                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
14801   %}
14802   ins_pipe(pipe_class_memory);
14803 %}
14804 
14805 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
14806                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
14807 %{
14808   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
14809   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14810   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14811 
14812   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
14813   ins_encode %{
14814     __ string_compare($str1$$Register, $str2$$Register,
14815                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
14816                       $tmp1$$Register, $tmp2$$Register,
14817                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
14818   %}
14819   ins_pipe(pipe_class_memory);
14820 %}
14821 
14822 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
14823                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
14824                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
14825 %{
14826   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
14827   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14828   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
14829          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14830 
14831   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
14832   ins_encode %{
14833     __ string_compare($str1$$Register, $str2$$Register,
14834                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
14835                       $tmp1$$Register, $tmp2$$Register,
14836                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
14837                       $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
14838   %}
14839   ins_pipe(pipe_class_memory);
14840 %}
14841 
14842 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
14843                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
14844                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
14845 %{
14846   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
14847   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14848   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
14849          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14850 
14851   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
14852   ins_encode %{
14853     __ string_compare($str1$$Register, $str2$$Register,
14854                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
14855                       $tmp1$$Register, $tmp2$$Register,
14856                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
14857                       $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
14858   %}
14859   ins_pipe(pipe_class_memory);
14860 %}
14861 
14862 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
14863        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
14864        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
14865 %{
14866   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
14867   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
14868   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
14869          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
14870   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
14871 
14872   ins_encode %{
14873     __ string_indexof($str1$$Register, $str2$$Register,
14874                       $cnt1$$Register, $cnt2$$Register,
14875                       $tmp1$$Register, $tmp2$$Register,
14876                       $tmp3$$Register, $tmp4$$Register,
14877                       $tmp5$$Register, $tmp6$$Register,
14878                       -1, $result$$Register, StrIntrinsicNode::UU);
14879   %}
14880   ins_pipe(pipe_class_memory);
14881 %}
14882 
14883 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
14884        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
14885        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
14886 %{
14887   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
14888   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
14889   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
14890          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
14891   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
14892 
14893   ins_encode %{
14894     __ string_indexof($str1$$Register, $str2$$Register,
14895                       $cnt1$$Register, $cnt2$$Register,
14896                       $tmp1$$Register, $tmp2$$Register,
14897                       $tmp3$$Register, $tmp4$$Register,
14898                       $tmp5$$Register, $tmp6$$Register,
14899                       -1, $result$$Register, StrIntrinsicNode::LL);
14900   %}
14901   ins_pipe(pipe_class_memory);
14902 %}
14903 
14904 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
14905        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
14906        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
14907 %{
14908   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
14909   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
14910   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
14911          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
14912   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
14913 
14914   ins_encode %{
14915     __ string_indexof($str1$$Register, $str2$$Register,
14916                       $cnt1$$Register, $cnt2$$Register,
14917                       $tmp1$$Register, $tmp2$$Register,
14918                       $tmp3$$Register, $tmp4$$Register,
14919                       $tmp5$$Register, $tmp6$$Register,
14920                       -1, $result$$Register, StrIntrinsicNode::UL);
14921   %}
14922   ins_pipe(pipe_class_memory);
14923 %}
14924 
14925 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
14926                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
14927                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
14928 %{
14929   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
14930   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
14931   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
14932          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
14933   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
14934 
14935   ins_encode %{
14936     int icnt2 = (int)$int_cnt2$$constant;
14937     __ string_indexof($str1$$Register, $str2$$Register,
14938                       $cnt1$$Register, zr,
14939                       $tmp1$$Register, $tmp2$$Register,
14940                       $tmp3$$Register, $tmp4$$Register, zr, zr,
14941                       icnt2, $result$$Register, StrIntrinsicNode::UU);
14942   %}
14943   ins_pipe(pipe_class_memory);
14944 %}
14945 
14946 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
14947                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
14948                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
14949 %{
14950   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
14951   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
14952   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
14953          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
14954   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
14955 
14956   ins_encode %{
14957     int icnt2 = (int)$int_cnt2$$constant;
14958     __ string_indexof($str1$$Register, $str2$$Register,
14959                       $cnt1$$Register, zr,
14960                       $tmp1$$Register, $tmp2$$Register,
14961                       $tmp3$$Register, $tmp4$$Register, zr, zr,
14962                       icnt2, $result$$Register, StrIntrinsicNode::LL);
14963   %}
14964   ins_pipe(pipe_class_memory);
14965 %}
14966 
14967 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
14968                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
14969                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
14970 %{
14971   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
14972   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
14973   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
14974          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
14975   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
14976 
14977   ins_encode %{
14978     int icnt2 = (int)$int_cnt2$$constant;
14979     __ string_indexof($str1$$Register, $str2$$Register,
14980                       $cnt1$$Register, zr,
14981                       $tmp1$$Register, $tmp2$$Register,
14982                       $tmp3$$Register, $tmp4$$Register, zr, zr,
14983                       icnt2, $result$$Register, StrIntrinsicNode::UL);
14984   %}
14985   ins_pipe(pipe_class_memory);
14986 %}
14987 
14988 instruct string_indexofU_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
14989                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
14990                               iRegINoSp tmp3, rFlagsReg cr)
14991 %{
14992   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
14993   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
14994          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
14995 
14996   format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result" %}
14997 
14998   ins_encode %{
14999     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15000                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
15001                            $tmp3$$Register);
15002   %}
15003   ins_pipe(pipe_class_memory);
15004 %}
15005 
15006 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
15007                         iRegI_R0 result, rFlagsReg cr)
15008 %{
15009   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
15010   match(Set result (StrEquals (Binary str1 str2) cnt));
15011   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
15012 
15013   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
15014   ins_encode %{
15015     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15016     __ string_equals($str1$$Register, $str2$$Register,
15017                      $result$$Register, $cnt$$Register, 1);
15018   %}
15019   ins_pipe(pipe_class_memory);
15020 %}
15021 
15022 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
15023                         iRegI_R0 result, rFlagsReg cr)
15024 %{
15025   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
15026   match(Set result (StrEquals (Binary str1 str2) cnt));
15027   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
15028 
15029   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
15030   ins_encode %{
15031     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15032     __ string_equals($str1$$Register, $str2$$Register,
15033                      $result$$Register, $cnt$$Register, 2);
15034   %}
15035   ins_pipe(pipe_class_memory);
15036 %}
15037 
15038 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
15039                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
15040                        iRegP_R10 tmp, rFlagsReg cr)
15041 %{
15042   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
15043   match(Set result (AryEq ary1 ary2));
15044   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15045 
15046   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
15047   ins_encode %{
15048     __ arrays_equals($ary1$$Register, $ary2$$Register,
15049                      $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
15050                      $result$$Register, $tmp$$Register, 1);
15051     %}
15052   ins_pipe(pipe_class_memory);
15053 %}
15054 
15055 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
15056                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
15057                        iRegP_R10 tmp, rFlagsReg cr)
15058 %{
15059   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
15060   match(Set result (AryEq ary1 ary2));
15061   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15062 
15063   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
15064   ins_encode %{
15065     __ arrays_equals($ary1$$Register, $ary2$$Register,
15066                      $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
15067                      $result$$Register, $tmp$$Register, 2);
15068   %}
15069   ins_pipe(pipe_class_memory);
15070 %}
15071 
15072 instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
15073 %{
15074   match(Set result (HasNegatives ary1 len));
15075   effect(USE_KILL ary1, USE_KILL len, KILL cr);
15076   format %{ "has negatives byte[] $ary1,$len -> $result" %}
15077   ins_encode %{
15078     __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
15079   %}
15080   ins_pipe( pipe_slow );
15081 %}
15082 
15083 // fast char[] to byte[] compression
15084 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
15085                          vRegD_V0 tmp1, vRegD_V1 tmp2,
15086                          vRegD_V2 tmp3, vRegD_V3 tmp4,
15087                          iRegI_R0 result, rFlagsReg cr)
15088 %{
15089   match(Set result (StrCompressedCopy src (Binary dst len)));
15090   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
15091 
15092   format %{ "String Compress $src,$dst -> $result    // KILL R1, R2, R3, R4" %}
15093   ins_encode %{
15094     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
15095                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
15096                            $tmp3$$FloatRegister, $tmp4$$FloatRegister,
15097                            $result$$Register);
15098   %}
15099   ins_pipe( pipe_slow );
15100 %}
15101 
15102 // fast byte[] to char[] inflation
15103 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
15104                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
15105 %{
15106   match(Set dummy (StrInflatedCopy src (Binary dst len)));
15107   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
15108 
15109   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
15110   ins_encode %{
15111     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
15112                           $tmp1$$FloatRegister, $tmp2$$FloatRegister, $tmp3$$FloatRegister, $tmp4$$Register);
15113   %}
15114   ins_pipe(pipe_class_memory);
15115 %}
15116 
15117 // encode char[] to byte[] in ISO_8859_1
15118 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
15119                           vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
15120                           vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
15121                           iRegI_R0 result, rFlagsReg cr)
15122 %{
15123   match(Set result (EncodeISOArray src (Binary dst len)));
15124   effect(USE_KILL src, USE_KILL dst, USE_KILL len,
15125          KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
15126 
15127   format %{ "Encode array $src,$dst,$len -> $result" %}
15128   ins_encode %{
15129     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
15130          $result$$Register, $Vtmp1$$FloatRegister,  $Vtmp2$$FloatRegister,
15131          $Vtmp3$$FloatRegister,  $Vtmp4$$FloatRegister);
15132   %}
15133   ins_pipe( pipe_class_memory );
15134 %}
15135 
15136 // ============================================================================
15137 // This name is KNOWN by the ADLC and cannot be changed.
15138 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
15139 // for this guy.
15140 instruct tlsLoadP(thread_RegP dst)
15141 %{
15142   match(Set dst (ThreadLocal));
15143 
15144   ins_cost(0);
15145 
15146   format %{ " -- \t// $dst=Thread::current(), empty" %}
15147 
15148   size(0);
15149 
15150   ins_encode( /*empty*/ );
15151 
15152   ins_pipe(pipe_class_empty);
15153 %}
15154 
15155 // ====================VECTOR INSTRUCTIONS=====================================
15156 
15157 // Load vector (32 bits)
15158 instruct loadV4(vecD dst, vmem4 mem)
15159 %{
15160   predicate(n->as_LoadVector()->memory_size() == 4);
15161   match(Set dst (LoadVector mem));
15162   ins_cost(4 * INSN_COST);
15163   format %{ "ldrs   $dst,$mem\t# vector (32 bits)" %}
15164   ins_encode( aarch64_enc_ldrvS(dst, mem) );
15165   ins_pipe(vload_reg_mem64);
15166 %}
15167 
15168 // Load vector (64 bits)
15169 instruct loadV8(vecD dst, vmem8 mem)
15170 %{
15171   predicate(n->as_LoadVector()->memory_size() == 8);
15172   match(Set dst (LoadVector mem));
15173   ins_cost(4 * INSN_COST);
15174   format %{ "ldrd   $dst,$mem\t# vector (64 bits)" %}
15175   ins_encode( aarch64_enc_ldrvD(dst, mem) );
15176   ins_pipe(vload_reg_mem64);
15177 %}
15178 
15179 // Load Vector (128 bits)
15180 instruct loadV16(vecX dst, vmem16 mem)
15181 %{
15182   predicate(n->as_LoadVector()->memory_size() == 16);
15183   match(Set dst (LoadVector mem));
15184   ins_cost(4 * INSN_COST);
15185   format %{ "ldrq   $dst,$mem\t# vector (128 bits)" %}
15186   ins_encode( aarch64_enc_ldrvQ(dst, mem) );
15187   ins_pipe(vload_reg_mem128);
15188 %}
15189 
15190 // Store Vector (32 bits)
15191 instruct storeV4(vecD src, vmem4 mem)
15192 %{
15193   predicate(n->as_StoreVector()->memory_size() == 4);
15194   match(Set mem (StoreVector mem src));
15195   ins_cost(4 * INSN_COST);
15196   format %{ "strs   $mem,$src\t# vector (32 bits)" %}
15197   ins_encode( aarch64_enc_strvS(src, mem) );
15198   ins_pipe(vstore_reg_mem64);
15199 %}
15200 
15201 // Store Vector (64 bits)
15202 instruct storeV8(vecD src, vmem8 mem)
15203 %{
15204   predicate(n->as_StoreVector()->memory_size() == 8);
15205   match(Set mem (StoreVector mem src));
15206   ins_cost(4 * INSN_COST);
15207   format %{ "strd   $mem,$src\t# vector (64 bits)" %}
15208   ins_encode( aarch64_enc_strvD(src, mem) );
15209   ins_pipe(vstore_reg_mem64);
15210 %}
15211 
15212 // Store Vector (128 bits)
15213 instruct storeV16(vecX src, vmem16 mem)
15214 %{
15215   predicate(n->as_StoreVector()->memory_size() == 16);
15216   match(Set mem (StoreVector mem src));
15217   ins_cost(4 * INSN_COST);
15218   format %{ "strq   $mem,$src\t# vector (128 bits)" %}
15219   ins_encode( aarch64_enc_strvQ(src, mem) );
15220   ins_pipe(vstore_reg_mem128);
15221 %}
15222 
15223 instruct replicate8B(vecD dst, iRegIorL2I src)
15224 %{
15225   predicate(n->as_Vector()->length() == 4 ||
15226             n->as_Vector()->length() == 8);
15227   match(Set dst (ReplicateB src));
15228   ins_cost(INSN_COST);
15229   format %{ "dup  $dst, $src\t# vector (8B)" %}
15230   ins_encode %{
15231     __ dup(as_FloatRegister($dst$$reg), __ T8B, as_Register($src$$reg));
15232   %}
15233   ins_pipe(vdup_reg_reg64);
15234 %}
15235 
15236 instruct replicate16B(vecX dst, iRegIorL2I src)
15237 %{
15238   predicate(n->as_Vector()->length() == 16);
15239   match(Set dst (ReplicateB src));
15240   ins_cost(INSN_COST);
15241   format %{ "dup  $dst, $src\t# vector (16B)" %}
15242   ins_encode %{
15243     __ dup(as_FloatRegister($dst$$reg), __ T16B, as_Register($src$$reg));
15244   %}
15245   ins_pipe(vdup_reg_reg128);
15246 %}
15247 
15248 instruct replicate8B_imm(vecD dst, immI con)
15249 %{
15250   predicate(n->as_Vector()->length() == 4 ||
15251             n->as_Vector()->length() == 8);
15252   match(Set dst (ReplicateB con));
15253   ins_cost(INSN_COST);
15254   format %{ "movi  $dst, $con\t# vector(8B)" %}
15255   ins_encode %{
15256     __ mov(as_FloatRegister($dst$$reg), __ T8B, $con$$constant & 0xff);
15257   %}
15258   ins_pipe(vmovi_reg_imm64);
15259 %}
15260 
15261 instruct replicate16B_imm(vecX dst, immI con)
15262 %{
15263   predicate(n->as_Vector()->length() == 16);
15264   match(Set dst (ReplicateB con));
15265   ins_cost(INSN_COST);
15266   format %{ "movi  $dst, $con\t# vector(16B)" %}
15267   ins_encode %{
15268     __ mov(as_FloatRegister($dst$$reg), __ T16B, $con$$constant & 0xff);
15269   %}
15270   ins_pipe(vmovi_reg_imm128);
15271 %}
15272 
15273 instruct replicate4S(vecD dst, iRegIorL2I src)
15274 %{
15275   predicate(n->as_Vector()->length() == 2 ||
15276             n->as_Vector()->length() == 4);
15277   match(Set dst (ReplicateS src));
15278   ins_cost(INSN_COST);
15279   format %{ "dup  $dst, $src\t# vector (4S)" %}
15280   ins_encode %{
15281     __ dup(as_FloatRegister($dst$$reg), __ T4H, as_Register($src$$reg));
15282   %}
15283   ins_pipe(vdup_reg_reg64);
15284 %}
15285 
15286 instruct replicate8S(vecX dst, iRegIorL2I src)
15287 %{
15288   predicate(n->as_Vector()->length() == 8);
15289   match(Set dst (ReplicateS src));
15290   ins_cost(INSN_COST);
15291   format %{ "dup  $dst, $src\t# vector (8S)" %}
15292   ins_encode %{
15293     __ dup(as_FloatRegister($dst$$reg), __ T8H, as_Register($src$$reg));
15294   %}
15295   ins_pipe(vdup_reg_reg128);
15296 %}
15297 
15298 instruct replicate4S_imm(vecD dst, immI con)
15299 %{
15300   predicate(n->as_Vector()->length() == 2 ||
15301             n->as_Vector()->length() == 4);
15302   match(Set dst (ReplicateS con));
15303   ins_cost(INSN_COST);
15304   format %{ "movi  $dst, $con\t# vector(4H)" %}
15305   ins_encode %{
15306     __ mov(as_FloatRegister($dst$$reg), __ T4H, $con$$constant & 0xffff);
15307   %}
15308   ins_pipe(vmovi_reg_imm64);
15309 %}
15310 
15311 instruct replicate8S_imm(vecX dst, immI con)
15312 %{
15313   predicate(n->as_Vector()->length() == 8);
15314   match(Set dst (ReplicateS con));
15315   ins_cost(INSN_COST);
15316   format %{ "movi  $dst, $con\t# vector(8H)" %}
15317   ins_encode %{
15318     __ mov(as_FloatRegister($dst$$reg), __ T8H, $con$$constant & 0xffff);
15319   %}
15320   ins_pipe(vmovi_reg_imm128);
15321 %}
15322 
15323 instruct replicate2I(vecD dst, iRegIorL2I src)
15324 %{
15325   predicate(n->as_Vector()->length() == 2);
15326   match(Set dst (ReplicateI src));
15327   ins_cost(INSN_COST);
15328   format %{ "dup  $dst, $src\t# vector (2I)" %}
15329   ins_encode %{
15330     __ dup(as_FloatRegister($dst$$reg), __ T2S, as_Register($src$$reg));
15331   %}
15332   ins_pipe(vdup_reg_reg64);
15333 %}
15334 
15335 instruct replicate4I(vecX dst, iRegIorL2I src)
15336 %{
15337   predicate(n->as_Vector()->length() == 4);
15338   match(Set dst (ReplicateI src));
15339   ins_cost(INSN_COST);
15340   format %{ "dup  $dst, $src\t# vector (4I)" %}
15341   ins_encode %{
15342     __ dup(as_FloatRegister($dst$$reg), __ T4S, as_Register($src$$reg));
15343   %}
15344   ins_pipe(vdup_reg_reg128);
15345 %}
15346 
15347 instruct replicate2I_imm(vecD dst, immI con)
15348 %{
15349   predicate(n->as_Vector()->length() == 2);
15350   match(Set dst (ReplicateI con));
15351   ins_cost(INSN_COST);
15352   format %{ "movi  $dst, $con\t# vector(2I)" %}
15353   ins_encode %{
15354     __ mov(as_FloatRegister($dst$$reg), __ T2S, $con$$constant);
15355   %}
15356   ins_pipe(vmovi_reg_imm64);
15357 %}
15358 
15359 instruct replicate4I_imm(vecX dst, immI con)
15360 %{
15361   predicate(n->as_Vector()->length() == 4);
15362   match(Set dst (ReplicateI con));
15363   ins_cost(INSN_COST);
15364   format %{ "movi  $dst, $con\t# vector(4I)" %}
15365   ins_encode %{
15366     __ mov(as_FloatRegister($dst$$reg), __ T4S, $con$$constant);
15367   %}
15368   ins_pipe(vmovi_reg_imm128);
15369 %}
15370 
15371 instruct replicate2L(vecX dst, iRegL src)
15372 %{
15373   predicate(n->as_Vector()->length() == 2);
15374   match(Set dst (ReplicateL src));
15375   ins_cost(INSN_COST);
15376   format %{ "dup  $dst, $src\t# vector (2L)" %}
15377   ins_encode %{
15378     __ dup(as_FloatRegister($dst$$reg), __ T2D, as_Register($src$$reg));
15379   %}
15380   ins_pipe(vdup_reg_reg128);
15381 %}
15382 
15383 instruct replicate2L_zero(vecX dst, immI0 zero)
15384 %{
15385   predicate(n->as_Vector()->length() == 2);
15386   match(Set dst (ReplicateI zero));
15387   ins_cost(INSN_COST);
15388   format %{ "movi  $dst, $zero\t# vector(4I)" %}
15389   ins_encode %{
15390     __ eor(as_FloatRegister($dst$$reg), __ T16B,
15391            as_FloatRegister($dst$$reg),
15392            as_FloatRegister($dst$$reg));
15393   %}
15394   ins_pipe(vmovi_reg_imm128);
15395 %}
15396 
15397 instruct replicate2F(vecD dst, vRegF src)
15398 %{
15399   predicate(n->as_Vector()->length() == 2);
15400   match(Set dst (ReplicateF src));
15401   ins_cost(INSN_COST);
15402   format %{ "dup  $dst, $src\t# vector (2F)" %}
15403   ins_encode %{
15404     __ dup(as_FloatRegister($dst$$reg), __ T2S,
15405            as_FloatRegister($src$$reg));
15406   %}
15407   ins_pipe(vdup_reg_freg64);
15408 %}
15409 
15410 instruct replicate4F(vecX dst, vRegF src)
15411 %{
15412   predicate(n->as_Vector()->length() == 4);
15413   match(Set dst (ReplicateF src));
15414   ins_cost(INSN_COST);
15415   format %{ "dup  $dst, $src\t# vector (4F)" %}
15416   ins_encode %{
15417     __ dup(as_FloatRegister($dst$$reg), __ T4S,
15418            as_FloatRegister($src$$reg));
15419   %}
15420   ins_pipe(vdup_reg_freg128);
15421 %}
15422 
15423 instruct replicate2D(vecX dst, vRegD src)
15424 %{
15425   predicate(n->as_Vector()->length() == 2);
15426   match(Set dst (ReplicateD src));
15427   ins_cost(INSN_COST);
15428   format %{ "dup  $dst, $src\t# vector (2D)" %}
15429   ins_encode %{
15430     __ dup(as_FloatRegister($dst$$reg), __ T2D,
15431            as_FloatRegister($src$$reg));
15432   %}
15433   ins_pipe(vdup_reg_dreg128);
15434 %}
15435 
15436 // ====================REDUCTION ARITHMETIC====================================
15437 
15438 instruct reduce_add2I(iRegINoSp dst, iRegIorL2I src1, vecD src2, iRegINoSp tmp, iRegINoSp tmp2)
15439 %{
15440   match(Set dst (AddReductionVI src1 src2));
15441   ins_cost(INSN_COST);
15442   effect(TEMP tmp, TEMP tmp2);
15443   format %{ "umov  $tmp, $src2, S, 0\n\t"
15444             "umov  $tmp2, $src2, S, 1\n\t"
15445             "addw  $dst, $src1, $tmp\n\t"
15446             "addw  $dst, $dst, $tmp2\t add reduction2i"
15447   %}
15448   ins_encode %{
15449     __ umov($tmp$$Register, as_FloatRegister($src2$$reg), __ S, 0);
15450     __ umov($tmp2$$Register, as_FloatRegister($src2$$reg), __ S, 1);
15451     __ addw($dst$$Register, $src1$$Register, $tmp$$Register);
15452     __ addw($dst$$Register, $dst$$Register, $tmp2$$Register);
15453   %}
15454   ins_pipe(pipe_class_default);
15455 %}
15456 
15457 instruct reduce_add4I(iRegINoSp dst, iRegIorL2I src1, vecX src2, vecX tmp, iRegINoSp tmp2)
15458 %{
15459   match(Set dst (AddReductionVI src1 src2));
15460   ins_cost(INSN_COST);
15461   effect(TEMP tmp, TEMP tmp2);
15462   format %{ "addv  $tmp, T4S, $src2\n\t"
15463             "umov  $tmp2, $tmp, S, 0\n\t"
15464             "addw  $dst, $tmp2, $src1\t add reduction4i"
15465   %}
15466   ins_encode %{
15467     __ addv(as_FloatRegister($tmp$$reg), __ T4S,
15468             as_FloatRegister($src2$$reg));
15469     __ umov($tmp2$$Register, as_FloatRegister($tmp$$reg), __ S, 0);
15470     __ addw($dst$$Register, $tmp2$$Register, $src1$$Register);
15471   %}
15472   ins_pipe(pipe_class_default);
15473 %}
15474 
15475 instruct reduce_mul2I(iRegINoSp dst, iRegIorL2I src1, vecD src2, iRegINoSp tmp)
15476 %{
15477   match(Set dst (MulReductionVI src1 src2));
15478   ins_cost(INSN_COST);
15479   effect(TEMP tmp, TEMP dst);
15480   format %{ "umov  $tmp, $src2, S, 0\n\t"
15481             "mul   $dst, $tmp, $src1\n\t"
15482             "umov  $tmp, $src2, S, 1\n\t"
15483             "mul   $dst, $tmp, $dst\t mul reduction2i\n\t"
15484   %}
15485   ins_encode %{
15486     __ umov($tmp$$Register, as_FloatRegister($src2$$reg), __ S, 0);
15487     __ mul($dst$$Register, $tmp$$Register, $src1$$Register);
15488     __ umov($tmp$$Register, as_FloatRegister($src2$$reg), __ S, 1);
15489     __ mul($dst$$Register, $tmp$$Register, $dst$$Register);
15490   %}
15491   ins_pipe(pipe_class_default);
15492 %}
15493 
15494 instruct reduce_mul4I(iRegINoSp dst, iRegIorL2I src1, vecX src2, vecX tmp, iRegINoSp tmp2)
15495 %{
15496   match(Set dst (MulReductionVI src1 src2));
15497   ins_cost(INSN_COST);
15498   effect(TEMP tmp, TEMP tmp2, TEMP dst);
15499   format %{ "ins   $tmp, $src2, 0, 1\n\t"
15500             "mul   $tmp, $tmp, $src2\n\t"
15501             "umov  $tmp2, $tmp, S, 0\n\t"
15502             "mul   $dst, $tmp2, $src1\n\t"
15503             "umov  $tmp2, $tmp, S, 1\n\t"
15504             "mul   $dst, $tmp2, $dst\t mul reduction4i\n\t"
15505   %}
15506   ins_encode %{
15507     __ ins(as_FloatRegister($tmp$$reg), __ D,
15508            as_FloatRegister($src2$$reg), 0, 1);
15509     __ mulv(as_FloatRegister($tmp$$reg), __ T2S,
15510            as_FloatRegister($tmp$$reg), as_FloatRegister($src2$$reg));
15511     __ umov($tmp2$$Register, as_FloatRegister($tmp$$reg), __ S, 0);
15512     __ mul($dst$$Register, $tmp2$$Register, $src1$$Register);
15513     __ umov($tmp2$$Register, as_FloatRegister($tmp$$reg), __ S, 1);
15514     __ mul($dst$$Register, $tmp2$$Register, $dst$$Register);
15515   %}
15516   ins_pipe(pipe_class_default);
15517 %}
15518 
15519 instruct reduce_add2F(vRegF dst, vRegF src1, vecD src2, vecD tmp)
15520 %{
15521   match(Set dst (AddReductionVF src1 src2));
15522   ins_cost(INSN_COST);
15523   effect(TEMP tmp, TEMP dst);
15524   format %{ "fadds $dst, $src1, $src2\n\t"
15525             "ins   $tmp, S, $src2, 0, 1\n\t"
15526             "fadds $dst, $dst, $tmp\t add reduction2f"
15527   %}
15528   ins_encode %{
15529     __ fadds(as_FloatRegister($dst$$reg),
15530              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15531     __ ins(as_FloatRegister($tmp$$reg), __ S,
15532            as_FloatRegister($src2$$reg), 0, 1);
15533     __ fadds(as_FloatRegister($dst$$reg),
15534              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15535   %}
15536   ins_pipe(pipe_class_default);
15537 %}
15538 
15539 instruct reduce_add4F(vRegF dst, vRegF src1, vecX src2, vecX tmp)
15540 %{
15541   match(Set dst (AddReductionVF src1 src2));
15542   ins_cost(INSN_COST);
15543   effect(TEMP tmp, TEMP dst);
15544   format %{ "fadds $dst, $src1, $src2\n\t"
15545             "ins   $tmp, S, $src2, 0, 1\n\t"
15546             "fadds $dst, $dst, $tmp\n\t"
15547             "ins   $tmp, S, $src2, 0, 2\n\t"
15548             "fadds $dst, $dst, $tmp\n\t"
15549             "ins   $tmp, S, $src2, 0, 3\n\t"
15550             "fadds $dst, $dst, $tmp\t add reduction4f"
15551   %}
15552   ins_encode %{
15553     __ fadds(as_FloatRegister($dst$$reg),
15554              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15555     __ ins(as_FloatRegister($tmp$$reg), __ S,
15556            as_FloatRegister($src2$$reg), 0, 1);
15557     __ fadds(as_FloatRegister($dst$$reg),
15558              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15559     __ ins(as_FloatRegister($tmp$$reg), __ S,
15560            as_FloatRegister($src2$$reg), 0, 2);
15561     __ fadds(as_FloatRegister($dst$$reg),
15562              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15563     __ ins(as_FloatRegister($tmp$$reg), __ S,
15564            as_FloatRegister($src2$$reg), 0, 3);
15565     __ fadds(as_FloatRegister($dst$$reg),
15566              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15567   %}
15568   ins_pipe(pipe_class_default);
15569 %}
15570 
15571 instruct reduce_mul2F(vRegF dst, vRegF src1, vecD src2, vecD tmp)
15572 %{
15573   match(Set dst (MulReductionVF src1 src2));
15574   ins_cost(INSN_COST);
15575   effect(TEMP tmp, TEMP dst);
15576   format %{ "fmuls $dst, $src1, $src2\n\t"
15577             "ins   $tmp, S, $src2, 0, 1\n\t"
15578             "fmuls $dst, $dst, $tmp\t add reduction4f"
15579   %}
15580   ins_encode %{
15581     __ fmuls(as_FloatRegister($dst$$reg),
15582              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15583     __ ins(as_FloatRegister($tmp$$reg), __ S,
15584            as_FloatRegister($src2$$reg), 0, 1);
15585     __ fmuls(as_FloatRegister($dst$$reg),
15586              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15587   %}
15588   ins_pipe(pipe_class_default);
15589 %}
15590 
15591 instruct reduce_mul4F(vRegF dst, vRegF src1, vecX src2, vecX tmp)
15592 %{
15593   match(Set dst (MulReductionVF src1 src2));
15594   ins_cost(INSN_COST);
15595   effect(TEMP tmp, TEMP dst);
15596   format %{ "fmuls $dst, $src1, $src2\n\t"
15597             "ins   $tmp, S, $src2, 0, 1\n\t"
15598             "fmuls $dst, $dst, $tmp\n\t"
15599             "ins   $tmp, S, $src2, 0, 2\n\t"
15600             "fmuls $dst, $dst, $tmp\n\t"
15601             "ins   $tmp, S, $src2, 0, 3\n\t"
15602             "fmuls $dst, $dst, $tmp\t add reduction4f"
15603   %}
15604   ins_encode %{
15605     __ fmuls(as_FloatRegister($dst$$reg),
15606              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15607     __ ins(as_FloatRegister($tmp$$reg), __ S,
15608            as_FloatRegister($src2$$reg), 0, 1);
15609     __ fmuls(as_FloatRegister($dst$$reg),
15610              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15611     __ ins(as_FloatRegister($tmp$$reg), __ S,
15612            as_FloatRegister($src2$$reg), 0, 2);
15613     __ fmuls(as_FloatRegister($dst$$reg),
15614              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15615     __ ins(as_FloatRegister($tmp$$reg), __ S,
15616            as_FloatRegister($src2$$reg), 0, 3);
15617     __ fmuls(as_FloatRegister($dst$$reg),
15618              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15619   %}
15620   ins_pipe(pipe_class_default);
15621 %}
15622 
15623 instruct reduce_add2D(vRegD dst, vRegD src1, vecX src2, vecX tmp)
15624 %{
15625   match(Set dst (AddReductionVD src1 src2));
15626   ins_cost(INSN_COST);
15627   effect(TEMP tmp, TEMP dst);
15628   format %{ "faddd $dst, $src1, $src2\n\t"
15629             "ins   $tmp, D, $src2, 0, 1\n\t"
15630             "faddd $dst, $dst, $tmp\t add reduction2d"
15631   %}
15632   ins_encode %{
15633     __ faddd(as_FloatRegister($dst$$reg),
15634              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15635     __ ins(as_FloatRegister($tmp$$reg), __ D,
15636            as_FloatRegister($src2$$reg), 0, 1);
15637     __ faddd(as_FloatRegister($dst$$reg),
15638              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15639   %}
15640   ins_pipe(pipe_class_default);
15641 %}
15642 
15643 instruct reduce_mul2D(vRegD dst, vRegD src1, vecX src2, vecX tmp)
15644 %{
15645   match(Set dst (MulReductionVD src1 src2));
15646   ins_cost(INSN_COST);
15647   effect(TEMP tmp, TEMP dst);
15648   format %{ "fmuld $dst, $src1, $src2\n\t"
15649             "ins   $tmp, D, $src2, 0, 1\n\t"
15650             "fmuld $dst, $dst, $tmp\t add reduction2d"
15651   %}
15652   ins_encode %{
15653     __ fmuld(as_FloatRegister($dst$$reg),
15654              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15655     __ ins(as_FloatRegister($tmp$$reg), __ D,
15656            as_FloatRegister($src2$$reg), 0, 1);
15657     __ fmuld(as_FloatRegister($dst$$reg),
15658              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15659   %}
15660   ins_pipe(pipe_class_default);
15661 %}
15662 
15663 instruct reduce_max2F(vRegF dst, vRegF src1, vecD src2, vecD tmp) %{
15664   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
15665   match(Set dst (MaxReductionV src1 src2));
15666   ins_cost(INSN_COST);
15667   effect(TEMP_DEF dst, TEMP tmp);
15668   format %{ "fmaxs $dst, $src1, $src2\n\t"
15669             "ins   $tmp, S, $src2, 0, 1\n\t"
15670             "fmaxs $dst, $dst, $tmp\t max reduction2F" %}
15671   ins_encode %{
15672     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15673     __ ins(as_FloatRegister($tmp$$reg), __ S, as_FloatRegister($src2$$reg), 0, 1);
15674     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15675   %}
15676   ins_pipe(pipe_class_default);
15677 %}
15678 
15679 instruct reduce_max4F(vRegF dst, vRegF src1, vecX src2) %{
15680   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
15681   match(Set dst (MaxReductionV src1 src2));
15682   ins_cost(INSN_COST);
15683   effect(TEMP_DEF dst);
15684   format %{ "fmaxv $dst, T4S, $src2\n\t"
15685             "fmaxs $dst, $dst, $src1\t max reduction4F" %}
15686   ins_encode %{
15687     __ fmaxv(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($src2$$reg));
15688     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
15689   %}
15690   ins_pipe(pipe_class_default);
15691 %}
15692 
15693 instruct reduce_max2D(vRegD dst, vRegD src1, vecX src2, vecX tmp) %{
15694   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
15695   match(Set dst (MaxReductionV src1 src2));
15696   ins_cost(INSN_COST);
15697   effect(TEMP_DEF dst, TEMP tmp);
15698   format %{ "fmaxd $dst, $src1, $src2\n\t"
15699             "ins   $tmp, D, $src2, 0, 1\n\t"
15700             "fmaxd $dst, $dst, $tmp\t max reduction2D" %}
15701   ins_encode %{
15702     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15703     __ ins(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($src2$$reg), 0, 1);
15704     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15705   %}
15706   ins_pipe(pipe_class_default);
15707 %}
15708 
15709 instruct reduce_min2F(vRegF dst, vRegF src1, vecD src2, vecD tmp) %{
15710   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
15711   match(Set dst (MinReductionV src1 src2));
15712   ins_cost(INSN_COST);
15713   effect(TEMP_DEF dst, TEMP tmp);
15714   format %{ "fmins $dst, $src1, $src2\n\t"
15715             "ins   $tmp, S, $src2, 0, 1\n\t"
15716             "fmins $dst, $dst, $tmp\t min reduction2F" %}
15717   ins_encode %{
15718     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15719     __ ins(as_FloatRegister($tmp$$reg), __ S, as_FloatRegister($src2$$reg), 0, 1);
15720     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15721   %}
15722   ins_pipe(pipe_class_default);
15723 %}
15724 
15725 instruct reduce_min4F(vRegF dst, vRegF src1, vecX src2) %{
15726   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
15727   match(Set dst (MinReductionV src1 src2));
15728   ins_cost(INSN_COST);
15729   effect(TEMP_DEF dst);
15730   format %{ "fminv $dst, T4S, $src2\n\t"
15731             "fmins $dst, $dst, $src1\t min reduction4F" %}
15732   ins_encode %{
15733     __ fminv(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($src2$$reg));
15734     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
15735   %}
15736   ins_pipe(pipe_class_default);
15737 %}
15738 
15739 instruct reduce_min2D(vRegD dst, vRegD src1, vecX src2, vecX tmp) %{
15740   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
15741   match(Set dst (MinReductionV src1 src2));
15742   ins_cost(INSN_COST);
15743   effect(TEMP_DEF dst, TEMP tmp);
15744   format %{ "fmind $dst, $src1, $src2\n\t"
15745             "ins   $tmp, D, $src2, 0, 1\n\t"
15746             "fmind $dst, $dst, $tmp\t min reduction2D" %}
15747   ins_encode %{
15748     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15749     __ ins(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($src2$$reg), 0, 1);
15750     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15751   %}
15752   ins_pipe(pipe_class_default);
15753 %}
15754 
15755 // ====================VECTOR ARITHMETIC=======================================
15756 
15757 // --------------------------------- ADD --------------------------------------
15758 
15759 instruct vadd8B(vecD dst, vecD src1, vecD src2)
15760 %{
15761   predicate(n->as_Vector()->length() == 4 ||
15762             n->as_Vector()->length() == 8);
15763   match(Set dst (AddVB src1 src2));
15764   ins_cost(INSN_COST);
15765   format %{ "addv  $dst,$src1,$src2\t# vector (8B)" %}
15766   ins_encode %{
15767     __ addv(as_FloatRegister($dst$$reg), __ T8B,
15768             as_FloatRegister($src1$$reg),
15769             as_FloatRegister($src2$$reg));
15770   %}
15771   ins_pipe(vdop64);
15772 %}
15773 
15774 instruct vadd16B(vecX dst, vecX src1, vecX src2)
15775 %{
15776   predicate(n->as_Vector()->length() == 16);
15777   match(Set dst (AddVB src1 src2));
15778   ins_cost(INSN_COST);
15779   format %{ "addv  $dst,$src1,$src2\t# vector (16B)" %}
15780   ins_encode %{
15781     __ addv(as_FloatRegister($dst$$reg), __ T16B,
15782             as_FloatRegister($src1$$reg),
15783             as_FloatRegister($src2$$reg));
15784   %}
15785   ins_pipe(vdop128);
15786 %}
15787 
15788 instruct vadd4S(vecD dst, vecD src1, vecD src2)
15789 %{
15790   predicate(n->as_Vector()->length() == 2 ||
15791             n->as_Vector()->length() == 4);
15792   match(Set dst (AddVS src1 src2));
15793   ins_cost(INSN_COST);
15794   format %{ "addv  $dst,$src1,$src2\t# vector (4H)" %}
15795   ins_encode %{
15796     __ addv(as_FloatRegister($dst$$reg), __ T4H,
15797             as_FloatRegister($src1$$reg),
15798             as_FloatRegister($src2$$reg));
15799   %}
15800   ins_pipe(vdop64);
15801 %}
15802 
15803 instruct vadd8S(vecX dst, vecX src1, vecX src2)
15804 %{
15805   predicate(n->as_Vector()->length() == 8);
15806   match(Set dst (AddVS src1 src2));
15807   ins_cost(INSN_COST);
15808   format %{ "addv  $dst,$src1,$src2\t# vector (8H)" %}
15809   ins_encode %{
15810     __ addv(as_FloatRegister($dst$$reg), __ T8H,
15811             as_FloatRegister($src1$$reg),
15812             as_FloatRegister($src2$$reg));
15813   %}
15814   ins_pipe(vdop128);
15815 %}
15816 
15817 instruct vadd2I(vecD dst, vecD src1, vecD src2)
15818 %{
15819   predicate(n->as_Vector()->length() == 2);
15820   match(Set dst (AddVI src1 src2));
15821   ins_cost(INSN_COST);
15822   format %{ "addv  $dst,$src1,$src2\t# vector (2S)" %}
15823   ins_encode %{
15824     __ addv(as_FloatRegister($dst$$reg), __ T2S,
15825             as_FloatRegister($src1$$reg),
15826             as_FloatRegister($src2$$reg));
15827   %}
15828   ins_pipe(vdop64);
15829 %}
15830 
15831 instruct vadd4I(vecX dst, vecX src1, vecX src2)
15832 %{
15833   predicate(n->as_Vector()->length() == 4);
15834   match(Set dst (AddVI src1 src2));
15835   ins_cost(INSN_COST);
15836   format %{ "addv  $dst,$src1,$src2\t# vector (4S)" %}
15837   ins_encode %{
15838     __ addv(as_FloatRegister($dst$$reg), __ T4S,
15839             as_FloatRegister($src1$$reg),
15840             as_FloatRegister($src2$$reg));
15841   %}
15842   ins_pipe(vdop128);
15843 %}
15844 
15845 instruct vadd2L(vecX dst, vecX src1, vecX src2)
15846 %{
15847   predicate(n->as_Vector()->length() == 2);
15848   match(Set dst (AddVL src1 src2));
15849   ins_cost(INSN_COST);
15850   format %{ "addv  $dst,$src1,$src2\t# vector (2L)" %}
15851   ins_encode %{
15852     __ addv(as_FloatRegister($dst$$reg), __ T2D,
15853             as_FloatRegister($src1$$reg),
15854             as_FloatRegister($src2$$reg));
15855   %}
15856   ins_pipe(vdop128);
15857 %}
15858 
15859 instruct vadd2F(vecD dst, vecD src1, vecD src2)
15860 %{
15861   predicate(n->as_Vector()->length() == 2);
15862   match(Set dst (AddVF src1 src2));
15863   ins_cost(INSN_COST);
15864   format %{ "fadd  $dst,$src1,$src2\t# vector (2S)" %}
15865   ins_encode %{
15866     __ fadd(as_FloatRegister($dst$$reg), __ T2S,
15867             as_FloatRegister($src1$$reg),
15868             as_FloatRegister($src2$$reg));
15869   %}
15870   ins_pipe(vdop_fp64);
15871 %}
15872 
15873 instruct vadd4F(vecX dst, vecX src1, vecX src2)
15874 %{
15875   predicate(n->as_Vector()->length() == 4);
15876   match(Set dst (AddVF src1 src2));
15877   ins_cost(INSN_COST);
15878   format %{ "fadd  $dst,$src1,$src2\t# vector (4S)" %}
15879   ins_encode %{
15880     __ fadd(as_FloatRegister($dst$$reg), __ T4S,
15881             as_FloatRegister($src1$$reg),
15882             as_FloatRegister($src2$$reg));
15883   %}
15884   ins_pipe(vdop_fp128);
15885 %}
15886 
15887 instruct vadd2D(vecX dst, vecX src1, vecX src2)
15888 %{
15889   match(Set dst (AddVD src1 src2));
15890   ins_cost(INSN_COST);
15891   format %{ "fadd  $dst,$src1,$src2\t# vector (2D)" %}
15892   ins_encode %{
15893     __ fadd(as_FloatRegister($dst$$reg), __ T2D,
15894             as_FloatRegister($src1$$reg),
15895             as_FloatRegister($src2$$reg));
15896   %}
15897   ins_pipe(vdop_fp128);
15898 %}
15899 
15900 // --------------------------------- SUB --------------------------------------
15901 
15902 instruct vsub8B(vecD dst, vecD src1, vecD src2)
15903 %{
15904   predicate(n->as_Vector()->length() == 4 ||
15905             n->as_Vector()->length() == 8);
15906   match(Set dst (SubVB src1 src2));
15907   ins_cost(INSN_COST);
15908   format %{ "subv  $dst,$src1,$src2\t# vector (8B)" %}
15909   ins_encode %{
15910     __ subv(as_FloatRegister($dst$$reg), __ T8B,
15911             as_FloatRegister($src1$$reg),
15912             as_FloatRegister($src2$$reg));
15913   %}
15914   ins_pipe(vdop64);
15915 %}
15916 
15917 instruct vsub16B(vecX dst, vecX src1, vecX src2)
15918 %{
15919   predicate(n->as_Vector()->length() == 16);
15920   match(Set dst (SubVB src1 src2));
15921   ins_cost(INSN_COST);
15922   format %{ "subv  $dst,$src1,$src2\t# vector (16B)" %}
15923   ins_encode %{
15924     __ subv(as_FloatRegister($dst$$reg), __ T16B,
15925             as_FloatRegister($src1$$reg),
15926             as_FloatRegister($src2$$reg));
15927   %}
15928   ins_pipe(vdop128);
15929 %}
15930 
15931 instruct vsub4S(vecD dst, vecD src1, vecD src2)
15932 %{
15933   predicate(n->as_Vector()->length() == 2 ||
15934             n->as_Vector()->length() == 4);
15935   match(Set dst (SubVS src1 src2));
15936   ins_cost(INSN_COST);
15937   format %{ "subv  $dst,$src1,$src2\t# vector (4H)" %}
15938   ins_encode %{
15939     __ subv(as_FloatRegister($dst$$reg), __ T4H,
15940             as_FloatRegister($src1$$reg),
15941             as_FloatRegister($src2$$reg));
15942   %}
15943   ins_pipe(vdop64);
15944 %}
15945 
15946 instruct vsub8S(vecX dst, vecX src1, vecX src2)
15947 %{
15948   predicate(n->as_Vector()->length() == 8);
15949   match(Set dst (SubVS src1 src2));
15950   ins_cost(INSN_COST);
15951   format %{ "subv  $dst,$src1,$src2\t# vector (8H)" %}
15952   ins_encode %{
15953     __ subv(as_FloatRegister($dst$$reg), __ T8H,
15954             as_FloatRegister($src1$$reg),
15955             as_FloatRegister($src2$$reg));
15956   %}
15957   ins_pipe(vdop128);
15958 %}
15959 
15960 instruct vsub2I(vecD dst, vecD src1, vecD src2)
15961 %{
15962   predicate(n->as_Vector()->length() == 2);
15963   match(Set dst (SubVI src1 src2));
15964   ins_cost(INSN_COST);
15965   format %{ "subv  $dst,$src1,$src2\t# vector (2S)" %}
15966   ins_encode %{
15967     __ subv(as_FloatRegister($dst$$reg), __ T2S,
15968             as_FloatRegister($src1$$reg),
15969             as_FloatRegister($src2$$reg));
15970   %}
15971   ins_pipe(vdop64);
15972 %}
15973 
15974 instruct vsub4I(vecX dst, vecX src1, vecX src2)
15975 %{
15976   predicate(n->as_Vector()->length() == 4);
15977   match(Set dst (SubVI src1 src2));
15978   ins_cost(INSN_COST);
15979   format %{ "subv  $dst,$src1,$src2\t# vector (4S)" %}
15980   ins_encode %{
15981     __ subv(as_FloatRegister($dst$$reg), __ T4S,
15982             as_FloatRegister($src1$$reg),
15983             as_FloatRegister($src2$$reg));
15984   %}
15985   ins_pipe(vdop128);
15986 %}
15987 
15988 instruct vsub2L(vecX dst, vecX src1, vecX src2)
15989 %{
15990   predicate(n->as_Vector()->length() == 2);
15991   match(Set dst (SubVL src1 src2));
15992   ins_cost(INSN_COST);
15993   format %{ "subv  $dst,$src1,$src2\t# vector (2L)" %}
15994   ins_encode %{
15995     __ subv(as_FloatRegister($dst$$reg), __ T2D,
15996             as_FloatRegister($src1$$reg),
15997             as_FloatRegister($src2$$reg));
15998   %}
15999   ins_pipe(vdop128);
16000 %}
16001 
16002 instruct vsub2F(vecD dst, vecD src1, vecD src2)
16003 %{
16004   predicate(n->as_Vector()->length() == 2);
16005   match(Set dst (SubVF src1 src2));
16006   ins_cost(INSN_COST);
16007   format %{ "fsub  $dst,$src1,$src2\t# vector (2S)" %}
16008   ins_encode %{
16009     __ fsub(as_FloatRegister($dst$$reg), __ T2S,
16010             as_FloatRegister($src1$$reg),
16011             as_FloatRegister($src2$$reg));
16012   %}
16013   ins_pipe(vdop_fp64);
16014 %}
16015 
16016 instruct vsub4F(vecX dst, vecX src1, vecX src2)
16017 %{
16018   predicate(n->as_Vector()->length() == 4);
16019   match(Set dst (SubVF src1 src2));
16020   ins_cost(INSN_COST);
16021   format %{ "fsub  $dst,$src1,$src2\t# vector (4S)" %}
16022   ins_encode %{
16023     __ fsub(as_FloatRegister($dst$$reg), __ T4S,
16024             as_FloatRegister($src1$$reg),
16025             as_FloatRegister($src2$$reg));
16026   %}
16027   ins_pipe(vdop_fp128);
16028 %}
16029 
16030 instruct vsub2D(vecX dst, vecX src1, vecX src2)
16031 %{
16032   predicate(n->as_Vector()->length() == 2);
16033   match(Set dst (SubVD src1 src2));
16034   ins_cost(INSN_COST);
16035   format %{ "fsub  $dst,$src1,$src2\t# vector (2D)" %}
16036   ins_encode %{
16037     __ fsub(as_FloatRegister($dst$$reg), __ T2D,
16038             as_FloatRegister($src1$$reg),
16039             as_FloatRegister($src2$$reg));
16040   %}
16041   ins_pipe(vdop_fp128);
16042 %}
16043 
16044 // --------------------------------- MUL --------------------------------------
16045 
16046 instruct vmul4S(vecD dst, vecD src1, vecD src2)
16047 %{
16048   predicate(n->as_Vector()->length() == 2 ||
16049             n->as_Vector()->length() == 4);
16050   match(Set dst (MulVS src1 src2));
16051   ins_cost(INSN_COST);
16052   format %{ "mulv  $dst,$src1,$src2\t# vector (4H)" %}
16053   ins_encode %{
16054     __ mulv(as_FloatRegister($dst$$reg), __ T4H,
16055             as_FloatRegister($src1$$reg),
16056             as_FloatRegister($src2$$reg));
16057   %}
16058   ins_pipe(vmul64);
16059 %}
16060 
16061 instruct vmul8S(vecX dst, vecX src1, vecX src2)
16062 %{
16063   predicate(n->as_Vector()->length() == 8);
16064   match(Set dst (MulVS src1 src2));
16065   ins_cost(INSN_COST);
16066   format %{ "mulv  $dst,$src1,$src2\t# vector (8H)" %}
16067   ins_encode %{
16068     __ mulv(as_FloatRegister($dst$$reg), __ T8H,
16069             as_FloatRegister($src1$$reg),
16070             as_FloatRegister($src2$$reg));
16071   %}
16072   ins_pipe(vmul128);
16073 %}
16074 
16075 instruct vmul2I(vecD dst, vecD src1, vecD src2)
16076 %{
16077   predicate(n->as_Vector()->length() == 2);
16078   match(Set dst (MulVI src1 src2));
16079   ins_cost(INSN_COST);
16080   format %{ "mulv  $dst,$src1,$src2\t# vector (2S)" %}
16081   ins_encode %{
16082     __ mulv(as_FloatRegister($dst$$reg), __ T2S,
16083             as_FloatRegister($src1$$reg),
16084             as_FloatRegister($src2$$reg));
16085   %}
16086   ins_pipe(vmul64);
16087 %}
16088 
16089 instruct vmul4I(vecX dst, vecX src1, vecX src2)
16090 %{
16091   predicate(n->as_Vector()->length() == 4);
16092   match(Set dst (MulVI src1 src2));
16093   ins_cost(INSN_COST);
16094   format %{ "mulv  $dst,$src1,$src2\t# vector (4S)" %}
16095   ins_encode %{
16096     __ mulv(as_FloatRegister($dst$$reg), __ T4S,
16097             as_FloatRegister($src1$$reg),
16098             as_FloatRegister($src2$$reg));
16099   %}
16100   ins_pipe(vmul128);
16101 %}
16102 
16103 instruct vmul2F(vecD dst, vecD src1, vecD src2)
16104 %{
16105   predicate(n->as_Vector()->length() == 2);
16106   match(Set dst (MulVF src1 src2));
16107   ins_cost(INSN_COST);
16108   format %{ "fmul  $dst,$src1,$src2\t# vector (2S)" %}
16109   ins_encode %{
16110     __ fmul(as_FloatRegister($dst$$reg), __ T2S,
16111             as_FloatRegister($src1$$reg),
16112             as_FloatRegister($src2$$reg));
16113   %}
16114   ins_pipe(vmuldiv_fp64);
16115 %}
16116 
16117 instruct vmul4F(vecX dst, vecX src1, vecX src2)
16118 %{
16119   predicate(n->as_Vector()->length() == 4);
16120   match(Set dst (MulVF src1 src2));
16121   ins_cost(INSN_COST);
16122   format %{ "fmul  $dst,$src1,$src2\t# vector (4S)" %}
16123   ins_encode %{
16124     __ fmul(as_FloatRegister($dst$$reg), __ T4S,
16125             as_FloatRegister($src1$$reg),
16126             as_FloatRegister($src2$$reg));
16127   %}
16128   ins_pipe(vmuldiv_fp128);
16129 %}
16130 
16131 instruct vmul2D(vecX dst, vecX src1, vecX src2)
16132 %{
16133   predicate(n->as_Vector()->length() == 2);
16134   match(Set dst (MulVD src1 src2));
16135   ins_cost(INSN_COST);
16136   format %{ "fmul  $dst,$src1,$src2\t# vector (2D)" %}
16137   ins_encode %{
16138     __ fmul(as_FloatRegister($dst$$reg), __ T2D,
16139             as_FloatRegister($src1$$reg),
16140             as_FloatRegister($src2$$reg));
16141   %}
16142   ins_pipe(vmuldiv_fp128);
16143 %}
16144 
16145 // --------------------------------- MLA --------------------------------------
16146 
16147 instruct vmla4S(vecD dst, vecD src1, vecD src2)
16148 %{
16149   predicate(n->as_Vector()->length() == 2 ||
16150             n->as_Vector()->length() == 4);
16151   match(Set dst (AddVS dst (MulVS src1 src2)));
16152   ins_cost(INSN_COST);
16153   format %{ "mlav  $dst,$src1,$src2\t# vector (4H)" %}
16154   ins_encode %{
16155     __ mlav(as_FloatRegister($dst$$reg), __ T4H,
16156             as_FloatRegister($src1$$reg),
16157             as_FloatRegister($src2$$reg));
16158   %}
16159   ins_pipe(vmla64);
16160 %}
16161 
16162 instruct vmla8S(vecX dst, vecX src1, vecX src2)
16163 %{
16164   predicate(n->as_Vector()->length() == 8);
16165   match(Set dst (AddVS dst (MulVS src1 src2)));
16166   ins_cost(INSN_COST);
16167   format %{ "mlav  $dst,$src1,$src2\t# vector (8H)" %}
16168   ins_encode %{
16169     __ mlav(as_FloatRegister($dst$$reg), __ T8H,
16170             as_FloatRegister($src1$$reg),
16171             as_FloatRegister($src2$$reg));
16172   %}
16173   ins_pipe(vmla128);
16174 %}
16175 
16176 instruct vmla2I(vecD dst, vecD src1, vecD src2)
16177 %{
16178   predicate(n->as_Vector()->length() == 2);
16179   match(Set dst (AddVI dst (MulVI src1 src2)));
16180   ins_cost(INSN_COST);
16181   format %{ "mlav  $dst,$src1,$src2\t# vector (2S)" %}
16182   ins_encode %{
16183     __ mlav(as_FloatRegister($dst$$reg), __ T2S,
16184             as_FloatRegister($src1$$reg),
16185             as_FloatRegister($src2$$reg));
16186   %}
16187   ins_pipe(vmla64);
16188 %}
16189 
16190 instruct vmla4I(vecX dst, vecX src1, vecX src2)
16191 %{
16192   predicate(n->as_Vector()->length() == 4);
16193   match(Set dst (AddVI dst (MulVI src1 src2)));
16194   ins_cost(INSN_COST);
16195   format %{ "mlav  $dst,$src1,$src2\t# vector (4S)" %}
16196   ins_encode %{
16197     __ mlav(as_FloatRegister($dst$$reg), __ T4S,
16198             as_FloatRegister($src1$$reg),
16199             as_FloatRegister($src2$$reg));
16200   %}
16201   ins_pipe(vmla128);
16202 %}
16203 
16204 // dst + src1 * src2
16205 instruct vmla2F(vecD dst, vecD src1, vecD src2) %{
16206   predicate(UseFMA && n->as_Vector()->length() == 2);
16207   match(Set dst (FmaVF  dst (Binary src1 src2)));
16208   format %{ "fmla  $dst,$src1,$src2\t# vector (2S)" %}
16209   ins_cost(INSN_COST);
16210   ins_encode %{
16211     __ fmla(as_FloatRegister($dst$$reg), __ T2S,
16212             as_FloatRegister($src1$$reg),
16213             as_FloatRegister($src2$$reg));
16214   %}
16215   ins_pipe(vmuldiv_fp64);
16216 %}
16217 
16218 // dst + src1 * src2
16219 instruct vmla4F(vecX dst, vecX src1, vecX src2) %{
16220   predicate(UseFMA && n->as_Vector()->length() == 4);
16221   match(Set dst (FmaVF  dst (Binary src1 src2)));
16222   format %{ "fmla  $dst,$src1,$src2\t# vector (4S)" %}
16223   ins_cost(INSN_COST);
16224   ins_encode %{
16225     __ fmla(as_FloatRegister($dst$$reg), __ T4S,
16226             as_FloatRegister($src1$$reg),
16227             as_FloatRegister($src2$$reg));
16228   %}
16229   ins_pipe(vmuldiv_fp128);
16230 %}
16231 
16232 // dst + src1 * src2
16233 instruct vmla2D(vecX dst, vecX src1, vecX src2) %{
16234   predicate(UseFMA && n->as_Vector()->length() == 2);
16235   match(Set dst (FmaVD  dst (Binary src1 src2)));
16236   format %{ "fmla  $dst,$src1,$src2\t# vector (2D)" %}
16237   ins_cost(INSN_COST);
16238   ins_encode %{
16239     __ fmla(as_FloatRegister($dst$$reg), __ T2D,
16240             as_FloatRegister($src1$$reg),
16241             as_FloatRegister($src2$$reg));
16242   %}
16243   ins_pipe(vmuldiv_fp128);
16244 %}
16245 
16246 // --------------------------------- MLS --------------------------------------
16247 
16248 instruct vmls4S(vecD dst, vecD src1, vecD src2)
16249 %{
16250   predicate(n->as_Vector()->length() == 2 ||
16251             n->as_Vector()->length() == 4);
16252   match(Set dst (SubVS dst (MulVS src1 src2)));
16253   ins_cost(INSN_COST);
16254   format %{ "mlsv  $dst,$src1,$src2\t# vector (4H)" %}
16255   ins_encode %{
16256     __ mlsv(as_FloatRegister($dst$$reg), __ T4H,
16257             as_FloatRegister($src1$$reg),
16258             as_FloatRegister($src2$$reg));
16259   %}
16260   ins_pipe(vmla64);
16261 %}
16262 
16263 instruct vmls8S(vecX dst, vecX src1, vecX src2)
16264 %{
16265   predicate(n->as_Vector()->length() == 8);
16266   match(Set dst (SubVS dst (MulVS src1 src2)));
16267   ins_cost(INSN_COST);
16268   format %{ "mlsv  $dst,$src1,$src2\t# vector (8H)" %}
16269   ins_encode %{
16270     __ mlsv(as_FloatRegister($dst$$reg), __ T8H,
16271             as_FloatRegister($src1$$reg),
16272             as_FloatRegister($src2$$reg));
16273   %}
16274   ins_pipe(vmla128);
16275 %}
16276 
16277 instruct vmls2I(vecD dst, vecD src1, vecD src2)
16278 %{
16279   predicate(n->as_Vector()->length() == 2);
16280   match(Set dst (SubVI dst (MulVI src1 src2)));
16281   ins_cost(INSN_COST);
16282   format %{ "mlsv  $dst,$src1,$src2\t# vector (2S)" %}
16283   ins_encode %{
16284     __ mlsv(as_FloatRegister($dst$$reg), __ T2S,
16285             as_FloatRegister($src1$$reg),
16286             as_FloatRegister($src2$$reg));
16287   %}
16288   ins_pipe(vmla64);
16289 %}
16290 
16291 instruct vmls4I(vecX dst, vecX src1, vecX src2)
16292 %{
16293   predicate(n->as_Vector()->length() == 4);
16294   match(Set dst (SubVI dst (MulVI src1 src2)));
16295   ins_cost(INSN_COST);
16296   format %{ "mlsv  $dst,$src1,$src2\t# vector (4S)" %}
16297   ins_encode %{
16298     __ mlsv(as_FloatRegister($dst$$reg), __ T4S,
16299             as_FloatRegister($src1$$reg),
16300             as_FloatRegister($src2$$reg));
16301   %}
16302   ins_pipe(vmla128);
16303 %}
16304 
16305 // dst - src1 * src2
16306 instruct vmls2F(vecD dst, vecD src1, vecD src2) %{
16307   predicate(UseFMA && n->as_Vector()->length() == 2);
16308   match(Set dst (FmaVF  dst (Binary (NegVF src1) src2)));
16309   match(Set dst (FmaVF  dst (Binary src1 (NegVF src2))));
16310   format %{ "fmls  $dst,$src1,$src2\t# vector (2S)" %}
16311   ins_cost(INSN_COST);
16312   ins_encode %{
16313     __ fmls(as_FloatRegister($dst$$reg), __ T2S,
16314             as_FloatRegister($src1$$reg),
16315             as_FloatRegister($src2$$reg));
16316   %}
16317   ins_pipe(vmuldiv_fp64);
16318 %}
16319 
16320 // dst - src1 * src2
16321 instruct vmls4F(vecX dst, vecX src1, vecX src2) %{
16322   predicate(UseFMA && n->as_Vector()->length() == 4);
16323   match(Set dst (FmaVF  dst (Binary (NegVF src1) src2)));
16324   match(Set dst (FmaVF  dst (Binary src1 (NegVF src2))));
16325   format %{ "fmls  $dst,$src1,$src2\t# vector (4S)" %}
16326   ins_cost(INSN_COST);
16327   ins_encode %{
16328     __ fmls(as_FloatRegister($dst$$reg), __ T4S,
16329             as_FloatRegister($src1$$reg),
16330             as_FloatRegister($src2$$reg));
16331   %}
16332   ins_pipe(vmuldiv_fp128);
16333 %}
16334 
16335 // dst - src1 * src2
16336 instruct vmls2D(vecX dst, vecX src1, vecX src2) %{
16337   predicate(UseFMA && n->as_Vector()->length() == 2);
16338   match(Set dst (FmaVD  dst (Binary (NegVD src1) src2)));
16339   match(Set dst (FmaVD  dst (Binary src1 (NegVD src2))));
16340   format %{ "fmls  $dst,$src1,$src2\t# vector (2D)" %}
16341   ins_cost(INSN_COST);
16342   ins_encode %{
16343     __ fmls(as_FloatRegister($dst$$reg), __ T2D,
16344             as_FloatRegister($src1$$reg),
16345             as_FloatRegister($src2$$reg));
16346   %}
16347   ins_pipe(vmuldiv_fp128);
16348 %}
16349 
16350 // --------------------------------- DIV --------------------------------------
16351 
16352 instruct vdiv2F(vecD dst, vecD src1, vecD src2)
16353 %{
16354   predicate(n->as_Vector()->length() == 2);
16355   match(Set dst (DivVF src1 src2));
16356   ins_cost(INSN_COST);
16357   format %{ "fdiv  $dst,$src1,$src2\t# vector (2S)" %}
16358   ins_encode %{
16359     __ fdiv(as_FloatRegister($dst$$reg), __ T2S,
16360             as_FloatRegister($src1$$reg),
16361             as_FloatRegister($src2$$reg));
16362   %}
16363   ins_pipe(vmuldiv_fp64);
16364 %}
16365 
16366 instruct vdiv4F(vecX dst, vecX src1, vecX src2)
16367 %{
16368   predicate(n->as_Vector()->length() == 4);
16369   match(Set dst (DivVF src1 src2));
16370   ins_cost(INSN_COST);
16371   format %{ "fdiv  $dst,$src1,$src2\t# vector (4S)" %}
16372   ins_encode %{
16373     __ fdiv(as_FloatRegister($dst$$reg), __ T4S,
16374             as_FloatRegister($src1$$reg),
16375             as_FloatRegister($src2$$reg));
16376   %}
16377   ins_pipe(vmuldiv_fp128);
16378 %}
16379 
16380 instruct vdiv2D(vecX dst, vecX src1, vecX src2)
16381 %{
16382   predicate(n->as_Vector()->length() == 2);
16383   match(Set dst (DivVD src1 src2));
16384   ins_cost(INSN_COST);
16385   format %{ "fdiv  $dst,$src1,$src2\t# vector (2D)" %}
16386   ins_encode %{
16387     __ fdiv(as_FloatRegister($dst$$reg), __ T2D,
16388             as_FloatRegister($src1$$reg),
16389             as_FloatRegister($src2$$reg));
16390   %}
16391   ins_pipe(vmuldiv_fp128);
16392 %}
16393 
16394 // --------------------------------- SQRT -------------------------------------
16395 
16396 instruct vsqrt2D(vecX dst, vecX src)
16397 %{
16398   predicate(n->as_Vector()->length() == 2);
16399   match(Set dst (SqrtVD src));
16400   format %{ "fsqrt  $dst, $src\t# vector (2D)" %}
16401   ins_encode %{
16402     __ fsqrt(as_FloatRegister($dst$$reg), __ T2D,
16403              as_FloatRegister($src$$reg));
16404   %}
16405   ins_pipe(vsqrt_fp128);
16406 %}
16407 
16408 // --------------------------------- ABS --------------------------------------
16409 
16410 instruct vabs2F(vecD dst, vecD src)
16411 %{
16412   predicate(n->as_Vector()->length() == 2);
16413   match(Set dst (AbsVF src));
16414   ins_cost(INSN_COST * 3);
16415   format %{ "fabs  $dst,$src\t# vector (2S)" %}
16416   ins_encode %{
16417     __ fabs(as_FloatRegister($dst$$reg), __ T2S,
16418             as_FloatRegister($src$$reg));
16419   %}
16420   ins_pipe(vunop_fp64);
16421 %}
16422 
16423 instruct vabs4F(vecX dst, vecX src)
16424 %{
16425   predicate(n->as_Vector()->length() == 4);
16426   match(Set dst (AbsVF src));
16427   ins_cost(INSN_COST * 3);
16428   format %{ "fabs  $dst,$src\t# vector (4S)" %}
16429   ins_encode %{
16430     __ fabs(as_FloatRegister($dst$$reg), __ T4S,
16431             as_FloatRegister($src$$reg));
16432   %}
16433   ins_pipe(vunop_fp128);
16434 %}
16435 
16436 instruct vabs2D(vecX dst, vecX src)
16437 %{
16438   predicate(n->as_Vector()->length() == 2);
16439   match(Set dst (AbsVD src));
16440   ins_cost(INSN_COST * 3);
16441   format %{ "fabs  $dst,$src\t# vector (2D)" %}
16442   ins_encode %{
16443     __ fabs(as_FloatRegister($dst$$reg), __ T2D,
16444             as_FloatRegister($src$$reg));
16445   %}
16446   ins_pipe(vunop_fp128);
16447 %}
16448 
16449 // --------------------------------- NEG --------------------------------------
16450 
16451 instruct vneg2F(vecD dst, vecD src)
16452 %{
16453   predicate(n->as_Vector()->length() == 2);
16454   match(Set dst (NegVF src));
16455   ins_cost(INSN_COST * 3);
16456   format %{ "fneg  $dst,$src\t# vector (2S)" %}
16457   ins_encode %{
16458     __ fneg(as_FloatRegister($dst$$reg), __ T2S,
16459             as_FloatRegister($src$$reg));
16460   %}
16461   ins_pipe(vunop_fp64);
16462 %}
16463 
16464 instruct vneg4F(vecX dst, vecX src)
16465 %{
16466   predicate(n->as_Vector()->length() == 4);
16467   match(Set dst (NegVF src));
16468   ins_cost(INSN_COST * 3);
16469   format %{ "fneg  $dst,$src\t# vector (4S)" %}
16470   ins_encode %{
16471     __ fneg(as_FloatRegister($dst$$reg), __ T4S,
16472             as_FloatRegister($src$$reg));
16473   %}
16474   ins_pipe(vunop_fp128);
16475 %}
16476 
16477 instruct vneg2D(vecX dst, vecX src)
16478 %{
16479   predicate(n->as_Vector()->length() == 2);
16480   match(Set dst (NegVD src));
16481   ins_cost(INSN_COST * 3);
16482   format %{ "fneg  $dst,$src\t# vector (2D)" %}
16483   ins_encode %{
16484     __ fneg(as_FloatRegister($dst$$reg), __ T2D,
16485             as_FloatRegister($src$$reg));
16486   %}
16487   ins_pipe(vunop_fp128);
16488 %}
16489 
16490 // --------------------------------- AND --------------------------------------
16491 
16492 instruct vand8B(vecD dst, vecD src1, vecD src2)
16493 %{
16494   predicate(n->as_Vector()->length_in_bytes() == 4 ||
16495             n->as_Vector()->length_in_bytes() == 8);
16496   match(Set dst (AndV src1 src2));
16497   ins_cost(INSN_COST);
16498   format %{ "and  $dst,$src1,$src2\t# vector (8B)" %}
16499   ins_encode %{
16500     __ andr(as_FloatRegister($dst$$reg), __ T8B,
16501             as_FloatRegister($src1$$reg),
16502             as_FloatRegister($src2$$reg));
16503   %}
16504   ins_pipe(vlogical64);
16505 %}
16506 
16507 instruct vand16B(vecX dst, vecX src1, vecX src2)
16508 %{
16509   predicate(n->as_Vector()->length_in_bytes() == 16);
16510   match(Set dst (AndV src1 src2));
16511   ins_cost(INSN_COST);
16512   format %{ "and  $dst,$src1,$src2\t# vector (16B)" %}
16513   ins_encode %{
16514     __ andr(as_FloatRegister($dst$$reg), __ T16B,
16515             as_FloatRegister($src1$$reg),
16516             as_FloatRegister($src2$$reg));
16517   %}
16518   ins_pipe(vlogical128);
16519 %}
16520 
16521 // --------------------------------- OR ---------------------------------------
16522 
16523 instruct vor8B(vecD dst, vecD src1, vecD src2)
16524 %{
16525   predicate(n->as_Vector()->length_in_bytes() == 4 ||
16526             n->as_Vector()->length_in_bytes() == 8);
16527   match(Set dst (OrV src1 src2));
16528   ins_cost(INSN_COST);
16529   format %{ "and  $dst,$src1,$src2\t# vector (8B)" %}
16530   ins_encode %{
16531     __ orr(as_FloatRegister($dst$$reg), __ T8B,
16532             as_FloatRegister($src1$$reg),
16533             as_FloatRegister($src2$$reg));
16534   %}
16535   ins_pipe(vlogical64);
16536 %}
16537 
16538 instruct vor16B(vecX dst, vecX src1, vecX src2)
16539 %{
16540   predicate(n->as_Vector()->length_in_bytes() == 16);
16541   match(Set dst (OrV src1 src2));
16542   ins_cost(INSN_COST);
16543   format %{ "orr  $dst,$src1,$src2\t# vector (16B)" %}
16544   ins_encode %{
16545     __ orr(as_FloatRegister($dst$$reg), __ T16B,
16546             as_FloatRegister($src1$$reg),
16547             as_FloatRegister($src2$$reg));
16548   %}
16549   ins_pipe(vlogical128);
16550 %}
16551 
16552 // --------------------------------- XOR --------------------------------------
16553 
16554 instruct vxor8B(vecD dst, vecD src1, vecD src2)
16555 %{
16556   predicate(n->as_Vector()->length_in_bytes() == 4 ||
16557             n->as_Vector()->length_in_bytes() == 8);
16558   match(Set dst (XorV src1 src2));
16559   ins_cost(INSN_COST);
16560   format %{ "xor  $dst,$src1,$src2\t# vector (8B)" %}
16561   ins_encode %{
16562     __ eor(as_FloatRegister($dst$$reg), __ T8B,
16563             as_FloatRegister($src1$$reg),
16564             as_FloatRegister($src2$$reg));
16565   %}
16566   ins_pipe(vlogical64);
16567 %}
16568 
16569 instruct vxor16B(vecX dst, vecX src1, vecX src2)
16570 %{
16571   predicate(n->as_Vector()->length_in_bytes() == 16);
16572   match(Set dst (XorV src1 src2));
16573   ins_cost(INSN_COST);
16574   format %{ "xor  $dst,$src1,$src2\t# vector (16B)" %}
16575   ins_encode %{
16576     __ eor(as_FloatRegister($dst$$reg), __ T16B,
16577             as_FloatRegister($src1$$reg),
16578             as_FloatRegister($src2$$reg));
16579   %}
16580   ins_pipe(vlogical128);
16581 %}
16582 
16583 // ------------------------------ Shift ---------------------------------------
16584 instruct vshiftcnt8B(vecD dst, iRegIorL2I cnt) %{
16585   predicate(n->as_Vector()->length_in_bytes() == 8);
16586   match(Set dst (LShiftCntV cnt));
16587   match(Set dst (RShiftCntV cnt));
16588   format %{ "dup  $dst, $cnt\t# shift count vector (8B)" %}
16589   ins_encode %{
16590     __ dup(as_FloatRegister($dst$$reg), __ T8B, as_Register($cnt$$reg));
16591   %}
16592   ins_pipe(vdup_reg_reg64);
16593 %}
16594 
16595 instruct vshiftcnt16B(vecX dst, iRegIorL2I cnt) %{
16596   predicate(n->as_Vector()->length_in_bytes() == 16);
16597   match(Set dst (LShiftCntV cnt));
16598   match(Set dst (RShiftCntV cnt));
16599   format %{ "dup  $dst, $cnt\t# shift count vector (16B)" %}
16600   ins_encode %{
16601     __ dup(as_FloatRegister($dst$$reg), __ T16B, as_Register($cnt$$reg));
16602   %}
16603   ins_pipe(vdup_reg_reg128);
16604 %}
16605 
16606 instruct vsll8B(vecD dst, vecD src, vecD shift) %{
16607   predicate(n->as_Vector()->length() == 4 ||
16608             n->as_Vector()->length() == 8);
16609   match(Set dst (LShiftVB src shift));
16610   ins_cost(INSN_COST);
16611   format %{ "sshl  $dst,$src,$shift\t# vector (8B)" %}
16612   ins_encode %{
16613     __ sshl(as_FloatRegister($dst$$reg), __ T8B,
16614             as_FloatRegister($src$$reg),
16615             as_FloatRegister($shift$$reg));
16616   %}
16617   ins_pipe(vshift64);
16618 %}
16619 
16620 instruct vsll16B(vecX dst, vecX src, vecX shift) %{
16621   predicate(n->as_Vector()->length() == 16);
16622   match(Set dst (LShiftVB src shift));
16623   ins_cost(INSN_COST);
16624   format %{ "sshl  $dst,$src,$shift\t# vector (16B)" %}
16625   ins_encode %{
16626     __ sshl(as_FloatRegister($dst$$reg), __ T16B,
16627             as_FloatRegister($src$$reg),
16628             as_FloatRegister($shift$$reg));
16629   %}
16630   ins_pipe(vshift128);
16631 %}
16632 
16633 // Right shifts with vector shift count on aarch64 SIMD are implemented
16634 // as left shift by negative shift count.
16635 // There are two cases for vector shift count.
16636 //
16637 // Case 1: The vector shift count is from replication.
16638 //        |            |
16639 //    LoadVector  RShiftCntV
16640 //        |       /
16641 //     RShiftVI
16642 // Note: In inner loop, multiple neg instructions are used, which can be
16643 // moved to outer loop and merge into one neg instruction.
16644 //
16645 // Case 2: The vector shift count is from loading.
16646 // This case isn't supported by middle-end now. But it's supported by
16647 // panama/vectorIntrinsics(JEP 338: Vector API).
16648 //        |            |
16649 //    LoadVector  LoadVector
16650 //        |       /
16651 //     RShiftVI
16652 //
16653 
16654 instruct vsra8B(vecD dst, vecD src, vecD shift, vecD tmp) %{
16655   predicate(n->as_Vector()->length() == 4 ||
16656             n->as_Vector()->length() == 8);
16657   match(Set dst (RShiftVB src shift));
16658   ins_cost(INSN_COST);
16659   effect(TEMP tmp);
16660   format %{ "negr  $tmp,$shift\t"
16661             "sshl  $dst,$src,$tmp\t# vector (8B)" %}
16662   ins_encode %{
16663     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
16664             as_FloatRegister($shift$$reg));
16665     __ sshl(as_FloatRegister($dst$$reg), __ T8B,
16666             as_FloatRegister($src$$reg),
16667             as_FloatRegister($tmp$$reg));
16668   %}
16669   ins_pipe(vshift64);
16670 %}
16671 
16672 instruct vsra16B(vecX dst, vecX src, vecX shift, vecX tmp) %{
16673   predicate(n->as_Vector()->length() == 16);
16674   match(Set dst (RShiftVB src shift));
16675   ins_cost(INSN_COST);
16676   effect(TEMP tmp);
16677   format %{ "negr  $tmp,$shift\t"
16678             "sshl  $dst,$src,$tmp\t# vector (16B)" %}
16679   ins_encode %{
16680     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
16681             as_FloatRegister($shift$$reg));
16682     __ sshl(as_FloatRegister($dst$$reg), __ T16B,
16683             as_FloatRegister($src$$reg),
16684             as_FloatRegister($tmp$$reg));
16685   %}
16686   ins_pipe(vshift128);
16687 %}
16688 
16689 instruct vsrl8B(vecD dst, vecD src, vecD shift, vecD tmp) %{
16690   predicate(n->as_Vector()->length() == 4 ||
16691             n->as_Vector()->length() == 8);
16692   match(Set dst (URShiftVB src shift));
16693   ins_cost(INSN_COST);
16694   effect(TEMP tmp);
16695   format %{ "negr  $tmp,$shift\t"
16696             "ushl  $dst,$src,$tmp\t# vector (8B)" %}
16697   ins_encode %{
16698     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
16699             as_FloatRegister($shift$$reg));
16700     __ ushl(as_FloatRegister($dst$$reg), __ T8B,
16701             as_FloatRegister($src$$reg),
16702             as_FloatRegister($tmp$$reg));
16703   %}
16704   ins_pipe(vshift64);
16705 %}
16706 
16707 instruct vsrl16B(vecX dst, vecX src, vecX shift, vecX tmp) %{
16708   predicate(n->as_Vector()->length() == 16);
16709   match(Set dst (URShiftVB src shift));
16710   ins_cost(INSN_COST);
16711   effect(TEMP tmp);
16712   format %{ "negr  $tmp,$shift\t"
16713             "ushl  $dst,$src,$tmp\t# vector (16B)" %}
16714   ins_encode %{
16715     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
16716             as_FloatRegister($shift$$reg));
16717     __ ushl(as_FloatRegister($dst$$reg), __ T16B,
16718             as_FloatRegister($src$$reg),
16719             as_FloatRegister($tmp$$reg));
16720   %}
16721   ins_pipe(vshift128);
16722 %}
16723 
16724 instruct vsll8B_imm(vecD dst, vecD src, immI shift) %{
16725   predicate(n->as_Vector()->length() == 4 ||
16726             n->as_Vector()->length() == 8);
16727   match(Set dst (LShiftVB src shift));
16728   ins_cost(INSN_COST);
16729   format %{ "shl    $dst, $src, $shift\t# vector (8B)" %}
16730   ins_encode %{
16731     int sh = (int)$shift$$constant;
16732     if (sh >= 8) {
16733       __ eor(as_FloatRegister($dst$$reg), __ T8B,
16734              as_FloatRegister($src$$reg),
16735              as_FloatRegister($src$$reg));
16736     } else {
16737       __ shl(as_FloatRegister($dst$$reg), __ T8B,
16738              as_FloatRegister($src$$reg), sh);
16739     }
16740   %}
16741   ins_pipe(vshift64_imm);
16742 %}
16743 
16744 instruct vsll16B_imm(vecX dst, vecX src, immI shift) %{
16745   predicate(n->as_Vector()->length() == 16);
16746   match(Set dst (LShiftVB src shift));
16747   ins_cost(INSN_COST);
16748   format %{ "shl    $dst, $src, $shift\t# vector (16B)" %}
16749   ins_encode %{
16750     int sh = (int)$shift$$constant;
16751     if (sh >= 8) {
16752       __ eor(as_FloatRegister($dst$$reg), __ T16B,
16753              as_FloatRegister($src$$reg),
16754              as_FloatRegister($src$$reg));
16755     } else {
16756       __ shl(as_FloatRegister($dst$$reg), __ T16B,
16757              as_FloatRegister($src$$reg), sh);
16758     }
16759   %}
16760   ins_pipe(vshift128_imm);
16761 %}
16762 
16763 instruct vsra8B_imm(vecD dst, vecD src, immI shift) %{
16764   predicate(n->as_Vector()->length() == 4 ||
16765             n->as_Vector()->length() == 8);
16766   match(Set dst (RShiftVB src shift));
16767   ins_cost(INSN_COST);
16768   format %{ "sshr    $dst, $src, $shift\t# vector (8B)" %}
16769   ins_encode %{
16770     int sh = (int)$shift$$constant;
16771     if (sh >= 8) sh = 7;
16772     __ sshr(as_FloatRegister($dst$$reg), __ T8B,
16773            as_FloatRegister($src$$reg), sh);
16774   %}
16775   ins_pipe(vshift64_imm);
16776 %}
16777 
16778 instruct vsra16B_imm(vecX dst, vecX src, immI shift) %{
16779   predicate(n->as_Vector()->length() == 16);
16780   match(Set dst (RShiftVB src shift));
16781   ins_cost(INSN_COST);
16782   format %{ "sshr    $dst, $src, $shift\t# vector (16B)" %}
16783   ins_encode %{
16784     int sh = (int)$shift$$constant;
16785     if (sh >= 8) sh = 7;
16786     __ sshr(as_FloatRegister($dst$$reg), __ T16B,
16787            as_FloatRegister($src$$reg), sh);
16788   %}
16789   ins_pipe(vshift128_imm);
16790 %}
16791 
16792 instruct vsrl8B_imm(vecD dst, vecD src, immI shift) %{
16793   predicate(n->as_Vector()->length() == 4 ||
16794             n->as_Vector()->length() == 8);
16795   match(Set dst (URShiftVB src shift));
16796   ins_cost(INSN_COST);
16797   format %{ "ushr    $dst, $src, $shift\t# vector (8B)" %}
16798   ins_encode %{
16799     int sh = (int)$shift$$constant;
16800     if (sh >= 8) {
16801       __ eor(as_FloatRegister($dst$$reg), __ T8B,
16802              as_FloatRegister($src$$reg),
16803              as_FloatRegister($src$$reg));
16804     } else {
16805       __ ushr(as_FloatRegister($dst$$reg), __ T8B,
16806              as_FloatRegister($src$$reg), sh);
16807     }
16808   %}
16809   ins_pipe(vshift64_imm);
16810 %}
16811 
16812 instruct vsrl16B_imm(vecX dst, vecX src, immI shift) %{
16813   predicate(n->as_Vector()->length() == 16);
16814   match(Set dst (URShiftVB src shift));
16815   ins_cost(INSN_COST);
16816   format %{ "ushr    $dst, $src, $shift\t# vector (16B)" %}
16817   ins_encode %{
16818     int sh = (int)$shift$$constant;
16819     if (sh >= 8) {
16820       __ eor(as_FloatRegister($dst$$reg), __ T16B,
16821              as_FloatRegister($src$$reg),
16822              as_FloatRegister($src$$reg));
16823     } else {
16824       __ ushr(as_FloatRegister($dst$$reg), __ T16B,
16825              as_FloatRegister($src$$reg), sh);
16826     }
16827   %}
16828   ins_pipe(vshift128_imm);
16829 %}
16830 
16831 instruct vsll4S(vecD dst, vecD src, vecD shift) %{
16832   predicate(n->as_Vector()->length() == 2 ||
16833             n->as_Vector()->length() == 4);
16834   match(Set dst (LShiftVS src shift));
16835   ins_cost(INSN_COST);
16836   format %{ "sshl  $dst,$src,$shift\t# vector (4H)" %}
16837   ins_encode %{
16838     __ sshl(as_FloatRegister($dst$$reg), __ T4H,
16839             as_FloatRegister($src$$reg),
16840             as_FloatRegister($shift$$reg));
16841   %}
16842   ins_pipe(vshift64);
16843 %}
16844 
16845 instruct vsll8S(vecX dst, vecX src, vecX shift) %{
16846   predicate(n->as_Vector()->length() == 8);
16847   match(Set dst (LShiftVS src shift));
16848   ins_cost(INSN_COST);
16849   format %{ "sshl  $dst,$src,$shift\t# vector (8H)" %}
16850   ins_encode %{
16851     __ sshl(as_FloatRegister($dst$$reg), __ T8H,
16852             as_FloatRegister($src$$reg),
16853             as_FloatRegister($shift$$reg));
16854   %}
16855   ins_pipe(vshift128);
16856 %}
16857 
16858 instruct vsra4S(vecD dst, vecD src, vecD shift, vecD tmp) %{
16859   predicate(n->as_Vector()->length() == 2 ||
16860             n->as_Vector()->length() == 4);
16861   match(Set dst (RShiftVS src shift));
16862   ins_cost(INSN_COST);
16863   effect(TEMP tmp);
16864   format %{ "negr  $tmp,$shift\t"
16865             "sshl  $dst,$src,$tmp\t# vector (4H)" %}
16866   ins_encode %{
16867     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
16868             as_FloatRegister($shift$$reg));
16869     __ sshl(as_FloatRegister($dst$$reg), __ T4H,
16870             as_FloatRegister($src$$reg),
16871             as_FloatRegister($tmp$$reg));
16872   %}
16873   ins_pipe(vshift64);
16874 %}
16875 
16876 instruct vsra8S(vecX dst, vecX src, vecX shift, vecX tmp) %{
16877   predicate(n->as_Vector()->length() == 8);
16878   match(Set dst (RShiftVS src shift));
16879   ins_cost(INSN_COST);
16880   effect(TEMP tmp);
16881   format %{ "negr  $tmp,$shift\t"
16882             "sshl  $dst,$src,$tmp\t# vector (8H)" %}
16883   ins_encode %{
16884     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
16885             as_FloatRegister($shift$$reg));
16886     __ sshl(as_FloatRegister($dst$$reg), __ T8H,
16887             as_FloatRegister($src$$reg),
16888             as_FloatRegister($tmp$$reg));
16889   %}
16890   ins_pipe(vshift128);
16891 %}
16892 
16893 instruct vsrl4S(vecD dst, vecD src, vecD shift, vecD tmp) %{
16894   predicate(n->as_Vector()->length() == 2 ||
16895             n->as_Vector()->length() == 4);
16896   match(Set dst (URShiftVS src shift));
16897   ins_cost(INSN_COST);
16898   effect(TEMP tmp);
16899   format %{ "negr  $tmp,$shift\t"
16900             "ushl  $dst,$src,$tmp\t# vector (4H)" %}
16901   ins_encode %{
16902     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
16903             as_FloatRegister($shift$$reg));
16904     __ ushl(as_FloatRegister($dst$$reg), __ T4H,
16905             as_FloatRegister($src$$reg),
16906             as_FloatRegister($tmp$$reg));
16907   %}
16908   ins_pipe(vshift64);
16909 %}
16910 
16911 instruct vsrl8S(vecX dst, vecX src, vecX shift, vecX tmp) %{
16912   predicate(n->as_Vector()->length() == 8);
16913   match(Set dst (URShiftVS src shift));
16914   ins_cost(INSN_COST);
16915   effect(TEMP tmp);
16916   format %{ "negr  $tmp,$shift\t"
16917             "ushl  $dst,$src,$tmp\t# vector (8H)" %}
16918   ins_encode %{
16919     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
16920             as_FloatRegister($shift$$reg));
16921     __ ushl(as_FloatRegister($dst$$reg), __ T8H,
16922             as_FloatRegister($src$$reg),
16923             as_FloatRegister($tmp$$reg));
16924   %}
16925   ins_pipe(vshift128);
16926 %}
16927 
16928 instruct vsll4S_imm(vecD dst, vecD src, immI shift) %{
16929   predicate(n->as_Vector()->length() == 2 ||
16930             n->as_Vector()->length() == 4);
16931   match(Set dst (LShiftVS src shift));
16932   ins_cost(INSN_COST);
16933   format %{ "shl    $dst, $src, $shift\t# vector (4H)" %}
16934   ins_encode %{
16935     int sh = (int)$shift$$constant;
16936     if (sh >= 16) {
16937       __ eor(as_FloatRegister($dst$$reg), __ T8B,
16938              as_FloatRegister($src$$reg),
16939              as_FloatRegister($src$$reg));
16940     } else {
16941       __ shl(as_FloatRegister($dst$$reg), __ T4H,
16942              as_FloatRegister($src$$reg), sh);
16943     }
16944   %}
16945   ins_pipe(vshift64_imm);
16946 %}
16947 
16948 instruct vsll8S_imm(vecX dst, vecX src, immI shift) %{
16949   predicate(n->as_Vector()->length() == 8);
16950   match(Set dst (LShiftVS src shift));
16951   ins_cost(INSN_COST);
16952   format %{ "shl    $dst, $src, $shift\t# vector (8H)" %}
16953   ins_encode %{
16954     int sh = (int)$shift$$constant;
16955     if (sh >= 16) {
16956       __ eor(as_FloatRegister($dst$$reg), __ T16B,
16957              as_FloatRegister($src$$reg),
16958              as_FloatRegister($src$$reg));
16959     } else {
16960       __ shl(as_FloatRegister($dst$$reg), __ T8H,
16961              as_FloatRegister($src$$reg), sh);
16962     }
16963   %}
16964   ins_pipe(vshift128_imm);
16965 %}
16966 
16967 instruct vsra4S_imm(vecD dst, vecD src, immI shift) %{
16968   predicate(n->as_Vector()->length() == 2 ||
16969             n->as_Vector()->length() == 4);
16970   match(Set dst (RShiftVS src shift));
16971   ins_cost(INSN_COST);
16972   format %{ "sshr    $dst, $src, $shift\t# vector (4H)" %}
16973   ins_encode %{
16974     int sh = (int)$shift$$constant;
16975     if (sh >= 16) sh = 15;
16976     __ sshr(as_FloatRegister($dst$$reg), __ T4H,
16977            as_FloatRegister($src$$reg), sh);
16978   %}
16979   ins_pipe(vshift64_imm);
16980 %}
16981 
16982 instruct vsra8S_imm(vecX dst, vecX src, immI shift) %{
16983   predicate(n->as_Vector()->length() == 8);
16984   match(Set dst (RShiftVS src shift));
16985   ins_cost(INSN_COST);
16986   format %{ "sshr    $dst, $src, $shift\t# vector (8H)" %}
16987   ins_encode %{
16988     int sh = (int)$shift$$constant;
16989     if (sh >= 16) sh = 15;
16990     __ sshr(as_FloatRegister($dst$$reg), __ T8H,
16991            as_FloatRegister($src$$reg), sh);
16992   %}
16993   ins_pipe(vshift128_imm);
16994 %}
16995 
16996 instruct vsrl4S_imm(vecD dst, vecD src, immI shift) %{
16997   predicate(n->as_Vector()->length() == 2 ||
16998             n->as_Vector()->length() == 4);
16999   match(Set dst (URShiftVS src shift));
17000   ins_cost(INSN_COST);
17001   format %{ "ushr    $dst, $src, $shift\t# vector (4H)" %}
17002   ins_encode %{
17003     int sh = (int)$shift$$constant;
17004     if (sh >= 16) {
17005       __ eor(as_FloatRegister($dst$$reg), __ T8B,
17006              as_FloatRegister($src$$reg),
17007              as_FloatRegister($src$$reg));
17008     } else {
17009       __ ushr(as_FloatRegister($dst$$reg), __ T4H,
17010              as_FloatRegister($src$$reg), sh);
17011     }
17012   %}
17013   ins_pipe(vshift64_imm);
17014 %}
17015 
17016 instruct vsrl8S_imm(vecX dst, vecX src, immI shift) %{
17017   predicate(n->as_Vector()->length() == 8);
17018   match(Set dst (URShiftVS src shift));
17019   ins_cost(INSN_COST);
17020   format %{ "ushr    $dst, $src, $shift\t# vector (8H)" %}
17021   ins_encode %{
17022     int sh = (int)$shift$$constant;
17023     if (sh >= 16) {
17024       __ eor(as_FloatRegister($dst$$reg), __ T16B,
17025              as_FloatRegister($src$$reg),
17026              as_FloatRegister($src$$reg));
17027     } else {
17028       __ ushr(as_FloatRegister($dst$$reg), __ T8H,
17029              as_FloatRegister($src$$reg), sh);
17030     }
17031   %}
17032   ins_pipe(vshift128_imm);
17033 %}
17034 
17035 instruct vsll2I(vecD dst, vecD src, vecD shift) %{
17036   predicate(n->as_Vector()->length() == 2);
17037   match(Set dst (LShiftVI src shift));
17038   ins_cost(INSN_COST);
17039   format %{ "sshl  $dst,$src,$shift\t# vector (2S)" %}
17040   ins_encode %{
17041     __ sshl(as_FloatRegister($dst$$reg), __ T2S,
17042             as_FloatRegister($src$$reg),
17043             as_FloatRegister($shift$$reg));
17044   %}
17045   ins_pipe(vshift64);
17046 %}
17047 
17048 instruct vsll4I(vecX dst, vecX src, vecX shift) %{
17049   predicate(n->as_Vector()->length() == 4);
17050   match(Set dst (LShiftVI src shift));
17051   ins_cost(INSN_COST);
17052   format %{ "sshl  $dst,$src,$shift\t# vector (4S)" %}
17053   ins_encode %{
17054     __ sshl(as_FloatRegister($dst$$reg), __ T4S,
17055             as_FloatRegister($src$$reg),
17056             as_FloatRegister($shift$$reg));
17057   %}
17058   ins_pipe(vshift128);
17059 %}
17060 
17061 instruct vsra2I(vecD dst, vecD src, vecD shift, vecD tmp) %{
17062   predicate(n->as_Vector()->length() == 2);
17063   match(Set dst (RShiftVI src shift));
17064   ins_cost(INSN_COST);
17065   effect(TEMP tmp);
17066   format %{ "negr  $tmp,$shift\t"
17067             "sshl  $dst,$src,$tmp\t# vector (2S)" %}
17068   ins_encode %{
17069     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
17070             as_FloatRegister($shift$$reg));
17071     __ sshl(as_FloatRegister($dst$$reg), __ T2S,
17072             as_FloatRegister($src$$reg),
17073             as_FloatRegister($tmp$$reg));
17074   %}
17075   ins_pipe(vshift64);
17076 %}
17077 
17078 instruct vsra4I(vecX dst, vecX src, vecX shift, vecX tmp) %{
17079   predicate(n->as_Vector()->length() == 4);
17080   match(Set dst (RShiftVI src shift));
17081   ins_cost(INSN_COST);
17082   effect(TEMP tmp);
17083   format %{ "negr  $tmp,$shift\t"
17084             "sshl  $dst,$src,$tmp\t# vector (4S)" %}
17085   ins_encode %{
17086     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17087             as_FloatRegister($shift$$reg));
17088     __ sshl(as_FloatRegister($dst$$reg), __ T4S,
17089             as_FloatRegister($src$$reg),
17090             as_FloatRegister($tmp$$reg));
17091   %}
17092   ins_pipe(vshift128);
17093 %}
17094 
17095 instruct vsrl2I(vecD dst, vecD src, vecD shift, vecD tmp) %{
17096   predicate(n->as_Vector()->length() == 2);
17097   match(Set dst (URShiftVI src shift));
17098   ins_cost(INSN_COST);
17099   effect(TEMP tmp);
17100   format %{ "negr  $tmp,$shift\t"
17101             "ushl  $dst,$src,$tmp\t# vector (2S)" %}
17102   ins_encode %{
17103     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
17104             as_FloatRegister($shift$$reg));
17105     __ ushl(as_FloatRegister($dst$$reg), __ T2S,
17106             as_FloatRegister($src$$reg),
17107             as_FloatRegister($tmp$$reg));
17108   %}
17109   ins_pipe(vshift64);
17110 %}
17111 
17112 instruct vsrl4I(vecX dst, vecX src, vecX shift, vecX tmp) %{
17113   predicate(n->as_Vector()->length() == 4);
17114   match(Set dst (URShiftVI src shift));
17115   ins_cost(INSN_COST);
17116   effect(TEMP tmp);
17117   format %{ "negr  $tmp,$shift\t"
17118             "ushl  $dst,$src,$tmp\t# vector (4S)" %}
17119   ins_encode %{
17120     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17121             as_FloatRegister($shift$$reg));
17122     __ ushl(as_FloatRegister($dst$$reg), __ T4S,
17123             as_FloatRegister($src$$reg),
17124             as_FloatRegister($tmp$$reg));
17125   %}
17126   ins_pipe(vshift128);
17127 %}
17128 
17129 instruct vsll2I_imm(vecD dst, vecD src, immI shift) %{
17130   predicate(n->as_Vector()->length() == 2);
17131   match(Set dst (LShiftVI src shift));
17132   ins_cost(INSN_COST);
17133   format %{ "shl    $dst, $src, $shift\t# vector (2S)" %}
17134   ins_encode %{
17135     __ shl(as_FloatRegister($dst$$reg), __ T2S,
17136            as_FloatRegister($src$$reg),
17137            (int)$shift$$constant);
17138   %}
17139   ins_pipe(vshift64_imm);
17140 %}
17141 
17142 instruct vsll4I_imm(vecX dst, vecX src, immI shift) %{
17143   predicate(n->as_Vector()->length() == 4);
17144   match(Set dst (LShiftVI src shift));
17145   ins_cost(INSN_COST);
17146   format %{ "shl    $dst, $src, $shift\t# vector (4S)" %}
17147   ins_encode %{
17148     __ shl(as_FloatRegister($dst$$reg), __ T4S,
17149            as_FloatRegister($src$$reg),
17150            (int)$shift$$constant);
17151   %}
17152   ins_pipe(vshift128_imm);
17153 %}
17154 
17155 instruct vsra2I_imm(vecD dst, vecD src, immI shift) %{
17156   predicate(n->as_Vector()->length() == 2);
17157   match(Set dst (RShiftVI src shift));
17158   ins_cost(INSN_COST);
17159   format %{ "sshr    $dst, $src, $shift\t# vector (2S)" %}
17160   ins_encode %{
17161     __ sshr(as_FloatRegister($dst$$reg), __ T2S,
17162             as_FloatRegister($src$$reg),
17163             (int)$shift$$constant);
17164   %}
17165   ins_pipe(vshift64_imm);
17166 %}
17167 
17168 instruct vsra4I_imm(vecX dst, vecX src, immI shift) %{
17169   predicate(n->as_Vector()->length() == 4);
17170   match(Set dst (RShiftVI src shift));
17171   ins_cost(INSN_COST);
17172   format %{ "sshr    $dst, $src, $shift\t# vector (4S)" %}
17173   ins_encode %{
17174     __ sshr(as_FloatRegister($dst$$reg), __ T4S,
17175             as_FloatRegister($src$$reg),
17176             (int)$shift$$constant);
17177   %}
17178   ins_pipe(vshift128_imm);
17179 %}
17180 
17181 instruct vsrl2I_imm(vecD dst, vecD src, immI shift) %{
17182   predicate(n->as_Vector()->length() == 2);
17183   match(Set dst (URShiftVI src shift));
17184   ins_cost(INSN_COST);
17185   format %{ "ushr    $dst, $src, $shift\t# vector (2S)" %}
17186   ins_encode %{
17187     __ ushr(as_FloatRegister($dst$$reg), __ T2S,
17188             as_FloatRegister($src$$reg),
17189             (int)$shift$$constant);
17190   %}
17191   ins_pipe(vshift64_imm);
17192 %}
17193 
17194 instruct vsrl4I_imm(vecX dst, vecX src, immI shift) %{
17195   predicate(n->as_Vector()->length() == 4);
17196   match(Set dst (URShiftVI src shift));
17197   ins_cost(INSN_COST);
17198   format %{ "ushr    $dst, $src, $shift\t# vector (4S)" %}
17199   ins_encode %{
17200     __ ushr(as_FloatRegister($dst$$reg), __ T4S,
17201             as_FloatRegister($src$$reg),
17202             (int)$shift$$constant);
17203   %}
17204   ins_pipe(vshift128_imm);
17205 %}
17206 
17207 instruct vsll2L(vecX dst, vecX src, vecX shift) %{
17208   predicate(n->as_Vector()->length() == 2);
17209   match(Set dst (LShiftVL src shift));
17210   ins_cost(INSN_COST);
17211   format %{ "sshl  $dst,$src,$shift\t# vector (2D)" %}
17212   ins_encode %{
17213     __ sshl(as_FloatRegister($dst$$reg), __ T2D,
17214             as_FloatRegister($src$$reg),
17215             as_FloatRegister($shift$$reg));
17216   %}
17217   ins_pipe(vshift128);
17218 %}
17219 
17220 instruct vsra2L(vecX dst, vecX src, vecX shift, vecX tmp) %{
17221   predicate(n->as_Vector()->length() == 2);
17222   match(Set dst (RShiftVL src shift));
17223   ins_cost(INSN_COST);
17224   effect(TEMP tmp);
17225   format %{ "negr  $tmp,$shift\t"
17226             "sshl  $dst,$src,$tmp\t# vector (2D)" %}
17227   ins_encode %{
17228     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17229             as_FloatRegister($shift$$reg));
17230     __ sshl(as_FloatRegister($dst$$reg), __ T2D,
17231             as_FloatRegister($src$$reg),
17232             as_FloatRegister($tmp$$reg));
17233   %}
17234   ins_pipe(vshift128);
17235 %}
17236 
17237 instruct vsrl2L(vecX dst, vecX src, vecX shift, vecX tmp) %{
17238   predicate(n->as_Vector()->length() == 2);
17239   match(Set dst (URShiftVL src shift));
17240   ins_cost(INSN_COST);
17241   effect(TEMP tmp);
17242   format %{ "negr  $tmp,$shift\t"
17243             "ushl  $dst,$src,$tmp\t# vector (2D)" %}
17244   ins_encode %{
17245     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17246             as_FloatRegister($shift$$reg));
17247     __ ushl(as_FloatRegister($dst$$reg), __ T2D,
17248             as_FloatRegister($src$$reg),
17249             as_FloatRegister($tmp$$reg));
17250   %}
17251   ins_pipe(vshift128);
17252 %}
17253 
17254 instruct vsll2L_imm(vecX dst, vecX src, immI shift) %{
17255   predicate(n->as_Vector()->length() == 2);
17256   match(Set dst (LShiftVL src shift));
17257   ins_cost(INSN_COST);
17258   format %{ "shl    $dst, $src, $shift\t# vector (2D)" %}
17259   ins_encode %{
17260     __ shl(as_FloatRegister($dst$$reg), __ T2D,
17261            as_FloatRegister($src$$reg),
17262            (int)$shift$$constant);
17263   %}
17264   ins_pipe(vshift128_imm);
17265 %}
17266 
17267 instruct vsra2L_imm(vecX dst, vecX src, immI shift) %{
17268   predicate(n->as_Vector()->length() == 2);
17269   match(Set dst (RShiftVL src shift));
17270   ins_cost(INSN_COST);
17271   format %{ "sshr    $dst, $src, $shift\t# vector (2D)" %}
17272   ins_encode %{
17273     __ sshr(as_FloatRegister($dst$$reg), __ T2D,
17274             as_FloatRegister($src$$reg),
17275             (int)$shift$$constant);
17276   %}
17277   ins_pipe(vshift128_imm);
17278 %}
17279 
17280 instruct vsrl2L_imm(vecX dst, vecX src, immI shift) %{
17281   predicate(n->as_Vector()->length() == 2);
17282   match(Set dst (URShiftVL src shift));
17283   ins_cost(INSN_COST);
17284   format %{ "ushr    $dst, $src, $shift\t# vector (2D)" %}
17285   ins_encode %{
17286     __ ushr(as_FloatRegister($dst$$reg), __ T2D,
17287             as_FloatRegister($src$$reg),
17288             (int)$shift$$constant);
17289   %}
17290   ins_pipe(vshift128_imm);
17291 %}
17292 
17293 instruct vmax2F(vecD dst, vecD src1, vecD src2)
17294 %{
17295   predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
17296   match(Set dst (MaxV src1 src2));
17297   ins_cost(INSN_COST);
17298   format %{ "fmax  $dst,$src1,$src2\t# vector (2F)" %}
17299   ins_encode %{
17300     __ fmax(as_FloatRegister($dst$$reg), __ T2S,
17301             as_FloatRegister($src1$$reg),
17302             as_FloatRegister($src2$$reg));
17303   %}
17304   ins_pipe(vdop_fp64);
17305 %}
17306 
17307 instruct vmax4F(vecX dst, vecX src1, vecX src2)
17308 %{
17309   predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
17310   match(Set dst (MaxV src1 src2));
17311   ins_cost(INSN_COST);
17312   format %{ "fmax  $dst,$src1,$src2\t# vector (4S)" %}
17313   ins_encode %{
17314     __ fmax(as_FloatRegister($dst$$reg), __ T4S,
17315             as_FloatRegister($src1$$reg),
17316             as_FloatRegister($src2$$reg));
17317   %}
17318   ins_pipe(vdop_fp128);
17319 %}
17320 
17321 instruct vmax2D(vecX dst, vecX src1, vecX src2)
17322 %{
17323   predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
17324   match(Set dst (MaxV src1 src2));
17325   ins_cost(INSN_COST);
17326   format %{ "fmax  $dst,$src1,$src2\t# vector (2D)" %}
17327   ins_encode %{
17328     __ fmax(as_FloatRegister($dst$$reg), __ T2D,
17329             as_FloatRegister($src1$$reg),
17330             as_FloatRegister($src2$$reg));
17331   %}
17332   ins_pipe(vdop_fp128);
17333 %}
17334 
17335 instruct vmin2F(vecD dst, vecD src1, vecD src2)
17336 %{
17337   predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
17338   match(Set dst (MinV src1 src2));
17339   ins_cost(INSN_COST);
17340   format %{ "fmin  $dst,$src1,$src2\t# vector (2F)" %}
17341   ins_encode %{
17342     __ fmin(as_FloatRegister($dst$$reg), __ T2S,
17343             as_FloatRegister($src1$$reg),
17344             as_FloatRegister($src2$$reg));
17345   %}
17346   ins_pipe(vdop_fp64);
17347 %}
17348 
17349 instruct vmin4F(vecX dst, vecX src1, vecX src2)
17350 %{
17351   predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
17352   match(Set dst (MinV src1 src2));
17353   ins_cost(INSN_COST);
17354   format %{ "fmin  $dst,$src1,$src2\t# vector (4S)" %}
17355   ins_encode %{
17356     __ fmin(as_FloatRegister($dst$$reg), __ T4S,
17357             as_FloatRegister($src1$$reg),
17358             as_FloatRegister($src2$$reg));
17359   %}
17360   ins_pipe(vdop_fp128);
17361 %}
17362 
17363 instruct vmin2D(vecX dst, vecX src1, vecX src2)
17364 %{
17365   predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
17366   match(Set dst (MinV src1 src2));
17367   ins_cost(INSN_COST);
17368   format %{ "fmin  $dst,$src1,$src2\t# vector (2D)" %}
17369   ins_encode %{
17370     __ fmin(as_FloatRegister($dst$$reg), __ T2D,
17371             as_FloatRegister($src1$$reg),
17372             as_FloatRegister($src2$$reg));
17373   %}
17374   ins_pipe(vdop_fp128);
17375 %}
17376 
17377 //----------PEEPHOLE RULES-----------------------------------------------------
17378 // These must follow all instruction definitions as they use the names
17379 // defined in the instructions definitions.
17380 //
17381 // peepmatch ( root_instr_name [preceding_instruction]* );
17382 //
17383 // peepconstraint %{
17384 // (instruction_number.operand_name relational_op instruction_number.operand_name
17385 //  [, ...] );
17386 // // instruction numbers are zero-based using left to right order in peepmatch
17387 //
17388 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17389 // // provide an instruction_number.operand_name for each operand that appears
17390 // // in the replacement instruction's match rule
17391 //
17392 // ---------VM FLAGS---------------------------------------------------------
17393 //
17394 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17395 //
17396 // Each peephole rule is given an identifying number starting with zero and
17397 // increasing by one in the order seen by the parser.  An individual peephole
17398 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17399 // on the command-line.
17400 //
17401 // ---------CURRENT LIMITATIONS----------------------------------------------
17402 //
17403 // Only match adjacent instructions in same basic block
17404 // Only equality constraints
17405 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17406 // Only one replacement instruction
17407 //
17408 // ---------EXAMPLE----------------------------------------------------------
17409 //
17410 // // pertinent parts of existing instructions in architecture description
17411 // instruct movI(iRegINoSp dst, iRegI src)
17412 // %{
17413 //   match(Set dst (CopyI src));
17414 // %}
17415 //
17416 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17417 // %{
17418 //   match(Set dst (AddI dst src));
17419 //   effect(KILL cr);
17420 // %}
17421 //
17422 // // Change (inc mov) to lea
17423 // peephole %{
17424 //   // increment preceeded by register-register move
17425 //   peepmatch ( incI_iReg movI );
17426 //   // require that the destination register of the increment
17427 //   // match the destination register of the move
17428 //   peepconstraint ( 0.dst == 1.dst );
17429 //   // construct a replacement instruction that sets
17430 //   // the destination to ( move's source register + one )
17431 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17432 // %}
17433 //
17434 
17435 // Implementation no longer uses movX instructions since
17436 // machine-independent system no longer uses CopyX nodes.
17437 //
17438 // peephole
17439 // %{
17440 //   peepmatch (incI_iReg movI);
17441 //   peepconstraint (0.dst == 1.dst);
17442 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17443 // %}
17444 
17445 // peephole
17446 // %{
17447 //   peepmatch (decI_iReg movI);
17448 //   peepconstraint (0.dst == 1.dst);
17449 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17450 // %}
17451 
17452 // peephole
17453 // %{
17454 //   peepmatch (addI_iReg_imm movI);
17455 //   peepconstraint (0.dst == 1.dst);
17456 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17457 // %}
17458 
17459 // peephole
17460 // %{
17461 //   peepmatch (incL_iReg movL);
17462 //   peepconstraint (0.dst == 1.dst);
17463 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17464 // %}
17465 
17466 // peephole
17467 // %{
17468 //   peepmatch (decL_iReg movL);
17469 //   peepconstraint (0.dst == 1.dst);
17470 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17471 // %}
17472 
17473 // peephole
17474 // %{
17475 //   peepmatch (addL_iReg_imm movL);
17476 //   peepconstraint (0.dst == 1.dst);
17477 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17478 // %}
17479 
17480 // peephole
17481 // %{
17482 //   peepmatch (addP_iReg_imm movP);
17483 //   peepconstraint (0.dst == 1.dst);
17484 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17485 // %}
17486 
17487 // // Change load of spilled value to only a spill
17488 // instruct storeI(memory mem, iRegI src)
17489 // %{
17490 //   match(Set mem (StoreI mem src));
17491 // %}
17492 //
17493 // instruct loadI(iRegINoSp dst, memory mem)
17494 // %{
17495 //   match(Set dst (LoadI mem));
17496 // %}
17497 //
17498 
17499 //----------SMARTSPILL RULES---------------------------------------------------
17500 // These must follow all instruction definitions as they use the names
17501 // defined in the instructions definitions.
17502 
17503 // Local Variables:
17504 // mode: c++
17505 // End: